diff --git a/Makefile b/Makefile index 57227d4c..0b9e4e3e 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ else ifeq ($(TM_GIT_TAG)$(TM_GIT_DIRTY),$(TM_BASE_VERSION)) else TM_VERSION ?= $(TM_BASE_VERSION)+g$(TM_GIT_SHA)$(TM_GIT_DIRTY) endif -CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++14 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) +CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++17 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) CFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c99 -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG) DEPFLAGS := -MD -MP DEPS := $(wildcard src/*.d src/external/*.d src/external/libdeflate/lib/*.d src/external/libdeflate/lib/*/*.d server/*.d test/*.d) @@ -156,13 +156,16 @@ test: \ test_append_vector \ test_attribute_store \ test_config_validator \ + test_correct \ test_deque_map \ test_geom \ test_helpers \ test_options_parser \ + test_ordered_batches \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ + test_shp_mem_tiles \ test_significant_tags \ test_sorted_node_store \ test_sorted_way_store \ @@ -193,6 +196,11 @@ src/config_schema.h: resources/config-schema.json src/config_validator.o: src/config_schema.h +test_correct: \ + src/geom.o \ + test/correct.test.o + $(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct + test_deque_map: \ test/deque_map.test.o $(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map @@ -223,6 +231,10 @@ test_options_parser: \ test/options_parser.test.o $(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser +test_ordered_batches: \ + test/ordered_batches.test.o + $(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches + test_osm_store: \ test/osm_store.test.o $(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store @@ -262,6 +274,22 @@ test_sorted_way_store: \ test/sorted_way_store.test.o $(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store +test_shp_mem_tiles: \ + src/attribute_store.o \ + src/coordinates.o \ + src/coordinates_geom.o \ + src/declutter.o \ + src/geom.o \ + src/mmap_allocator.o \ + src/output_object.o \ + src/pooled_string.o \ + src/shp_mem_tiles.o \ + src/tile_coordinates_set.o \ + src/tile_data.o \ + src/tile_sorting.o \ + test/shp_mem_tiles.test.o + $(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles + test_tile_coordinates_set: \ src/tile_coordinates_set.o \ test/tile_coordinates_set.test.o diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f7d94183..6df1463a 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -33,6 +33,7 @@ It also includes these global settings: * `basezoom` - the zoom level for which tilemaker will generate tiles internally (should usually be the same as `maxzoom`) * `include_ids` - whether you want to store the OpenStreetMap IDs for each way/node within your vector tiles. This option is not compatible with the merging options defined by the `combine_xxx` settings (see the dedicated paragraph below) * `compress` - for mbtiles output, whether to compress vector tiles (Any of "gzip","deflate" or "none"(default)). For pmtiles output, compression is hardcoded to gzip +* `compression_level` (optional) - how hard to compress the tiles, from 1 (fastest) to 12 (smallest); defaults to 6. On the planet with the OpenMapTiles profile, 1 gave a 1.8% larger file and a 3% shorter run * `name`, `version` and `description` - about your project (these are written into the MBTiles file) * `high_resolution` (optional) - whether to use extra coordinate precision at the maximum zoom level (makes tiles a bit bigger) * `bounding_box` (optional) - the bounding box to output, in [minlon, minlat, maxlon, maxlat] order @@ -210,6 +211,8 @@ To do that, you use these methods: * `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6). * `Score(number)`: set how important this point feature is (a whole number; anything else is rounded down), so that tilemaker can decide which zoom level to show it from. Only meaningful for points (i.e. features written with `Layer` for a node, or `LayerAsCentroid`) in a layer with `declutter_below` set - see 'Decluttering point features' below. * `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires Boost 1.67+. +* `MercatorArea()`: return the area of the current object on the Web Mercator plane, in square metres at the scale of the equator (so it grows towards the poles, as the map does). It is cheap to compute and tells how much of a tile the object covers, which makes it the right measure for minimum zoom rules. +* `MinZoomByArea(limit, thresholds)`: set the minimum zoom of the current object from its `MercatorArea()`: the first zoom z from `limit` on whose threshold the area exceeds, where `thresholds` lists the thresholds for zoom 1, 2, 3...; if there is none, one more than the number of thresholds. Unlike the same rule written in Lua with `MercatorArea()`, this keeps `way_function` replayable (see `way_memo` above). * `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon). The simplest possible function, to include roads/paths and nothing else, might look like this: @@ -238,6 +241,19 @@ If your Lua file causes an error due to mistaken syntax, you can test it at the `way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}` +`way_memo = true` lets tilemaker replay `way_function` instead of running it, for ways that read the same as an earlier way. This is only allowed if `way_function` is a function of what it reads with `Find`, `Holds`, `IsClosed` and `NextRelation` (no other state, such as global counters), and what it does is call tilemaker's functions (`Layer`, `Attribute`... `MinZoom`): tilemaker records which reads lead to which calls, and replays the calls for a way whose reads give the same results. Anything else (geometry queries such as `Area`, relation tags, `Id`...) makes ways with those reads run through Lua. A tag whose value is only tested for being empty and copied unchanged into some attributes can be declared in `way_memo_values`, e.g. `way_memo_values = { name = { "name:latin" } }`: ways are then replayed with their own value of the tag, instead of being told apart by it. An attribute that is set to the length of the value (in bytes, as Lua's `#value` or `value:len()`) is declared as `attribute = "length"`, e.g. `way_memo_values = { ref = { "ref", ref_length = "length" } }`. + +Two more declarations let the memo replay ways that use some of the other functions: + +* `way_memo_area = { 1000, 5000 }`: `way_function` uses `MercatorArea()` only to compare it (`<`, `<=`, `>`, `>=`, `==`, `~=`) with these numbers. The memo then tells ways apart by the results of these comparisons, not by the area (e.g. `if MercatorArea() < 1000 then MinZoom(8) else MinZoomByArea(0, thresholds) end`; `MinZoomByArea` needs no declaration, it is worked out for each way). +* `way_memo_relations = true`: `way_function` uses the relation id `NextRelation()` returns only to test whether there is one (`if not rel then break end`), and reads the relations' tags only with `FindInRelation` (no `RestartRelations`). The memo then tells ways apart by the roles and the relation tags read. + +As with `way_memo_values`, tilemaker can't check these promises: if `way_function` does more with the area or the relations, ways are replayed with the wrong calls. + +Multipolygon relations, which also run `way_function`, are replayed the same way (with a record of their own, apart from that of the ways): the promises above are about `way_function`, so they hold for them too. For a multipolygon, `NextRelation` and `FindInRelation` are about the relations it is a member of. + +`node_memo = true` and `node_memo_values`, `node_memo_area`, `node_memo_relations` do the same for `node_function` (with its own record). + `init_function(name, is_first)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. `is_first` will be true only the first time `init_function` is called. Other functions are described below and in RELATIONS.md. diff --git a/docs/RUNNING.md b/docs/RUNNING.md index ecdf44c5..cbdc3987 100644 --- a/docs/RUNNING.md +++ b/docs/RUNNING.md @@ -56,6 +56,10 @@ usage but runs faster. * `--shard-stores`: Group temporary storage by area. Reduces RAM usage on large files (e.g. whole planet) but runs slower. +On Linux, .pmtiles output is written with direct I/O (`O_DIRECT`), bypassing the page cache, which +is considerably faster on a fast disk. Where the file system doesn't support it, tilemaker +falls back to normal writes; setting the environment variable `TM_NO_DIRECT` forces normal writes. + You can also tell tilemaker to only look at .pbf objects with certain tags. If you're making a thematic map, this allows tilemaker to skip data it won't need. Specify this in your Lua file like one of these three examples: diff --git a/include/append_vector.h b/include/append_vector.h index 28b29a8b..01008564 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -2,6 +2,7 @@ #define _APPEND_VECTOR_H #include "mmap_allocator.h" +#include #include #include #include @@ -162,6 +163,10 @@ namespace AppendVectorNS { return el; } + const T& operator [](int idx) const { + return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE]; + } + Iterator begin() { return Iterator(*this, 0, 0); } @@ -184,6 +189,22 @@ namespace AppendVectorNS { count++; } + // The same as push_back of each of p[0, n) + void append(const T* p, size_t n) { + while (n > 0) { + auto& vec = vecs.back(); + if (vec.capacity() == 0) + vec.reserve(APPEND_VECTOR_SIZE); + const size_t take = std::min(n, vec.capacity() - vec.size()); + vec.insert(vec.end(), p, p + take); + p += take; n -= take; count += take; + if (vec.size() == vec.capacity()) { + vecs.push_back(std::vector>()); + vecs.back().reserve(APPEND_VECTOR_SIZE); + } + } + } + size_t count; std::deque>> vecs; }; diff --git a/include/attribute_store.h b/include/attribute_store.h index 295c495c..4c26e892 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include #include #include "pooled_string.h" @@ -30,11 +33,16 @@ class AttributeKeyStore { uint16_t key2index(const std::string& key); const std::string& getKey(uint16_t index) const; const std::string& getKeyUnsafe(uint16_t index) const; - void finalize() { finalized = true; } + // Position of the key in alphabetical order. Unlike the key index, which is + // handed out in first-use order (i.e. depends on thread scheduling), it is + // the same in every run. Only valid after finalize(). + uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; } + void finalize(); std::atomic keys2indexSize; private: bool finalized; + std::vector ranks; mutable std::mutex keys2indexMutex; // NB: we use a deque, not a vector, because a deque never invalidates // pointers to its members as long as you only push_back @@ -152,14 +160,29 @@ struct AttributePair { return true; } + // Hashes the string 8 bytes at a time (the hash only picks shards and hash table slots) + static uint64_t hashBytes(uint64_t h, const char* p, size_t n) { + h ^= n * 0x9E3779B97F4A7C15ULL; + while (n >= 8) { + uint64_t w; memcpy(&w, p, 8); + h = (h ^ w) * 0xff51afd7ed558ccdULL; h ^= h >> 32; + p += 8; n -= 8; + } + if (n > 0) { + uint64_t w = 0; memcpy(&w, p, n); + h = (h ^ w) * 0xc4ceb9fe1a85ec53ULL; h ^= h >> 29; + } + h *= 0x94d049bb133111ebULL; h ^= h >> 31; + return h; + } + size_t hash() const { std::size_t rv = minzoom; boost::hash_combine(rv, keyIndex); boost::hash_combine(rv, valueType); if(hasStringValue()) { - const char* data = pooledString().data(); - boost::hash_range(rv, data, data + pooledString().size()); + rv = hashBytes(rv, pooledString().data(), pooledString().size()); } else if(hasFloatValue()) boost::hash_combine(rv, doubleValue_); else if(hasIntValue()) @@ -216,6 +239,9 @@ class AttributePairStore { friend class AttributeStore; std::vector> pairs; bool finalized; + // Set (under its lock) once the hot shard is full: from then on it never changes, so it is + // looked up without the lock + std::atomic hotFull{false}; // We refer to all attribute pairs by index. // // Each shard is responsible for a portion of the key space. @@ -439,6 +465,24 @@ struct AttributeStore { } attrSet->forEachPair(fn); } + // As forEachPairIndexUnsafe, but in an order that is the same in every run + // (by key name), not by pair index, which depends on thread scheduling. + template + void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const { + boost::container::small_vector order; + forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); }); + if (order.size() > 1) + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return comparePairsUnsafe(a, b) < 0; + }); + for (const uint32_t pairIndex : order) + fn(pairIndex); + } + // Order and hash attribute pairs/sets by content (key names and values), not by + // index, so that they are the same in every run. Only valid after finalize(). + int comparePairsUnsafe(uint32_t a, uint32_t b) const; + int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const; + uint64_t contentHashUnsafe(AttributeIndex index) const; void reset(); // used for testing size_t size() const; void reportSize() const; @@ -453,6 +497,14 @@ struct AttributeStore { void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom); void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom); void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom); + // The pair index that addAttributeWithKeyIndex adds to the set. It is the same for the same + // arguments for as long as the store lives, so callers may keep it. + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, double v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, int v, char minzoom); + uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, bool v, char minzoom); + // Counts an object whose AttributeIndex the caller knew without add() (for reportSize) + void noteLookup(); AttributeStore(): finalized(false), diff --git a/include/clip_cache.h b/include/clip_cache.h index 66a400ab..e650671a 100644 --- a/include/clip_cache.h +++ b/include/clip_cache.h @@ -16,6 +16,9 @@ class TileBbox; extern thread_local uint64_t clipCacheBatchId; // Set when the current batch can't reuse clips (a single tile), so add() is skipped. extern thread_local bool clipCacheNoStore; +// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces): +// the batch cache is then neither read nor written. +extern thread_local bool clipCacheBypass; template class ClipCache { @@ -36,6 +39,8 @@ class ClipCache { } Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const { + if (clipCacheBypass) + return Entry(); // Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first) LocalStore& s = local(zoom, TileCoordinates(x, y)); Entry rv; @@ -59,7 +64,7 @@ class ClipCache { void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) { // The point of caching is to reuse the clip, so caching at the terminal zoom is // pointless. - if (bbox.zoom == baseZoom || clipCacheNoStore) + if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass) return; // Levels beyond MAX_LEVELS could never be looked up by a child tile anyway diff --git a/include/clip_line.h b/include/clip_line.h new file mode 100644 index 00000000..6c0c8519 --- /dev/null +++ b/include/clip_line.h @@ -0,0 +1,164 @@ +/*! \file */ +#ifndef _CLIP_LINE_H +#define _CLIP_LINE_H + +#include +#include +#include "geom.h" +#ifdef __SSE2__ +#include +#endif + +// buildWayGeometry for a linestring: splits `ls` into the runs of segments that intersect +// clippingBox (appended to `out`), and returns whether any point lies outside extBox (then the +// caller intersects the result with extBox). +// +// Reference version: a segment with both ends within the closed clippingBox always intersects +// it (Boost's cartesian segment/box disjoint test then sees t ranges that contain the whole +// segment in both dimensions: every ti_min <= 0 <= ti_max, diff >= 0), so Boost isn't called for +// those. Every other segment is tested with boost::geometry::intersects. +template +bool split_linestring_for_tile_reference(const LS &ls, Box const &clippingBox, Box const &extBox, MultiLinestring &out) { + const double minX = extBox.min_corner().x(), maxX = extBox.max_corner().x(); + const double minY = extBox.min_corner().y(), maxY = extBox.max_corner().y(); + auto pointInsideExtBox = [minX, maxX, minY, maxY](const Point& p) { + return p.x() >= minX && p.x() <= maxX && p.y() >= minY && p.y() <= maxY; + }; + const double cx0 = clippingBox.min_corner().x(), cx1 = clippingBox.max_corner().x(); + const double cy0 = clippingBox.min_corner().y(), cy1 = clippingBox.max_corner().y(); + auto coveredByClip = [cx0, cx1, cy0, cy1](const Point& p) { + return p.x() >= cx0 && p.x() <= cx1 && p.y() >= cy0 && p.y() <= cy1; + }; + bool needsIntersection = !pointInsideExtBox(ls[0]); + + Linestring current_ls; + current_ls.push_back(ls[0]); + + bool prevCovered = coveredByClip(ls[0]); + for (size_t i = 1; i < ls.size(); ++i) { + const bool covered = coveredByClip(ls[i]); + if (!(prevCovered && covered)) { + boost::geometry::model::segment segment(ls[i-1], ls[i]); + if (!boost::geometry::intersects(segment, clippingBox)) { + if (current_ls.size() > 1) + out.push_back(std::move(current_ls)); + current_ls.clear(); + } + } + prevCovered = covered; + current_ls.push_back(ls[i]); + if (!needsIntersection) + needsIntersection = !pointInsideExtBox(ls[i]); + } + + if (current_ls.size() > 1) + out.push_back(std::move(current_ls)); + return needsIntersection; +} + +// Fast version, same result. Per point, one SSE2 pass computes whether it is within the closed +// clippingBox / extBox and on which sides it lies more than M outside clippingBox. A segment +// then +// - continues the run if both ends are within clippingBox (as in the reference); +// - ends it if both ends are more than M left (right) of the box: Boost's x step returns +// disjoint. With the ends left, either p1 >= p0 and fl(x0 - p0) > fl(p1 - p0) (the exact +// values differ by x0 - p1 > M, far more than their rounding errors), or p1 < p0 and +// fl(p0 - x0) < 0. Neither value is near enough to 0 for the diff ~ 0 branch to replace it. +// Right: symmetric; +// - ends it if both ends are more than M below (above) the box, and the x extent s = |p1x - p0x| +// (as Boost computes it) is either exactly 0 or at least D: if the x step passes +// (t_min.first <= s, t_max.first >= 0), the y step either returns disjoint at once or updates +// t_min to (ti_min, dy) with ti_min > dy (below, p1y >= p0y: the products ti_min*s and +// t_min.first*dy differ by at least M*s, more than their relative rounding errors), resp. +// t_max to (ti_max, dy) with ti_max < 0 (below, p1y < p0y: ti_max*s < 0 <= t_max.first*dy, +// no underflow as s >= D), and then returns disjoint. With s exactly 0, the x step leaves +// t_min.first < 0 < t_max.first (it replaces values ~0 by -1/1), and the same updates +// happen. Above: symmetric; +// - is tested with boost::geometry::intersects otherwise, as in the reference. +// The margins need bounded coordinates (|c| <= BOUND); if the box or any point (NaN included) +// isn't, the reference version runs instead. Each run is copied into its part in one go (only +// the capacity of the parts differs from appending point by point). +namespace clip_line_detail { + constexpr double M = 1e-9, D = 1e-12, BOUND = 1000.0; +} + +template +bool split_linestring_for_tile(const LS &ls, Box const &clippingBox, Box const &extBox, MultiLinestring &out) { +#ifdef __SSE2__ + using namespace clip_line_detail; + const size_t n = ls.size(); + const double cx0 = clippingBox.min_corner().x(), cx1 = clippingBox.max_corner().x(); + const double cy0 = clippingBox.min_corner().y(), cy1 = clippingBox.max_corner().y(); + if (!(std::fabs(cx0) <= BOUND && std::fabs(cx1) <= BOUND && std::fabs(cy0) <= BOUND && std::fabs(cy1) <= BOUND)) + return split_linestring_for_tile_reference(ls, clippingBox, extBox, out); + const size_t outSize = out.size(); + + const __m128d cmin = _mm_set_pd(cy0, cx0), cmax = _mm_set_pd(cy1, cx1); + const __m128d emin = _mm_set_pd(extBox.min_corner().y(), extBox.min_corner().x()); + const __m128d emax = _mm_set_pd(extBox.max_corner().y(), extBox.max_corner().x()); + const __m128d smin = _mm_set_pd(cy0 - M, cx0 - M), smax = _mm_set_pd(cy1 + M, cx1 + M); + const __m128d absMask = _mm_castsi128_pd(_mm_set1_epi64x(0x7fffffffffffffffLL)), bound = _mm_set1_pd(BOUND); + __m128d bounded = _mm_castsi128_pd(_mm_set1_epi32(-1)); + // code bits (two per compare): 0-3 within clippingBox (x, y), 4-7 within extBox, 8-9 left, + // 10-11 below, 12-13 right, 14-15 above + auto code = [&](const Point &p) -> unsigned { + const __m128d v = _mm_loadu_pd(reinterpret_cast(&p)); + bounded = _mm_and_pd(bounded, _mm_cmple_pd(_mm_and_pd(v, absMask), bound)); + const __m128i in = _mm_castpd_si128(_mm_and_pd(_mm_cmpge_pd(v, cmin), _mm_cmple_pd(v, cmax))); + const __m128i inExt = _mm_castpd_si128(_mm_and_pd(_mm_cmpge_pd(v, emin), _mm_cmple_pd(v, emax))); + const __m128i lo = _mm_castpd_si128(_mm_cmplt_pd(v, smin)), hi = _mm_castpd_si128(_mm_cmpgt_pd(v, smax)); + return unsigned(_mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(in, inExt), _mm_packs_epi32(lo, hi)))); + }; + constexpr unsigned C_COV = 0xf, C_EXT = 0xf0, C_LR = 0x100 | 0x1000, C_BT = 0x400 | 0x4000; + + unsigned prev = code(ls[0]); + unsigned all = prev; + size_t start = 0; + for (size_t i = 1; i < n; ++i) { + const unsigned cur = code(ls[i]); + all &= cur; + const unsigned both = prev & cur; + prev = cur; + if ((both & C_COV) == C_COV) continue; + if (!(both & C_LR)) { + bool sure = false; + if (both & C_BT) { + const double x0 = ls[i-1].x(), x1 = ls[i].x(); + const double s = x1 >= x0 ? x1 - x0 : x0 - x1; + sure = s == 0 || s >= D; + } + if (!sure && boost::geometry::intersects(boost::geometry::model::segment(ls[i-1], ls[i]), clippingBox)) + continue; + } + // segment (i-1, i) doesn't intersect the box: the run so far ends at i-1 + if (i - 1 > start) + out.emplace_back(ls.begin() + start, ls.begin() + i); + start = i; + } + if (_mm_movemask_pd(bounded) != 3) { + // a point out of range (or NaN): the shortcuts may not match Boost + out.resize(outSize); + return split_linestring_for_tile_reference(ls, clippingBox, extBox, out); + } + if (n - 1 > start) + out.emplace_back(ls.begin() + start, ls.end()); + return (all & C_EXT) != C_EXT; +#else + return split_linestring_for_tile_reference(ls, clippingBox, extBox, out); +#endif +} + +// buildWayGeometry LINESTRING_: the parts of `ls` for the tile +template +MultiLinestring clip_linestring_for_tile(const LS &ls, Box const &clippingBox, Box const &extBox) { + MultiLinestring out; + if (ls.empty()) + return out; + if (!split_linestring_for_tile(ls, clippingBox, extBox, out)) + return out; + MultiLinestring result; + boost::geometry::intersection(out, extBox, result); + return result; +} + +#endif //_CLIP_LINE_H diff --git a/include/coordinates.h b/include/coordinates.h index d826cddd..ae756e9f 100644 --- a/include/coordinates.h +++ b/include/coordinates.h @@ -101,6 +101,9 @@ constexpr double MinLatp = -MaxLatp; // (if calling with raw coords, remember to divide/multiply by 10000000.0) double lat2latp(double lat); double latp2lat(double latp); +// int(lat2latp(lat / 1e7) * 1e7) for a latitude in 1e-7 degrees (as read from .osm.pbf), exactly +// the same value, fast (see coordinates.cpp) +int32_t lat2latpE7(int32_t lat); // Tile conversions double lon2tilexf(double lon, uint8_t z); diff --git a/include/coordinates_geom.h b/include/coordinates_geom.h index f233ad3e..f1964940 100644 --- a/include/coordinates_geom.h +++ b/include/coordinates_geom.h @@ -26,8 +26,11 @@ class TileBbox { TileBbox(TileCoordinates i, uint z, bool h, bool e); std::pair scaleLatpLon(double latp, double lon) const; + // scaleLatpLon() of every point of a line, without points repeating their predecessor + void scaleLine(std::vector> &dst, Linestring const &src) const; void scaleRing(Ring &dst, Ring const &src) const; Ring scaleRing(Ring const &src) const; + void scaleRingNoBacktrack(Ring &dst, Ring const &src) const; void scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const; MultiPolygon scaleGeometry(MultiPolygon const &src) const; std::pair floorLatpLon(double latp, double lon) const; diff --git a/include/deque_map.h b/include/deque_map.h index ea57f669..10007234 100644 --- a/include/deque_map.h +++ b/include/deque_map.h @@ -2,9 +2,10 @@ #define DEQUE_MAP_H #include -#include -#include +#include #include +#include +#include #include // A class which looks deep within the soul of some instance of @@ -13,6 +14,8 @@ // // Used to translate an 8-byte pointer into a 4-byte ID that can be // used repeatedly. +// +// T needs operator== and operator<, and a hash() member (std::string uses std::hash). template class DequeMap { public: @@ -28,71 +31,57 @@ class DequeMap { // add the item and return its index. // Otherwise, return -1. int32_t add(const T& entry) { - // Search to see if we've already got this entry. - const auto offsets = boost::irange(0, keys.size()); - const auto it = std::lower_bound( - offsets.begin(), - offsets.end(), - entry, - [&](const auto &e, auto id) { - return objects.at(keys[e]) < id; - } - ); - - // We do, return its index. - if (it != offsets.end() && objects[keys[*it]] == entry) - return keys[*it]; + return addWithHash(entry, rawHash(entry)); + } + // As add/find, with rawHash = entry.hash() (std::hash for std::string) given + int32_t addWithHash(const T& entry, uint64_t rawHash) { + const uint64_t h = mix(rawHash); + const int32_t found = lookup(entry, h); + if (found >= 0) + return found; + return addNew(entry, h); + } + int32_t findWithHash(const T& entry, uint64_t rawHash) const { + return lookup(entry, mix(rawHash)); + } + // As addWithHash, for an entry known not to be in the map + int32_t addAbsentWithHash(const T& entry, uint64_t rawHash) { + return addNew(entry, mix(rawHash)); + } + +private: + int32_t addNew(const T& entry, uint64_t h) { if (maxSize > 0 && objects.size() >= maxSize) return -1; // We don't, so store it... const uint32_t newIndex = objects.size(); objects.push_back(entry); - - // ...and add its index to our keys vector. - const uint32_t keysOffset = it == offsets.end() ? offsets.size() : *it; - - const uint32_t desiredSize = keys.size() + 1; - - // Amortize growth - if (keys.capacity() < desiredSize) - keys.reserve(keys.capacity() * 1.5); - - keys.resize(desiredSize); - - // Unless we're adding to the end, we need to shuffle existing keys down - // to make room for our new index. - if (keysOffset != newIndex) { - std::memmove(&keys[keysOffset + 1], &keys[keysOffset], sizeof(uint32_t) * (keys.size() - 1 - keysOffset)); + sortedValid = false; + + // ...and index it, growing the table to keep it at most half full + if (2 * (objects.size() + 1) > slots.size()) { + std::vector old(std::max(16, slots.size() * 2), 0); + slots.swap(old); + for (uint32_t i = 0; i < newIndex; i++) + insertSlot(hashOf(objects[i]), i); } - - keys[keysOffset] = newIndex; + insertSlot(h, newIndex); return newIndex; } +public: void clear() { objects.clear(); - keys.clear(); + slots.clear(); + sorted.clear(); + sortedValid = false; } // Returns the index of `entry` if present, -1 otherwise. int32_t find(const T& entry) const { - const auto offsets = boost::irange(0, keys.size()); - const auto it = std::lower_bound( - offsets.begin(), - offsets.end(), - entry, - [&](const auto &e, auto id) { - return objects.at(keys[e]) < id; - } - ); - - // We do, return its index. - if (it != offsets.end() && objects[keys[*it]] == entry) - return keys[*it]; - - return -1; + return lookup(entry, hashOf(entry)); } inline const T& operator[](uint32_t index) const { @@ -105,28 +94,69 @@ class DequeMap { size_t size() const { return objects.size(); } + // Iterates in the order of T's operator< struct iterator { const DequeMap& dm; size_t offset; iterator(const DequeMap& dm, size_t offset): dm(dm), offset(offset) {} void operator++() { offset++; } bool operator!=(iterator& other) { return offset != other.offset; } - const T& operator*() const { return dm.objects[dm.keys[offset]]; } + const T& operator*() const { return dm.objects[dm.sorted[offset]]; } }; - iterator begin() const { return iterator{*this, 0}; } - iterator end() const { return iterator{*this, keys.size()}; } + iterator begin() const { + if (!sortedValid) { + sorted.resize(objects.size()); + for (uint32_t i = 0; i < sorted.size(); i++) sorted[i] = i; + std::sort(sorted.begin(), sorted.end(), [&](uint32_t a, uint32_t b) { return objects[a] < objects[b]; }); + sortedValid = true; + } + return iterator{*this, 0}; + } + iterator end() const { return iterator{*this, objects.size()}; } private: + template static uint64_t rawHash(const U& entry) { return entry.hash(); } + static uint64_t rawHash(const std::string& entry) { return std::hash()(entry); } + + // The entries of one DequeMap often share bits of their hash (sharded by it), so mix it + static uint64_t hashOf(const T& entry) { + return mix(rawHash(entry)); + } + static uint64_t mix(uint64_t h) { + h ^= h >> 33; h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; h *= 0xc4ceb9fe1a85ec53ULL; + h ^= h >> 33; + return h; + } + + int32_t lookup(const T& entry, uint64_t h) const { + if (slots.empty()) return -1; + const size_t mask = slots.size() - 1; + for (size_t i = h & mask; slots[i] != 0; i = (i + 1) & mask) + if (objects[slots[i] - 1] == entry) + return slots[i] - 1; + return -1; + } + + void insertSlot(uint64_t h, uint32_t index) { + const size_t mask = slots.size() - 1; + size_t i = h & mask; + while (slots[i] != 0) i = (i + 1) & mask; + slots[i] = index + 1; + } + uint32_t maxSize; // Using a deque is necessary, as it provides pointer-stability for previously // added objects when it grows the storage (as opposed to, e.g., vector). std::deque objects; - // Whereas `objects` is ordered by insertion-time, `keys` is sorted such that - // objects[key[0]] < objects[key[1]] < ... < objects[key[$]] - // operator< of T. - std::vector keys; + // Open-addressing hash table of the objects: index + 1, 0 = empty + std::vector slots; + + // For iteration only: object indexes in the order of operator< + mutable std::vector sorted; + mutable bool sortedValid = false; }; #endif diff --git a/include/geom.h b/include/geom.h index cc33884c..9a8db70f 100644 --- a/include/geom.h +++ b/include/geom.h @@ -39,6 +39,7 @@ typedef boost::geometry::ring_type::type Ring; typedef boost::geometry::interior_type::type InteriorRing; typedef boost::variant Geometry; typedef std::pair IndexValue; +#include "ring_point_index.h" typedef boost::geometry::index::rtree< IndexValue, boost::geometry::index::quadratic<16> > RTree; // Perform self-intersection aware simplification of geometry types @@ -101,10 +102,14 @@ char bit_code(Point const &p, Box const &bbox); // fast_clip returns true if the geometry was modified (false means identity). bool fast_clip(Ring &points, Box const &bbox); bool fast_clip(MultiPolygon &mp, Box const &bbox); +// Same as geom::assign(out, in) followed by fast_clip(out, bbox); out must be empty +bool fast_clip_copy(const MultiPolygon &in, Box const &bbox, MultiPolygon &out); // Exact clip of a valid multipolygon to bbox that keeps it valid (fast_clip can // leave zero-width bridges along the box edge). Returns false if it could not // clip robustly; `out` is then unspecified and the caller must use geom::intersection. bool rect_clip(const MultiPolygon &mp, Box const &bbox, MultiPolygon &out); +// The polygon clip box at the end zoom (see TileDataSource::buildWayGeometry) +Box tile_clip_box_end_zoom(const MultiPolygon &mp, Box const &clippingBox, Box const &extBox); // Cheap conservative validity tests, run before geom::is_valid. They return // true only when the geometry is certainly valid (is_valid would return true @@ -112,6 +117,32 @@ bool rect_clip(const MultiPolygon &mp, Box const &bbox, MultiPolygon &out); // Tolerances are relative to the coordinate magnitude. bool polygonDefinitelyValid(const Polygon &p); bool multiPolygonDefinitelyValid(const MultiPolygon &mp); +// As multiPolygonDefinitelyValid, but segments are taken as apart wherever Boost can tell the +// sides (rounding bound instead of the distance tolerance). Only where the answer just saves a +// Boost is_valid, not where it chooses between ways to compute something. +bool multiPolygonDefinitelyValidAfterClip(const MultiPolygon &mp); +// multiPolygonDefinitelyValid (the same answer) for mp clipped to box by fast_clip: false at +// once when two segments of a polygon overlap along the box edge (a bridge left by the clip). +bool multiPolygonDefinitelyValidClipped(const MultiPolygon &mp, Box const &box); +// As polygonDefinitelyValid / multiPolygonDefinitelyValid (1 exactly when they return true), +// but also recognises a geometry whose only fault is that every ring has the opposite +// orientation: -1 = is_valid reports failure_wrong_orientation (the first exterior ring is +// clean but clearly wrongly oriented, which Boost checks before anything else of the geometry), +// and the geometry with every ring reversed is certainly valid. 0 = unsure. +int polygonDefinitelyValidOrReversed(const Polygon &p); +int multiPolygonDefinitelyValidOrReversed(const MultiPolygon &mp); + +// True only when boost::geometry::intersects(a, b) is certainly false: every segment of one is +// clearly apart from every segment of the other, and the first point of each ring of either is +// clearly outside the other polygon (outside its outer ring, or in one of its holes). +bool polygonsDefinitelyApart(const Polygon &a, const Polygon &b); + +// The conservative tests make_valid uses to skip Boost overlay work (geometry/correct.hpp) +namespace geometry { namespace impl { template struct precheck; } } +template<> struct geometry::impl::precheck { + static bool valid(const MultiPolygon &mp) { return multiPolygonDefinitelyValid(mp); } + static bool apart(const Polygon &a, const Polygon &b) { return polygonsDefinitelyApart(a, b); } +}; diff --git a/include/geometry/correct.hpp b/include/geometry/correct.hpp index fba96c69..ee105d60 100644 --- a/include/geometry/correct.hpp +++ b/include/geometry/correct.hpp @@ -10,9 +10,18 @@ * ---------------------------------------------------------------------------- */ +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include "geom.h" #include #include #include @@ -23,13 +32,29 @@ namespace geometry { namespace impl { -template -static inline void result_combine(C &result, T &&new_element) +// Conservative tests (geom.h specializes this for MultiPolygon with the precheck of geom.cpp); false +// means unsure: valid(mp), mp is certainly valid; apart(a, b), the polygons certainly don't intersect +template +struct precheck +{ + using polygon_t = typename boost::range_value::type; + static bool valid(multi_polygon_t const &) { return false; } + static bool apart(polygon_t const &, polygon_t const &) { return false; } +}; + +// boxes[i] is the envelope of result[i]. Polygons whose envelopes are disjoint +// cannot intersect, so the box test skips most of the costly intersects() calls +// (otherwise quadratic in the number of rings, e.g. a lake with 14000 islands). +template +static inline void result_combine(C &result, std::vector &boxes, T &&new_element) { result.push_back(std::forward(new_element)); + boxes.push_back(boost::geometry::return_envelope(result.back())); for(std::size_t i = 0; i < result.size() - 1; ) { - if(!boost::geometry::intersects(result[i], result.back())) { + if(boost::geometry::disjoint(boxes[i], boxes.back()) || + precheck::apart(result[i], result.back()) || + !boost::geometry::intersects(result[i], result.back())) { ++i; continue; } @@ -43,8 +68,66 @@ static inline void result_combine(C &result, T &&new_element) } result.back() = std::move(union_result[0]); + boxes.back() = boost::geometry::return_envelope(result.back()); result.erase(result.begin() + i); - } + boxes.erase(boxes.begin() + i); + } +} + +// boost::geometry::difference(outers, holes, output) for one outer without holes and holes without +// holes, when the polygon of the outer and the holes reversed (in order) is certainly valid by +// precheck::valid (clear of each other: Boost finds no turns between the outer and the holes; +// each hole clearly inside the outer). false, nothing done: not that case. Boost's overlay then +// keeps the outer (its point is in no hole) and every hole (its point is in the outer), reversed; +// assign_parents makes the outer the parent of each hole (positive outer, negative holes, the +// outer's area the larger; the hole's point in the outer's box and ring); add_rings adds the outer +// with the holes as children in ring id order, i.e. the holes' order, if the polygon has 4 points +// or more and an area positive and not 0 by math::equals. +template +static inline bool difference_inside(multi_polygon_t const &outers, multi_polygon_t const &holes, multi_polygon_t &output) +{ + using polygon_t = typename boost::range_value::type; + if(outers.size() != 1 || holes.empty() || !outers[0].inners().empty()) return false; + auto const outer_area = boost::geometry::area(outers[0].outer()); + using area_t = decltype(outer_area); + if(!boost::geometry::math::larger(outer_area, area_t(0))) return false; + for(auto const &hole: holes) { + if(!hole.inners().empty()) return false; + auto const hole_area = boost::geometry::area(hole.outer()); + if(!boost::geometry::math::larger(hole_area, area_t(0)) || outer_area < hole_area) return false; + } + multi_polygon_t candidate; + candidate.resize(1); + polygon_t &result = candidate[0]; + result.outer() = outers[0].outer(); + result.inners().resize(holes.size()); + for(std::size_t i = 0; i < holes.size(); i++) + result.inners()[i].assign(holes[i].outer().rbegin(), holes[i].outer().rend()); + if(!precheck::valid(candidate)) return false; + if(boost::geometry::num_points(result) >= 4) { + auto const area = boost::geometry::area(result); + if(!boost::geometry::math::equals(area, decltype(area)(0)) && area > 0) + output.push_back(std::move(result)); + } + return true; +} + +// boost::geometry::difference(outers, empty, output) for outers whose polygons have no holes and +// rings of positive area (false, nothing done: not that case); outers is moved from +template +static inline bool difference_without_holes(multi_polygon_t &outers, multi_polygon_t &output) +{ + for(auto const &poly: outers) { + if(!poly.inners().empty() || boost::size(poly.outer()) == 0) return false; + if(!(boost::geometry::area(poly.outer()) > 0)) return false; + } + for(auto &poly: outers) { + if(boost::size(poly.outer()) < 4) continue; + auto const area = boost::geometry::area(poly); + if(boost::geometry::math::equals(area, decltype(area)(0)) || !(area > 0)) continue; + output.push_back(std::move(poly)); + } + return true; } struct pseudo_vertice_key @@ -139,6 +222,286 @@ static inline void dissolve_find_intersections( } } +// A self-intersection of a ring: segments i < j (segment k runs ring[k] -> ring[k + 1]) meet in p, +// at scale_1 along segment i and scale_2 along segment j (comparable distances, both below 1) +template +struct dissolve_crossing +{ + std::size_t i, j; + double scale_1, scale_2; + point_t p; +}; + +// Boost's intersection of the segments (a, s) and (s, c), which meet in s, when they are no points +// and not collinear: then Boost computes the point from the ratios along the segments (cramers_rule, +// segment_intersection_info::calculate). That is done here with the same Boost functions and the +// same values, without the side tests Boost runs before (two of the four sides involve s and are 0, +// so the segments are never disjoint; collinear needs all four 0, so side(s, c, a) != 0 rules it +// out). -1: not that case (or not sure), use boost::geometry::intersection; 0: the point is s; 1: +// the point is ip. +// (Not with FMA: the products could be contracted differently from Boost's.) +#if !defined(__FMA__) && !defined(__FMA4__) && (!defined(__FLT_EVAL_METHOD__) || __FLT_EVAL_METHOD__ == 0) +#define DISSOLVE_ADJACENT_FAST 1 +#endif +template +static inline int dissolve_adjacent_intersection(point_t const &a, point_t const &s, point_t const &c, point_t &ip) +{ + namespace bg = boost::geometry; + using coordinate_t = typename bg::coordinate_type::type; + using strategy_t = bg::strategy::intersection::cartesian_segments<>; + using ratio_t = bg::segment_ratio; + if(!std::is_same::value) return -1; + const double eps = std::numeric_limits::epsilon(); + // math::equals (finite values), point by point as cartesian_point_point + auto equal = [eps](double x, double y) { return x == y || std::abs(x - y) <= eps * std::max({ std::abs(x), std::abs(y), 1.0 }); }; + auto same_point = [&](point_t const &u, point_t const &v) { return equal(bg::get<0>(u), bg::get<0>(v)) && equal(bg::get<1>(u), bg::get<1>(v)); }; + // unified(): p = (a, s), q = (s, c); neither is a point + if(same_point(a, s) || same_point(s, c)) + return -1; + coordinate_t const dx_p = bg::get<0>(s) - bg::get<0>(a); + coordinate_t const dx_q = bg::get<0>(c) - bg::get<0>(s); + coordinate_t const dy_p = bg::get<1>(s) - bg::get<1>(a); + coordinate_t const dy_q = bg::get<1>(c) - bg::get<1>(s); + // sides: side(s, c, a), side(s, c, s) = 0, side(a, s, s) = 0, side(a, s, c): not all 0. + // side_by_triangle(s, c, a) is 0 when two of the points are equal (math::equals), else it is + // a determinant of two of the differences from one of the points (picked by their order), + // 0 when within epsilon * max(1, its differences). Each such determinant is within + // 6.02 u m^2 (u = eps / 2, m the largest coordinate difference) of the exact one, as is t + // below: |t| > 16 eps (m^2 + max(1, m)) leaves every one of them far from 0. + { + coordinate_t const dx_ac = bg::get<0>(c) - bg::get<0>(a), dy_ac = bg::get<1>(c) - bg::get<1>(a); + coordinate_t const t = dx_q * (bg::get<1>(a) - bg::get<1>(s)) - dy_q * (bg::get<0>(a) - bg::get<0>(s)); + coordinate_t const m = std::max({ std::abs(dx_p), std::abs(dy_p), std::abs(dx_q), std::abs(dy_q), std::abs(dx_ac), std::abs(dy_ac) }); + const bool side_clear = !same_point(a, c) && std::abs(t) > 16 * eps * (m * m + std::max(1.0, m)); + if(!side_clear && bg::strategy::side::side_by_triangle<>::apply(s, c, a) == 0) + return -1; + } + coordinate_t denominator_a, nominator_a; + coordinate_t denominator_b, nominator_b; + strategy_t::cramers_rule(dx_p, dy_p, dx_q, dy_q, + bg::get<0>(a) - bg::get<0>(s), bg::get<1>(a) - bg::get<1>(s), + nominator_a, denominator_a); + strategy_t::cramers_rule(dx_q, dy_q, dx_p, dy_p, + bg::get<0>(s) - bg::get<0>(a), bg::get<1>(s) - bg::get<1>(a), + nominator_b, denominator_b); + bg::math::detail::equals_factor_policy policy(dx_p, dy_p, dx_q, dy_q); + coordinate_t const zero = 0; + // (Boost takes these as collinear) + if(bg::math::detail::equals_by_policy(denominator_a, zero, policy) + || bg::math::detail::equals_by_policy(denominator_b, zero, policy)) + return -1; + // calculate() takes the point from segment a or b. From b it is s: b's ratio is 0 / d + // (nominator_b is dx_p * dy_p - dy_p * dx_p), s + 0 * dx / d. When the one from a (as + // assign(): a + numerator * dx / denominator, the ratio's sign normalized) is s as well, the + // point is s either way (and the nearly collinear correction leaves s, the end of both). + { + coordinate_t numerator = nominator_a, denominator = denominator_a; + if(denominator < zero) { numerator = -numerator; denominator = -denominator; } + coordinate_t const x = bg::get<0>(a) + numerator * dx_p / denominator; + coordinate_t const y = bg::get<1>(a) + numerator * dy_p / denominator; + if(x == bg::get<0>(s) && y == bg::get<1>(s)) + return 0; + } + typename strategy_t::template segment_intersection_info sinfo; + sinfo.dx_a = dx_p; + sinfo.dx_b = dx_q; + sinfo.dy_a = dy_p; + sinfo.dy_b = dy_q; +#if BOOST_VERSION >= 108700 + sinfo.ra.assign(nominator_a, denominator_a); + sinfo.rb.assign(nominator_b, denominator_b); +#else + sinfo.robust_ra.assign(nominator_a, denominator_a); + sinfo.robust_rb.assign(nominator_b, denominator_b); +#endif + bg::model::referring_segment const p(a, s); + bg::model::referring_segment const q(s, c); + sinfo.calculate(ip, p, q); + return 1; +} + +// Whether Boost's intersects(box of (s, c), segment (a, s)) certainly holds, given s - a: when each +// coordinate difference is 0 or not 0 by math::equals (see dissolve_find_crossings) +static inline bool dissolve_envelope_holds(double dx, double dy) +{ + const double eps = std::numeric_limits::epsilon(); + return (dx == 0 || std::abs(dx) > eps) && (dy == 0 || std::abs(dy) > eps); +} + +// The crossings dissolve_find_intersections finds, without its rtree, which it fills segment by +// segment (that cost the most). It reports the pairs of segments i < j for which +// intersects(segment j, segment i) holds and, while it queries segment i, the boxes of the rtree +// nodes above segment j intersect segment i. Those boxes contain the envelope of segment j: when +// segment i intersects that envelope, it intersects them too, and the pair is reported. When it +// doesn't (segment j only just touches segment i, within Boost's tolerance), whether the pair is +// reported depends on the shape of the rtree: then false (nothing is known, use +// dissolve_find_intersections). The pairs are taken from boxes that overlap with a margin far +// above that tolerance, and the order of the pairs doesn't matter (each gives its own pseudo +// vertex keys). +template< + typename point_t = boost::geometry::model::d2::point_xy, + typename ring_t = boost::geometry::model::ring + > +static inline bool dissolve_find_crossings(ring_t const &ring, std::vector> &crossings) +{ + if(ring.size() < 2) return true; + using segment_t = boost::geometry::model::segment; + using box_t = boost::geometry::model::box>; + const std::size_t segments = ring.size() - 1; + + struct seg_box { double min_x, max_x, min_y, max_y; }; + std::vector boxes(segments); + for(std::size_t k = 0; k < segments; k++) { + const double x0 = boost::geometry::get<0>(ring[k]), y0 = boost::geometry::get<1>(ring[k]); + const double x1 = boost::geometry::get<0>(ring[k + 1]), y1 = boost::geometry::get<1>(ring[k + 1]); + if(!std::isfinite(x0) || !std::isfinite(y0) || !std::isfinite(x1) || !std::isfinite(y1)) return false; + const double margin = 1e-9 * std::max({ 1.0, std::abs(x0), std::abs(y0), std::abs(x1), std::abs(y1) }); + boxes[k] = seg_box{ std::min(x0, x1) - margin, std::max(x0, x1) + margin, std::min(y0, y1) - margin, std::max(y0, y1) + margin }; + } + + std::vector output; + output.reserve(2); + // segments i < j whose boxes overlap; false: not decided (see above) + auto pair = [&](std::size_t i, std::size_t j) { + segment_t line_1(ring[i], ring[i + 1]); + segment_t line_2(ring[j], ring[j + 1]); + output.clear(); +#ifdef DISSOLVE_ADJACENT_FAST + if(j == i + 1) { + // Consecutive segments (a, s) and (s, c). Boost's intersection of them gives no crossing + // (no point with both scales below 1) when one is exactly a point (a repeated point) or + // when the first is at least as long as the second (cla >= clb, Boost's comparable + // lengths, which are also the comparable distances below): + // - collinear (all sides 0, or a denominator 0 by Boost's tolerance): its points are + // end points, a, s or c; at a, scale 2 is cla / clb (>= 1), at s scale 1 is 1, at c + // scale 2 is 1 (0 / 0 for a point: NaN). With a point segment, the point is a (both + // points; a = s or scale 2 cla / clb or x / 0) or the point segment's first point. + // - else Boost computes it from segment b = (s, c) (use_a: edge value of b's ratio + // 0 / d is 1, a's at most 1, a's relative length 0): s + 0 * dx / d = s, which the + // nearly collinear correction leaves; scale 1 is 1. + // Then the pair adds nothing; it is undecided only when Boost's intersects(line_2, + // line_1) holds and the envelope test fails. + // The envelope test (disjoint_segment_box_impl, a slab test of line_1 = (a, s) against the + // box of (s, c), which holds s) can only fail when a coordinate difference of line_1 is + // 0 by math::equals without being 0: in each dimension the box's distances from a bracket + // line_1's (ti_min <= diff <= ti_max, rounding is monotonic), so with diff 0 or clearly + // not 0 every comparison it makes keeps t = 1 (s) inside. + point_t const &a = ring[i], &s = ring[j], &c = ring[j + 1]; + const double ax = boost::geometry::get<0>(a), ay = boost::geometry::get<1>(a); + const double sx = boost::geometry::get<0>(s), sy = boost::geometry::get<1>(s); + const double cx = boost::geometry::get<0>(c), cy = boost::geometry::get<1>(c); + const bool point_1 = ax == sx && ay == sy; + const bool point_2 = sx == cx && sy == cy; + const double dx_p = sx - ax, dy_p = sy - ay, dx_q = cx - sx, dy_q = cy - sy; + const double cla = dx_p * dx_p + dy_p * dy_p, clb = dx_q * dx_q + dy_q * dy_q; + const bool envelope_holds = dissolve_envelope_holds(dx_p, dy_p); + if(point_1 || point_2 || cla >= clb) + return envelope_holds + || boost::geometry::intersects(boost::geometry::return_envelope(line_2), line_1) + || !boost::geometry::intersects(line_2, line_1); + point_t ip; + const int found = dissolve_adjacent_intersection(a, s, c, ip); + if(found == 0) + return envelope_holds + || boost::geometry::intersects(boost::geometry::return_envelope(line_2), line_1); + if(found > 0) { + if(!envelope_holds && !boost::geometry::intersects(boost::geometry::return_envelope(line_2), line_1)) return false; + output.push_back(ip); + } + } + if(output.empty()) { +#endif + if(!boost::geometry::intersects(line_2, line_1)) return true; + if(!boost::geometry::intersects(boost::geometry::return_envelope(line_2), line_1)) return false; + + output.clear(); + boost::geometry::intersection(line_1, line_2, output); +#ifdef DISSOLVE_ADJACENT_FAST + } +#endif + + for(auto const &p: output) { + double scale_1 = boost::geometry::comparable_distance(p, ring[i]) / boost::geometry::comparable_distance(ring[i + 1], ring[i]); + double scale_2 = boost::geometry::comparable_distance(p, ring[j]) / boost::geometry::comparable_distance(ring[j + 1], ring[j]); + if(scale_1 < 1.0 && scale_2 < 1.0) + crossings.push_back(dissolve_crossing{ i, j, scale_1, scale_2, p }); + } + return true; + }; + auto overlap = [](seg_box const &a, seg_box const &b) { + return a.min_x <= b.max_x && b.min_x <= a.max_x && a.min_y <= b.max_y && b.min_y <= a.max_y; + }; + + if(segments <= 32) { + for(std::size_t i = 0; i < segments; i++) + for(std::size_t j = i + 1; j < segments; j++) + if(overlap(boxes[i], boxes[j]) && !pair(i, j)) return false; + return true; + } + + // Sweep over the boxes by their lower side along one axis: every pair of boxes that overlap + // along it is looked at, and the overlapping ones are passed to pair (the order of the pairs + // doesn't matter, see above). The axis is the one along which the boxes are narrower compared + // to the ring's extent (fewer pairs to look at). A ring whose boxes overlap a lot along it (too + // many pairs) is left to dissolve_find_intersections. + double width = 0, height = 0, min_x = boxes[0].min_x, max_x = boxes[0].max_x, min_y = boxes[0].min_y, max_y = boxes[0].max_y; + for(auto const &b: boxes) { + width += b.max_x - b.min_x; height += b.max_y - b.min_y; + min_x = std::min(min_x, b.min_x); max_x = std::max(max_x, b.max_x); + min_y = std::min(min_y, b.min_y); max_y = std::max(max_y, b.max_y); + } + const bool along_y = height * (max_x - min_x) < width * (max_y - min_y); + // sorted boxes, as (lo, hi) along the axis and (lo, hi) across, with their segment + struct sweep_box { double lo, hi, across_lo, across_hi; uint32_t k; }; + std::vector sorted(segments); + for(std::size_t k = 0; k < segments; k++) { + seg_box const &b = boxes[k]; + sorted[k] = along_y ? sweep_box{ b.min_y, b.max_y, b.min_x, b.max_x, uint32_t(k) } + : sweep_box{ b.min_x, b.max_x, b.min_y, b.max_y, uint32_t(k) }; + } + std::sort(sorted.begin(), sorted.end(), [](sweep_box const &a, sweep_box const &b) { return a.lo < b.lo; }); + std::size_t budget = 64 * segments + 4096; + for(std::size_t a = 0; a < segments; a++) { + const sweep_box ba = sorted[a]; + for(std::size_t b = a + 1; b < segments && sorted[b].lo <= ba.hi; b++) { + if(!budget--) return false; + const sweep_box &bb = sorted[b]; + if(bb.across_lo > ba.across_hi || ba.across_lo > bb.across_hi) continue; + const std::size_t i = std::min(ba.k, bb.k), j = std::max(ba.k, bb.k); + if(!pair(i, j)) return false; + } + } + return true; +} + +// The pseudo vertices of a ring with crossings, as dissolve_find_intersections adds them: its +// points, and two for each crossing (one on each segment) that link to each other +template< + typename point_t = boost::geometry::model::d2::point_xy, + typename ring_t = boost::geometry::model::ring + > +static inline void dissolve_pseudo_vertices( + ring_t const &ring, + std::vector> const &crossings, + std::map, compare_pseudo_vertice_key> &pseudo_vertices, + std::set &start_keys) +{ + for(std::size_t i = ring.size(); i--; ) + pseudo_vertices.emplace(pseudo_vertice_key(i, i, 0.0), ring[i]); + for(auto const &c: crossings) { + const std::size_t i = c.i, j = c.j; + pseudo_vertice_key key_j(j, i, c.scale_2); + pseudo_vertices.emplace(pseudo_vertice_key(i, j, c.scale_1, true), pseudo_vertice(c.p, key_j)); + pseudo_vertices.emplace(key_j, c.p); + start_keys.insert(key_j); + + pseudo_vertice_key key_i(i, j, c.scale_1); + pseudo_vertices.emplace(pseudo_vertice_key(j, i, c.scale_2, true), pseudo_vertice(c.p, key_i)); + pseudo_vertices.emplace(key_i, c.p); + start_keys.insert(key_i); + } +} + // Remove invalid points (NaN) from ring template< typename point_t = boost::geometry::model::d2::point_xy, @@ -199,27 +562,40 @@ static inline std::vector dissolve_generate_rings( // Generate all polygons by tracing all the intersections // Perform union to combine all polygons into single polygon again - while(!start_keys.empty()) { - ring_t new_ring; - - // Store point in generated polygon - auto push_point = [&new_ring](auto const &p) { - if(new_ring.empty() || boost::geometry::comparable_distance(new_ring.back(), p) > 0) - new_ring.push_back(p); - }; + // Position of each point in the ring being traced. Its points are distinct until + // it closes, so a point already seen closes it: looking that up here instead of + // scanning the whole ring for every new point keeps tracing linear, not quadratic. + struct point_hash { + std::size_t operator()(std::pair const &p) const { + return std::hash()(p.first) * 31 + std::hash()(p.second); + } + }; + std::unordered_map, std::size_t, point_hash> seen; - auto is_closed = [](ring_t &ring) { - if(ring.size() < 2) return false; - for(std::size_t i = 0; i < ring.size() - 1; ++i) { - if(boost::geometry::comparable_distance(ring[i], ring.back()) == 0) { - ring.erase(ring.begin(), ring.begin() + i); - return true; - } + while(!start_keys.empty()) { + ring_t new_ring; + seen.clear(); + bool closed = false; + + // Store point in generated polygon; when it repeats an earlier point, the + // ring is closed and the points before that one are dropped + auto push_point = [&new_ring, &seen, &closed](auto const &p) { + if(!new_ring.empty() && !(boost::geometry::comparable_distance(new_ring.back(), p) > 0)) + return; + new_ring.push_back(p); + // + 0.0 turns -0.0 into 0.0, which compares equal to it + std::pair key(boost::geometry::get<0>(p) + 0.0, boost::geometry::get<1>(p) + 0.0); + auto found = seen.find(key); + if(found == seen.end()) { + seen.emplace(key, new_ring.size() - 1); + } else { + new_ring.erase(new_ring.begin(), new_ring.begin() + found->second); + closed = true; } - - return false; }; + auto is_closed = [&closed](ring_t &) { return closed; }; + auto start_iter = pseudo_vertices.find(*start_keys.begin()); auto i = start_iter; @@ -280,8 +656,13 @@ static inline std::vector correct(ring_t const &ring, boost::geometry::o // Correct orientation correct_orientation(new_ring, order); - // Detect self-intersection points - dissolve_find_intersections(new_ring, pseudo_vertices, start_keys); + // Detect self-intersection points (the pseudo vertices only for a ring that has some) + std::vector> crossings; + if(dissolve_find_crossings(new_ring, crossings)) { + if(!crossings.empty()) + dissolve_pseudo_vertices(new_ring, crossings, pseudo_vertices, start_keys); + } else + dissolve_find_intersections(new_ring, pseudo_vertices, start_keys); if(start_keys.empty()) { if(std::abs(boost::geometry::area(new_ring)) > remove_spike_min_area) { @@ -302,13 +683,15 @@ template< > struct combine_non_zero_winding { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &outer_boxes, multi_polygon_t &combined_inners, std::vector &inner_boxes, polygon_t &poly) { if(boost::geometry::area(poly) > 0) - result_combine(combined_outers, std::move(poly)); + result_combine(combined_outers, outer_boxes, std::move(poly)); else { std::reverse(poly.outer().begin(), poly.outer().end()); - result_combine(combined_inners, std::move(poly)); + result_combine(combined_inners, inner_boxes, std::move(poly)); } } }; @@ -320,7 +703,9 @@ template< > struct combine_odd_even { - inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly) + using box_t = boost::geometry::model::box; + + inline void operator()(multi_polygon_t &combined_outers, std::vector &, multi_polygon_t &combined_inners, std::vector &, polygon_t &poly) { if(boost::geometry::area(poly) < 0) std::reverse(poly.outer().begin(), poly.outer().end()); @@ -346,11 +731,12 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub // Calculate all outers and combine them if possible multi_polygon_t combined_outers; multi_polygon_t combined_inners; + std::vector> outer_boxes, inner_boxes; for(auto &ring: outer_rings) { polygon_t poly; poly.outer() = std::move(ring); - combine(combined_outers, combined_inners, poly); + combine(combined_outers, outer_boxes, combined_inners, inner_boxes, poly); } // Calculate all inners and combine them if possible @@ -362,12 +748,26 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub correct(poly, new_inners, remove_spike_min_area, combine); for(auto &poly: new_inners) { - result_combine(combined_inners, std::move(poly)); - } + result_combine(combined_inners, inner_boxes, std::move(poly)); + } } - // Cut out all inners from all the outers - boost::geometry::difference(combined_outers, combined_inners, output); + // Nothing to cut out of outers without holes: Boost's difference returns each outer as it is + // (overlay's return_if_one_input_is_empty: select_rings keeps every ring, none within the empty + // second input; assign_parents returns at once when every ring has a positive area; add_rings + // copies each ring into a polygon, in input order, if it has 4 points or more and an area that + // is positive and not 0 by math::equals) + if(combined_inners.empty() && difference_without_holes(combined_outers, output)) + return; + if(difference_inside(combined_outers, combined_inners, output)) + return; + + // Cut out all inners from all the outers. Index the rings for the many + // point-in-ring tests (a hole each), see ring_point_index.h + { + RingPointIndexScope pointIndex; + boost::geometry::difference(combined_outers, combined_inners, output); + } } template< @@ -379,13 +779,18 @@ template< > static inline void correct(multi_polygon_t const &input, multi_polygon_t &output, double remove_spike_min_area, combine_function_t combine) { + using box_t = boost::geometry::model::box; + std::vector output_boxes; + for(auto const &polygon: output) + output_boxes.push_back(boost::geometry::return_envelope(polygon)); + for(auto const &polygon: input) { multi_polygon_t new_polygons; correct(polygon, new_polygons, remove_spike_min_area, combine); - for(auto &new_polygon: new_polygons) - result_combine(output, std::move(new_polygon)); + for(auto &new_polygon: new_polygons) + result_combine(output, output_boxes, std::move(new_polygon)); } } diff --git a/include/geometry_cache.h b/include/geometry_cache.h index e8c39d9c..f2943e5f 100644 --- a/include/geometry_cache.h +++ b/include/geometry_cache.h @@ -14,10 +14,10 @@ template class GeometryCache { public: - // 64 x 32 entries: the cache only lives for one tile batch (see clear()), so a + // 256 x 32 entries: the cache only lives for one tile batch (see clear()), so a // smaller table loses few hits and pins much less heap in every tile thread. // Keep bucket size small so linear search is fast. - enum : size_t { NUM_BUCKETS = 64, BUCKET_SIZE = 32 }; + enum : size_t { NUM_BUCKETS = 256, BUCKET_SIZE = 32 }; GeometryCache(): offsets(NUM_BUCKETS), diff --git a/include/leased_store.h b/include/leased_store.h index 0349d64d..019b110d 100644 --- a/include/leased_store.h +++ b/include/leased_store.h @@ -2,6 +2,7 @@ #define _LEASED_STORE_H #include "tile_data.h" +#include // LeasedStore implements "leases" for generated geometries. // @@ -45,23 +46,26 @@ inline std::vector>& g template class LeasedStore { - std::vector>> leases; + std::vector>> leases; // source, its leaseNumber, lease public: ~LeasedStore() { + // Only to the sources still alive (see registerLeaseSource) + std::lock_guard live(leaseSourcesMutex()); for (const auto& lease : leases) { - auto source = lease.first; + auto source = std::get<0>(lease); + if (!leaseSourceAlive(source, std::get<1>(lease))) continue; std::lock_guard lock(source->storeMutex); std::vector>& availableLeases = getAvailableLeases(source); - availableLeases.push_back(lease.second); + availableLeases.push_back(std::get<2>(lease)); } } std::pair get(TileDataSource* source) { for (const auto& lease : leases) { - if (lease.first == source) - return lease.second; + if (std::get<0>(lease) == source && std::get<1>(lease) == source->leaseNumber) + return std::get<2>(lease); } std::lock_guard lock(source->storeMutex); @@ -74,7 +78,7 @@ class LeasedStore { std::pair entry = availableLeases.back(); availableLeases.pop_back(); - leases.push_back(std::make_pair(source, entry)); + leases.push_back(std::make_tuple(source, source->leaseNumber, entry)); return entry; } diff --git a/include/mbtiles.h b/include/mbtiles.h index ca671b07..8181fafa 100644 --- a/include/mbtiles.h +++ b/include/mbtiles.h @@ -3,9 +3,11 @@ #define _MBTILES_H #include +#include #include #include #include "external/sqlite_modern_cpp.h" +#include "ordered_batches.h" struct PendingStatement { int zoom; @@ -27,6 +29,7 @@ class MBTiles { std::shared_ptr> pendingStatements1, pendingStatements2; std::mutex pendingStatementsMutex; + std::atomic pendingBytes; void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge); void flushPendingStatements(); @@ -39,6 +42,9 @@ class MBTiles { void saveTile(int zoom, int x, int y, std::string *data, bool isMerge); void closeForWriting(); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + void openForReading(std::string &filename); void readBoundingBox(double &minLon, double &maxLon, double &minLat, double &maxLat); void readTileList(std::vector> &tileList); diff --git a/include/memo_keys.h b/include/memo_keys.h new file mode 100644 index 00000000..0506d785 --- /dev/null +++ b/include/memo_keys.h @@ -0,0 +1,273 @@ +/*! \file */ +#ifndef _MEMO_KEYS_H +#define _MEMO_KEYS_H + +// The way_function memo's walk (see OsmLuaProcessing::runWayFunction) without TagMap lookups. +// +// The memo tree reads few distinct keys (tens), but each way is tested against ~50 of them, and +// most are absent. So the keys the tree reads get small ids (MemoKeyTable), the way's tags are +// looked up once each (MemoTagSlots: which ids are present, with their values), and the walk runs +// on a compact copy of the tree (MemoHotNode: kind, key id and the children of the common results). +// The results are those of TagMap::getKey / getValueFromKey: a key matches by its exact bytes and a +// key added twice has its last value. + +#include +#include +#include +#include +#include + +class MemoKeyTable { +public: + static constexpr uint32_t NONE = UINT32_MAX; + + uint32_t find(const char* key, size_t size) const { + if (slots.empty()) return NONE; + const uint64_t p = prefixOf(key, size); + const size_t mask = slots.size() - 1; + for (size_t i = hashOf(p, size) & mask; ; i = (i + 1) & mask) { + const Slot& s = slots[i]; + if (s.id == NONE) return NONE; + if (s.prefix == p && s.size == size && (size <= 8 || memcmp(names[s.id].data() + 8, key + 8, size - 8) == 0)) + return s.id; + } + } + + // The id of a key, given a new one if it hasn't one yet + uint32_t add(std::string_view key) { + uint32_t id = find(key.data(), key.size()); + if (id != NONE) return id; + id = names.size(); + names.emplace_back(key); + if ((names.size() + 1) * 4 > slots.size()) rehash(slots.empty() ? 64 : slots.size() * 2); + else place(id); + return id; + } + + size_t size() const { return names.size(); } + // Per id (kept up to date by the owner, see OsmLuaProcessing::runWayFunction): a key whose + // value the tree only tests for being empty (way_memo_values) + std::vector valueOnly; + const std::string& name(uint32_t id) const { return names[id]; } + +private: + struct Slot { uint64_t prefix; uint32_t size; uint32_t id; }; + std::vector names; + std::vector slots; // open addressing, at most a quarter full + + // The first (up to) 8 bytes of a key as an integer + static uint64_t prefixOf(const char* key, size_t size) { + uint64_t p = 0; + if (size >= 8) memcpy(&p, key, 8); + else if (size >= 4) { + // bytes 0..3 and size-4..size-1 (the overlap has the same bytes in both) + uint32_t lo, hi; + memcpy(&lo, key, 4); memcpy(&hi, key + size - 4, 4); + p = uint64_t(lo) | uint64_t(hi) << (8 * (size - 4)); + } else if (size > 0) { + const auto b = [&](size_t i) { return uint64_t(static_cast(key[i])) << (8 * i); }; + p = b(0) | b(size / 2) | b(size - 1); + } + return p; + } + static size_t hashOf(uint64_t prefix, size_t size) { + const uint64_t h = (prefix ^ (uint64_t(size) << 56)) * 0x9e3779b97f4a7c15ull; + return h >> 40; + } + void place(uint32_t id) { + const std::string& k = names[id]; + const uint64_t p = prefixOf(k.data(), k.size()); + const size_t mask = slots.size() - 1; + size_t i = hashOf(p, k.size()) & mask; + while (slots[i].id != NONE) i = (i + 1) & mask; + slots[i] = Slot{ p, uint32_t(k.size()), id }; + } + void rehash(size_t n) { + slots.assign(n, Slot{ 0, 0, NONE }); + for (uint32_t id = 0; id < names.size(); id++) place(id); + } +}; + +// The current object's tags with a key of the table: present or not, and the value +class MemoTagSlots { +public: + // Look up the tags (as added: a later one replaces an earlier one of the same key), and make + // the object's signature for MemoSigCache: `flags`, then per tag with a key of the table (as + // added) its id and value (valueOnly keys: whether it is empty). Equal signatures mean equal + // tags as the walk sees them. Returns its length, 0 if too long for the cache. + template + size_t fill(const MemoKeyTable& table, const Tags& tags, uint8_t flags, char* sig, size_t sigMax) { + if (stamp.size() < table.size()) { stamp.resize(table.size() + 16, 0); value.resize(table.size() + 16); } + if (++gen == 0) { std::fill(stamp.begin(), stamp.end(), 0); gen = 1; } + present = 1ull << 63; + size_t len = 0; + sig[len++] = flags; + for (const auto& tag : tags) { + const uint32_t id = table.find(tag.key.data(), tag.key.size()); + if (id == MemoKeyTable::NONE) continue; + stamp[id] = gen; + value[id] = std::string_view(tag.value.data(), tag.value.size()); + present |= keyBit(id); + if (len == 0) continue; // too long already + const size_t vlen = tag.value.size(); + const bool valueOnly = table.valueOnly[id]; + // id (1 or 2 bytes), then 1 byte (valueOnly: empty or not), else the length and the bytes + const size_t need = (id < 128 ? 1 : 2) + 1 + (valueOnly ? 0 : vlen); + if (id >= 16384 || vlen >= 256 || len + need > sigMax) { len = 0; continue; } + if (id < 128) sig[len++] = char(id); + else { sig[len++] = char(0x80 | (id >> 8)); sig[len++] = char(id & 0xff); } + if (valueOnly) sig[len++] = char(vlen != 0); + else { sig[len++] = char(vlen); memcpy(sig + len, tag.value.data(), vlen); len += vlen; } + } + return len; + } + // A bit per key id (ids share bits: a clear bit means all its keys are absent); bit 63 always set + static uint64_t keyBit(uint32_t id) { return 1ull << (id % 63); } + uint64_t present = 0; + // (ids the table got after fill(): absent) + bool has(uint32_t id) const { return id < stamp.size() && stamp[id] == gen; } + std::string_view get(uint32_t id) const { return has(id) ? value[id] : std::string_view(); } + +private: + uint32_t gen = 0; + std::vector stamp; // == gen: present + std::vector value; +}; + +// A direct-mapped cache of recent signatures (short byte strings) -> 32-bit values. Used for +// - the walk: the node it reached (a Leaf or Lua node) per object signature (MemoTagSlots::fill). +// The walk is a function of the tags it can read, IsClosed and the relations, and objects in a +// row are often alike (buildings, streets of a town). An edge of the tree never changes once +// added, so a path once complete keeps ending at the same node while nodes are added; +// memoCompact renumbers the nodes (clear()). +// - the AttributeIndex of a replayed segment with way_memo_values attributes, per segment and +// values (see OsmLuaProcessing::memoSegmentSig) +class MemoSigCache { +public: + static constexpr size_t SIG_MAX = 52; + explicit MemoSigCache(size_t size) : entries(size) {} // size: a power of 2 + static uint64_t hashOf(const char* sig, size_t len) { + uint64_t h = len * 0x9e3779b97f4a7c15ull; + size_t i = 0; + for (; i + 8 <= len; i += 8) { uint64_t w; memcpy(&w, sig + i, 8); h = (h ^ w) * 0xff51afd7ed558ccdull; h ^= h >> 29; } + uint64_t w = 0; + for (size_t k = 0; i < len; i++, k++) w |= uint64_t(static_cast(sig[i])) << (8 * k); + h = (h ^ w) * 0xc4ceb9fe1a85ec53ull; h ^= h >> 32; + return h; + } + uint32_t find(uint64_t hash, const char* sig, size_t len) const { + const Entry& e = entries[hash & (entries.size() - 1)]; + return e.len == len && e.hash == hash && memcmp(e.sig, sig, len) == 0 ? e.node : UINT32_MAX; + } + void put(uint64_t hash, const char* sig, size_t len, uint32_t node) { + Entry& e = entries[hash & (entries.size() - 1)]; + e.hash = hash; e.node = node; e.len = len; memcpy(e.sig, sig, len); + } + void clear() { for (Entry& e : entries) e.len = 0; } +private: + struct Entry { uint64_t hash = 0; uint32_t node = 0; uint16_t len = 0; char sig[SIG_MAX]; }; // len 0: empty + std::vector entries; +}; + +#ifndef MEMO_LEVELS +#define MEMO_LEVELS 2 +#endif +#ifndef MEMO_FAN +#define MEMO_FAN 4 +#endif + +// A memo tree node in compact form: what the walk needs, for the results "", "0", "1" and "\x01" +// (the other results: the full node's child()) +// Jumps: from a read node, the walk of a way that has none of the keys along the next (up to) 4 +// resp. 16 reads takes their "absent" children ("" for Find, "0" otherwise); jump[l] is where it +// ends up, mask[l] the bits of those keys (IsClosed/NextRelation: CLOSED_BIT/RELATION_BIT). +// A node without a jump has mask bit 63, which `present` always has. +struct MemoHotNode { + uint8_t kind = 0; // OsmLuaProcessing::MemoNode::Kind + bool valueOnly = false; // a tag of way_memo_values: branches on empty / not empty only + bool overflow = false; // too many results: the others run through Lua + uint16_t children = 0; // number of children (all results) + uint32_t key = MemoKeyTable::NONE; // Find, Holds: the key's id + uint32_t quick[4] = { UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX }; // children for "", "0", "1", "\x01" + uint32_t more = UINT32_MAX; // the node's other children / its ops (Leaf): index in the owner's side table + uint32_t jump[MEMO_LEVELS]; + uint64_t mask[MEMO_LEVELS]; + MemoHotNode() { for (int l = 0; l < MEMO_LEVELS; l++) { jump[l] = 0; mask[l] = 1ull << 63; } } +}; +static constexpr uint64_t MEMO_CLOSED_BIT = 1ull << 61, MEMO_RELATION_BIT = 1ull << 62; + +// (Re)compute the jumps of nodes [from, size). A child has a higher index than its parent, so going +// backwards, the jumps of the children are known. Jumps of a growing tree stay right (an edge +// once added never changes), they only get shorter than they could be until recomputed. +inline void memoComputeJumps(MemoHotNode* hot, size_t size, size_t from = 0) { + for (size_t i = size; i-- > from; ) { + MemoHotNode& node = hot[i]; + // level 0: up to MEMO_FAN single steps + uint32_t m = i; uint64_t mask = 0; int steps = 0; + for (; steps < MEMO_FAN; steps++) { + const MemoHotNode& x = hot[m]; + if (x.kind < 1 || x.kind > 4) break; + const uint32_t next = x.quick[x.kind == 1 ? 0 : 1]; + if (next == UINT32_MAX || next >= size) break; + mask |= x.kind == 1 || x.kind == 2 ? MemoTagSlots::keyBit(x.key) : x.kind == 3 ? MEMO_CLOSED_BIT : MEMO_RELATION_BIT; + m = next; + } + node.jump[0] = m; node.mask[0] = steps ? mask : 1ull << 63; + // level l: up to MEMO_FAN jumps of level l-1 (of the children: computed already) + for (int l = 1; l < MEMO_LEVELS; l++) { + m = i; mask = 0; steps = 0; + for (int k = 0; k < MEMO_FAN; k++) { + const MemoHotNode& x = hot[m]; + if (x.mask[l - 1] >> 63) break; + mask |= x.mask[l - 1]; m = x.jump[l - 1]; steps++; + } + node.jump[l] = m; node.mask[l] = steps > 1 ? mask : 1ull << 63; + } + } +} + +// Walk the tree from the root as runWayFunction does. Kinds: 0 Unset, 1 Find, 2 Holds, 3 IsClosed, +// 4 NextRelation, 5 Leaf, 6 Lua, 7 MercatorArea, 8 FindInRelation. Returns the node reached that isn't +// a read (Unset, Leaf, Lua), or UINT32_MAX when the result read at node `last` has no child yet. +// childOf(n, result): the child of node n for a result other than the four quick ones. +// extra: the results of the reads that aren't tags: nextRelation() (only called with relations), +// area(), findInRelation(key id) +template +inline uint32_t memoWalk(const MemoHotNode* hot, const MemoTagSlots& slots, bool isClosed, bool hasRelations, const ChildOf& childOf, Extra& extra, uint32_t& last) { + auto byResult = [&](uint32_t n, std::string_view r) { + if (r.empty()) return hot[n].quick[0]; + if (r.size() == 1 && (r[0] == '0' || r[0] == '1' || r[0] == '\x01')) return hot[n].quick[r[0] == '0' ? 1 : r[0] == '1' ? 2 : 3]; + return childOf(n, r); + }; + const uint64_t present = slots.present | (isClosed ? MEMO_CLOSED_BIT : 0) | (hasRelations ? MEMO_RELATION_BIT : 0); + uint32_t n = 0; + while (true) { + const MemoHotNode& node = hot[n]; + bool jumped = false; + for (int l = MEMO_LEVELS; l-- > 0; ) + if (!(present & node.mask[l])) { n = node.jump[l]; jumped = true; break; } + if (jumped) continue; + uint32_t next; + switch (node.kind) { + case 1: { // Find + if (!slots.has(node.key)) { next = node.quick[0]; break; } + const std::string_view v = slots.get(node.key); + if (v.empty()) next = node.quick[0]; + else if (node.valueOnly) next = node.quick[3]; + else if (v.size() == 1 && (v[0] == '0' || v[0] == '1' || v[0] == '\x01')) next = node.quick[v[0] == '0' ? 1 : v[0] == '1' ? 2 : 3]; + else next = childOf(n, v); + break; + } + case 2: next = node.quick[slots.has(node.key) ? 2 : 1]; break; // Holds + case 3: next = node.quick[isClosed ? 2 : 1]; break; + case 4: next = hasRelations ? byResult(n, extra.nextRelation()) : node.quick[1]; break; + case 7: next = byResult(n, extra.area()); break; + case 8: next = byResult(n, extra.findInRelation(node.key)); break; + default: return n; + } + if (next == UINT32_MAX) { last = n; return UINT32_MAX; } + n = next; + } +} + +#endif //_MEMO_KEYS_H diff --git a/include/ordered_batches.h b/include/ordered_batches.h new file mode 100644 index 00000000..2defcefc --- /dev/null +++ b/include/ordered_batches.h @@ -0,0 +1,235 @@ +/*! \file */ +#ifndef _ORDERED_BATCHES_H +#define _ORDERED_BATCHES_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Writes batches of output in the order they were handed out, not the order threads +// finish them, so the output file doesn't depend on thread scheduling. +// +// A thread collects a batch's items between begin() and end(). end() writes the batch +// if it's the next one, followed by any batches that were waiting for it; otherwise it +// holds the batch. Batches are numbered in the order they're written, from 0 on: 0, 1, 2, ... +// unless a batch names the number of the one after it (setNext), +// and a thread calls waitForRoom() before it takes its next batch: that keeps what is +// held below the limit set with setMaxHeldBytes(). Batches can be handed out ahead of their +// place, so the batch everyone waits for may not be handed out yet, and then nobody would work +// on it: setFirstUnhanded() tells the number of the first batch not handed out yet, and when +// it's the one to be written next, waitForRoom() lets one thread go on to take it. +// +// writeItem is called by one thread at a time, with writeMutex() held, in batch order. +// If flushItem is given, a batch then goes through flushItem as well: also one thread at a time +// and in batch order, but without holding up writeItem. Whichever thread is free does it, so the +// two steps run side by side. A batch counts as held until it has been flushed. +template +class OrderedBatches { +public: + OrderedBatches(std::function writeItem, std::function itemBytes, + std::function flushItem = nullptr) + : writeItem(writeItem), itemBytes(itemBytes), flushItem(flushItem) { } + + ~OrderedBatches() { finishFlushThread(); } + + // From now on, the flush step runs on a thread of its own (instead of on whichever thread is + // free), so that it can go on after the last batch has been written: see finishFlushThread + void startFlushThread() { + if (!flushItem || flushThread.joinable()) return; + flushThread = std::thread([this]() { + std::unique_lock lock(m); + while (true) { + flushWanted.wait(lock, [&]() { return !toFlush.empty() || stopFlushing; }); + if (toFlush.empty()) return; + flushing = true; + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + flushing = false; + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + }); + } + + // Waits until the flush thread has flushed everything handed over, and ends it + void finishFlushThread() { + if (!flushThread.joinable()) return; + { + std::lock_guard lock(m); + stopFlushing = true; + } + flushWanted.notify_all(); + flushThread.join(); + } + + void setMaxHeldBytes(uint64_t bytes) { + std::lock_guard lock(m); + maxHeldBytes = bytes; + } + + // The number of the first batch in order that isn't handed out yet (UINT64_MAX: none), kept up + // to date by whoever hands batches out. nullptr: every batch is handed out before it's waited for. + void setFirstUnhanded(const std::atomic *key) { + std::lock_guard lock(m); + firstUnhanded = key; + } + + bool inBatch() const { return current().active; } + + void begin(uint64_t seq) { + Current& c = current(); + c.active = true; + c.seq = seq; + c.next = seq + 1; + c.items.clear(); + } + + // The number of the batch that is written after the one begun (default: seq + 1). The numbers + // needn't be consecutive, but each batch must name the one that follows it. + void setNext(uint64_t next) { current().next = next; } + + void add(T&& item) { current().items.emplace_back(std::move(item)); } + + // Hand the batch over. If it's the next one to be written and no other thread is + // writing, this thread writes it and then any held batches that follow it; meanwhile + // other threads just hand their batches over and carry on. + void end() { + Current& c = current(); + c.active = false; + uint64_t bytes = 0; + for (const auto& item : c.items) bytes += itemBytes(item); + std::unique_lock lock(m); + heldBytes += bytes; + held.emplace(c.seq, std::make_pair(std::make_pair(bytes, c.next), std::move(c.items))); + c.items = std::vector(); + if (draining) { + flushQueued(lock); + return; + } + draining = true; + while (!held.empty() && held.begin()->first == nextSeq) { + auto entry = std::move(held.begin()->second); + held.erase(held.begin()); + std::pair> batch(entry.first.first, std::move(entry.second)); + lock.unlock(); + { + std::lock_guard writeLock(w); + for (auto& item : batch.second) writeItem(item); + } + lock.lock(); + nextSeq = entry.first.second; + if (flushItem) { + toFlush.emplace_back(std::move(batch)); + if (flushThread.joinable()) flushWanted.notify_one(); + else heldShrunk.notify_one(); // a waiting thread can flush it + continue; + } + heldBytes -= batch.first; + // Wake the waiting threads only once there is room. Woken for every batch + // written, they all found no room and went back to sleep, again and again: + // most of the system time of a 96-thread planet run. + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + draining = false; + // The next batch to be written may not be handed out yet: wake a thread to take it + if (heldBytes > maxHeldBytes && unhandedNext()) heldShrunk.notify_one(); + flushQueued(lock); + } + + // Waits while more than the limit is held. Returns UINT64_MAX, or the number of the next batch + // to be written if nobody has it yet (see setFirstUnhanded): the caller then takes that batch, + // unless it was handed out meanwhile. Only one thread is let go on per batch number. + uint64_t waitForRoom() { + std::unique_lock lock(m); + while (heldBytes > maxHeldBytes) { + if (!toFlush.empty() && !flushing && !flushThread.joinable()) flushQueued(lock); + else if (unhandedNext()) { tookUnhanded = nextSeq; return nextSeq; } + else heldShrunk.wait(lock); + } + return UINT64_MAX; + } + + // The number of the next batch to be written: every batch before it is written + uint64_t nextToWrite() { + std::lock_guard lock(m); + return nextSeq; + } + + bool allWritten() { + std::lock_guard lock(m); + return held.empty() && toFlush.empty() && !flushing; + } + + // Held while items are written; lock it to write anything outside a batch + std::mutex& writeMutex() { return w; } + +private: + struct Current { + bool active = false; + uint64_t seq = 0, next = 1; + std::vector items; + }; + // Flush the batches waiting for flushItem, unless another thread already does; called and + // returns with m held + void flushQueued(std::unique_lock &lock) { + if (!flushItem || flushing || flushThread.joinable()) return; + flushing = true; + while (!toFlush.empty()) { + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + flushing = false; + // batches queued while finishing the last one are taken by the next thread that comes by; + // wake one in case all others wait for room + heldShrunk.notify_one(); + } + + // The next batch to be written isn't handed out yet, and no thread was let go on to take it + bool unhandedNext() const { + return firstUnhanded && nextSeq >= firstUnhanded->load() && tookUnhanded != nextSeq; + } + + // The batch this thread is working on (one per thread and item type) + static Current& current() { + static thread_local Current c; + return c; + } + + std::function writeItem; + std::function itemBytes; + std::function flushItem; + uint64_t maxHeldBytes = 1ull << 30; + std::mutex m; // guards the fields below + std::mutex w; // held while writing + std::condition_variable heldShrunk; + bool draining = false; + uint64_t nextSeq = 0; + uint64_t heldBytes = 0; + std::map, std::vector>> held; // seq -> ((bytes, next seq), items) + std::deque>> toFlush; // written in order, waiting for flushItem + bool flushing = false; // a thread is flushing + std::thread flushThread; // see startFlushThread + std::condition_variable flushWanted; + bool stopFlushing = false; + const std::atomic *firstUnhanded = nullptr; + uint64_t tookUnhanded = UINT64_MAX; // waitForRoom() let a thread go on to take this batch +}; + +#endif //_ORDERED_BATCHES_H diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index ca22a5f0..d323a9de 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -3,6 +3,7 @@ #define _OSM_LUA_PROCESSING_H #include +#include #include #include #include @@ -32,6 +33,7 @@ extern "C" { #include "external/kaguya.hpp" #include "lua_pool.h" +#include "memo_keys.h" // FIXME: why is this global ? extern bool verbose; @@ -155,6 +157,10 @@ class OsmLuaProcessing { // Returns area double Area(); + // Area in Web Mercator square metres, the unit the zoom levels' pixel sizes are given in + double MercatorArea(); + // MinZoom(z) for the first z = 1, 2... with z >= limit and the area above thresholds[z-1]; else one beyond + void MinZoomByArea(double limit, const std::vector &thresholds); double multiPolygonArea(const MultiPolygon &mp) const; // Returns length @@ -197,37 +203,89 @@ class OsmLuaProcessing { template static bool PrecheckValid(const GeometryT &) { return false; } static bool PrecheckValid(const MultiPolygon &geom) { return multiPolygonDefinitelyValid(geom); } + // As PrecheckValid, but -1 when the only fault is the orientation of every ring (see geom.h) + template + static int PrecheckValidOrReversed(const GeometryT &) { return 0; } + static int PrecheckValidOrReversed(const MultiPolygon &geom) { return multiPolygonDefinitelyValidOrReversed(geom); } + + // geom::is_valid with the point-in-ring index (ring_point_index.h: the same answers). Boost + // locates every hole of a polygon in its outer ring, which is quadratic for a big relation + // with many holes without the index. + template + static bool IsValidIndexed(const GeometryT &geom, geom::validity_failure_type &failure) { + RingPointIndexScope pointIndex; + return geom::is_valid(geom, failure); + } + template + static bool IsValidIndexed(const GeometryT &geom) { + RingPointIndexScope pointIndex; + return geom::is_valid(geom); + } // ---- Requests from Lua to write this way/node to a vector tile's Layer template CorrectGeometryResult CorrectGeometry(GeometryT &geom, bool allowReverse = false) { - return CorrectGeometryAfterCheck(geom, CheckGeometry(geom), allowReverse); + bool reversedValid = false; + const geom::validity_failure_type failure = CheckGeometry(geom, &reversedValid); + return CorrectGeometryAfterCheck(geom, failure, allowReverse, reversedValid); } - // Validity check part of CorrectGeometry (reports the failure if verbose) + // Validity check part of CorrectGeometry (reports the failure if verbose). + // reversedValid (if given) is set when geom is certainly valid once all its rings are reversed. template - geom::validity_failure_type CheckGeometry(const GeometryT &geom) + geom::validity_failure_type CheckGeometry(const GeometryT &geom, bool *reversedValid = nullptr) { geom::validity_failure_type failure = geom::validity_failure_type::no_failure; - // When the precheck proves validity, failure stays no_failure: Boost's result. - const bool pre = (isRelation || isWay) && PrecheckValid(geom); - if (isRelation && !pre && !geom::is_valid(geom,failure)) { + // When the precheck proves validity, failure stays no_failure: Boost's result. When it + // proves that every ring is just reversed, Boost reports failure_wrong_orientation. + const int pre = (isRelation || isWay) ? PrecheckValidOrReversed(geom) : 0; + if (pre < 0) failure = boost::geometry::failure_wrong_orientation; + if (reversedValid) *reversedValid = pre < 0; + if (isRelation && pre <= 0 && (pre < 0 || !IsValidIndexed(geom,failure))) { if (verbose) std::cout << "Relation " << originalOsmID << " has " << boost_validity_error(failure) << std::endl; - } else if (isWay && !pre && !geom::is_valid(geom,failure)) { + } else if (isWay && pre <= 0 && (pre < 0 || !IsValidIndexed(geom,failure))) { if (verbose && failure!=22) std::cout << "Way " << originalOsmID << " has " << boost_validity_error(failure) << std::endl; } return failure; } + // A linestring: Boost's is_valid reports no_failure for any linestring with finite coordinates + // and at least two distinct points (spikes are allowed for linear geometries: has_spikes, which + // Boost runs last, can only make is_valid return false with failure no_failure). So when that + // is certain, the failure is known without is_valid; in verbose mode, is_valid runs (the message). + geom::validity_failure_type CheckGeometry(const Linestring &geom, bool *reversedValid = nullptr) + { + if ((isRelation || isWay) && !verbose && linestringCertainlyValid(geom)) { + if (reversedValid) *reversedValid = false; + return geom::validity_failure_type::no_failure; + } + return CheckGeometry(geom, reversedValid); + } + // Finite coordinates and a point that Boost's equals() certainly tells from the first one + // (it takes coordinates within epsilon * max(1, |a|, |b|) as equal) + static bool linestringCertainlyValid(const Linestring &ls) { + if (ls.size() < 2) return false; + bool finite = true, distinct = false; + const double x0 = ls.front().x(), y0 = ls.front().y(); + for (const Point &p : ls) { + const double x = p.x(), y = p.y(); + finite &= std::isfinite(x) && std::isfinite(y); + distinct |= std::abs(x - x0) > 1e-12 * std::max({ 1.0, std::abs(x), std::abs(x0) }) + || std::abs(y - y0) > 1e-12 * std::max({ 1.0, std::abs(y), std::abs(y0) }); + } + return finite && distinct; + } + // Correction part of CorrectGeometry, given the result of CheckGeometry(geom) template - CorrectGeometryResult CorrectGeometryAfterCheck(GeometryT &geom, geom::validity_failure_type failure, bool allowReverse = false) + // reversedValid: geom is certainly valid once all its rings are reversed (from CheckGeometry) + CorrectGeometryResult CorrectGeometryAfterCheck(GeometryT &geom, geom::validity_failure_type failure, bool allowReverse = false, bool reversedValid = false) { // Fast path: orientation-only defect. Reversing the rings is what make_valid // produces in this case; fall back to make_valid if it isn't enough. if (allowReverse && failure == boost::geometry::failure_wrong_orientation && ReverseRings(geom)) { - if (PrecheckValid(geom) || geom::is_valid(geom)) + if (reversedValid || PrecheckValid(geom) || IsValidIndexed(geom)) return CorrectGeometryResult::Reversed; ReverseRings(geom); } @@ -307,10 +365,241 @@ class OsmLuaProcessing { static std::unordered_map dataStore; static std::mutex dataStoreMutex; + // ---- Memo of way_function (opt-in: way_memo = true in the profile) + // way_function is a function of what it reads (Find, Holds, IsClosed, NextRelation) and what it + // writes is a sequence of calls (Layer, Attribute..., MinZoom...). The first time, a way runs + // through Lua and both are recorded into a decision tree: each node is a read, branched by its + // result, and a leaf holds the calls. A later way that reads the same results reaches the same + // leaf, and the calls are replayed without Lua. Anything else the profile asks for (geometry, + // relations, Id, SetData...) makes that path run through Lua from then on. + struct MemoOp { + enum Kind : uint8_t { Layer, Centroid, Attr, AttrNumeric, AttrBoolean, AttrInteger, FastStr, FastNumeric, FastBoolean, FastInteger, MinZoom, ZOrder, Score, ModifyId, MinZoomByArea } kind; + bool flag = false; + char minzoom = 0; + double number = 0; + std::string key, str; + std::vector numbers; + std::string valueKey; // not empty: str is the current object's value of this tag (see way_memo_values) + bool valueLength = false; // valueKey, numeric ops: the attribute is the value's length in bytes + uint32_t valueKeyId = MemoKeyTable::NONE; // its id in memoKeys + // Filled by the replays (on the tree's thread), see memoReplay + enum Segment : uint8_t { SegmentKnown = 1, SegmentAttributes = 2, SegmentConstant = 4 }; + uint8_t segment = 0; // Layer, Centroid: about the attribute ops up to the next Layer/Centroid op + int16_t layerIndex = -1; // Layer, Centroid: layers.layerMap[key] + uint32_t segmentSet = UINT32_MAX; // Layer, Centroid (constant segment): AttributeIndex of an output with this segment's attributes + uint32_t segmentOps = 0; // Layer, Centroid: the number of ops of its segment (after it) + uint32_t segmentId = 0; // Layer, Centroid: the segment's id in memoSegmentSets + int16_t keyIndex = -1; // attribute ops: attributeStore.keyStore.key2index(key) + int16_t seenLayer = -1; // attribute ops: setVectorLayerMetadata has run for this layer and key + uint32_t pairIndex = 0; // attribute ops without valueKey: the pair added (0: not yet known) + }; + // The tree's nodes are MemoHotNodes (memo_keys.h: kind, key id, the children for the results + // "", "0", "1" and "\x01"); a node with children for other results, or a Leaf, has a MemoMore. + struct MemoNode { + // MercatorArea (way_memo_area): the area compared with each threshold (memoAreaResult); + // FindInRelation (way_memo_relations): the current relation's value of the key + enum Kind : uint8_t { Unset, Find, Holds, IsClosed, NextRelation, Leaf, Lua, MercatorArea, FindInRelation }; + static int quickOf(std::string_view r) { + if (r.empty()) return 0; + if (r.size() != 1) return -1; + return r[0] == '0' ? 1 : r[0] == '1' ? 2 : r[0] == '\x01' ? 3 : -1; + } + }; + struct MemoMore { + std::vector> children; // result -> node, results other than the quick ones + std::vector index; // with many children: open-addressing hash of results -> children position + 1 + std::vector ops; // Leaf + uint32_t uses = 0; // Leaf, Lua: objects that ended here since the last memoCompact + static uint64_t hashOf(std::string_view r) { + uint64_t h = 1469598103934665603ull; + for (unsigned char c : r) { h ^= c; h *= 1099511628211ull; } + return h; + } + uint32_t child(std::string_view r) const { + if (index.empty()) { + for (const auto &c : children) if (c.first.size() == r.size() && c.first == r) return c.second; + return UINT32_MAX; + } + const size_t mask = index.size() - 1; + for (size_t i = hashOf(r) & mask; index[i] != 0; i = (i + 1) & mask) { + const auto &c = children[index[i] - 1]; + if (c.first.size() == r.size() && c.first == r) return c.second; + } + return UINT32_MAX; + } + void addChild(std::string_view r, uint32_t node) { + children.emplace_back(std::string(r), node); + if (children.size() < 8) return; + if (children.size() * 4 > index.size()) { + // (re)build, at most a quarter full + size_t size = 256; + while (size < children.size() * 4) size *= 2; + index.assign(size, 0); + for (size_t k = 0; k + 1 < children.size(); k++) placeChild(k); + } + placeChild(children.size() - 1); + } + void placeChild(size_t k) { + const size_t mask = index.size() - 1; + size_t i = hashOf(children[k].first) & mask; + while (index[i] != 0) i = (i + 1) & mask; + index[i] = k + 1; + } + }; + uint32_t memoChild(uint32_t n, std::string_view r) const { + const MemoHotNode &node = memoNodes[n]; + const int q = MemoNode::quickOf(r); + if (q >= 0) return node.quick[q]; + return node.more == UINT32_MAX ? UINT32_MAX : memoMore[node.more].child(r); + } + MemoMore &memoMoreOf(uint32_t n) { + if (memoNodes[n].more == UINT32_MAX) { memoNodes[n].more = memoMore.size(); memoMore.emplace_back(); } + return memoMore[memoNodes[n].more]; + } + void memoAddChild(uint32_t n, std::string_view r, uint32_t child) { + memoNodes[n].children++; + const int q = MemoNode::quickOf(r); + if (q >= 0) memoNodes[n].quick[q] = child; + else memoMoreOf(n).addChild(r, child); + } + struct MemoRead { MemoNode::Kind kind; std::string key, result; bool valueOnly = false; }; + struct MemoTrace { + bool active = false; + bool luaOnly = false; // asked for something that isn't recorded + // A string attribute had the value of a memo_values tag it isn't declared for: a copy, or the + // same string by chance? Other objects with the same reads can't tell either, so this trace + // isn't added to the tree (they may replay a leaf recorded from an object without the doubt) + bool ambiguous = false; + std::vector reads; + std::vector ops; + std::vector> values; // tags of way_memo_values read, with their values + std::vector emptyValues; // tags of way_memo_values read empty + }; + // way_memo_values: tags whose values way_function only tests for being empty and copies unchanged + // into the listed attributes; the memo replays such ways with the current values + std::unordered_map> memoValueTags; + bool memoValueTag(const char *key, size_t keyLen) const { + return !memoValueTags.empty() && memoValueTags.count(std::string(key, keyLen)) > 0; + } + // way_memo_values entries `attribute = "length"`: attribute -> tag (the attribute is the length of the tag's value) + std::unordered_map memoValueLengths; + // way_memo_area: way_function uses MercatorArea() only to compare it with these numbers, so the + // memo reads it as the results of those comparisons (memoAreaResult) + std::vector memoArea; + // way_memo_relations: way_function uses the id NextRelation() returns only to test whether there + // is one, and the relations' tags only through FindInRelation: the memo reads the roles and tags + bool memoRelations = false; + std::string memoAreaResult(double area) const; + // NextRelation() has returned a relation, and FindInRelation reads its tags + bool relationCurrent() const { return relationSubscript >= 0 && size_t(relationSubscript) < relationList.size(); } + // Record a string attribute: a copy of a way_memo_values tag becomes a placeholder + void memoStringAttribute(MemoOp &&op); + // Record a numeric attribute: the length of a way_memo_values tag, declared so, becomes a placeholder + void memoNumberAttribute(MemoOp &&op); + bool memoEnabled = false; + MemoTrace memoTrace; + std::vector memoNodes; + std::vector memoMore; + MemoKeyTable memoKeys; // the keys the tree reads, and those of way_memo_values + MemoTagSlots memoSlots; // the current way's tags with those keys + MemoSigCache memoWalkCache { 4096 }; // recent walks (of the tree in memoNodes) + // AttributeIndex of outputs of segments with way_memo_values attributes, by segment and values + MemoSigCache memoSegmentSets { 1024 }; + uint32_t memoSegmentIds = 0; // segment ids given out + size_t memoSegmentSig(const MemoOp &layerOp, char *sig) const; + protozero::data_view memoOpValue(const MemoOp &op) const; + size_t memoJumpsAt = 0; // memoNodes.size() when the jumps were last computed + uint64_t memoHits = 0, memoRecorded = 0, memoLua = 0; + // The memo of node_function (opt-in: node_memo = true, node_memo_values as way_memo_values) is + // the same, with its own tree; so is that of way_function for multipolygon relations (with + // way_memo, a tree of its own: LayerAsCentroid, Layer and the geometry act differently for + // them). The members above hold the memo of the objects being processed (memoKind), + // memoSaved those of the other kinds; memoSelect swaps them. + enum MemoKind : uint8_t { MemoWays, MemoNodes, MemoRelations, MEMO_KINDS }; + struct MemoState { + bool enabled = false; + std::unordered_map> valueTags; + std::unordered_map valueLengths; + std::vector area; + bool relations = false; + std::vector nodes; + std::vector more; + MemoKeyTable keys; + size_t jumpsAt = 0, fullMisses = 0; + uint64_t hits = 0, recorded = 0, lua = 0; + }; + MemoState memoSaved[MEMO_KINDS]; // [memoKind] is unused + MemoKind memoKind = MemoWays; + bool memoForNodes = false; // memoKind == MemoNodes + bool nodeMemoEnabled = false; // node_memo = true in the profile + void memoSwap(MemoState &o) { + std::swap(memoEnabled, o.enabled); + std::swap(memoValueTags, o.valueTags); + std::swap(memoValueLengths, o.valueLengths); + std::swap(memoArea, o.area); + std::swap(memoRelations, o.relations); + std::swap(memoNodes, o.nodes); + std::swap(memoMore, o.more); + std::swap(memoKeys, o.keys); + std::swap(memoJumpsAt, o.jumpsAt); + std::swap(memoHits, o.hits); + std::swap(memoRecorded, o.recorded); + std::swap(memoLua, o.lua); + std::swap(memoFullMisses, o.fullMisses); + } + void memoSelect(MemoKind kind) { + if (kind == memoKind) return; + memoSwap(memoSaved[memoKind]); // the current memo to its place, the unused state here + memoSwap(memoSaved[kind]); // and that to the place of the one selected + memoWalkCache.clear(); + memoKind = kind; + memoForNodes = kind == MemoNodes; + } + void memoSelect(bool nodes) { memoSelect(nodes ? MemoNodes : MemoWays); } + // hits, recorded, through Lua, tree nodes of each memo (by MemoKind) + void memoCounts(uint64_t counts[MEMO_KINDS][4]) const { + for (int k = 0; k < MEMO_KINDS; k++) { + uint64_t *c = counts[k]; + if (k == memoKind) { c[0] += memoHits; c[1] += memoRecorded; c[2] += memoLua; c[3] += memoNodes.size(); } + else { const MemoState &o = memoSaved[k]; c[0] += o.hits; c[1] += o.recorded; c[2] += o.lua; c[3] += o.nodes.size(); } + } + } + void memoRead(MemoNode::Kind kind, const char *key, size_t keyLen, const std::string &result) { + if (!memoTrace.luaOnly) memoTrace.reads.push_back({ kind, std::string(key, keyLen), result }); + } + // Find() of a way_memo_values tag: the branch is only whether it is empty + void memoReadValue(const char *key, size_t keyLen, const std::string &value) { + if (memoTrace.luaOnly) return; + memoTrace.reads.push_back({ MemoNode::Find, std::string(key, keyLen), value.empty() ? std::string() : std::string(1, '\x01'), true }); + if (value.empty()) { + for (const auto &e : memoTrace.emptyValues) if (e.size() == keyLen && e.compare(0, keyLen, key, keyLen) == 0) return; + memoTrace.emptyValues.emplace_back(key, keyLen); + return; + } + for (const auto &v : memoTrace.values) if (v.first.size() == keyLen && v.first.compare(0, keyLen, key, keyLen) == 0) return; // read before + memoTrace.values.emplace_back(std::string(key, keyLen), value); + } + void memoOp(MemoOp &&op) { memoTrace.ops.push_back(std::move(op)); } + void memoLuaOnly() { memoTrace.luaOnly = true; } + void runWayFunction(); // way_function (memoForNodes: node_function) through the memo + void runWayFunctionInLua(bool record); + void memoInsert(); + size_t memoFullMisses = 0; // objects since the last memoCompact that weren't recorded as the tree was full + void memoCompact(); + void memoReplay(std::vector &ops); + void memoAttribute(MemoOp &op, const protozero::data_view &str); + void memoSegmentAttributes(std::vector &ops, size_t layerOp); + void layerAsCentroid(const std::string &layerName, CentroidAlgorithm algorithm, const std::vector &relationRoles); + void layerAsCentroidAt(unsigned layerIndex, CentroidAlgorithm algorithm, const std::vector &relationRoles); + void layerAt(unsigned layerIndex, bool area); + static void printMemoStats(); private: /// Internal: clear current cached state inline void reset() { outputs.clear(); + outputPreset.clear(); + outputLearn.clear(); + outputSegment.clear(); currentRelation = nullptr; stringTable = nullptr; llVecPtr = nullptr; @@ -376,6 +665,7 @@ class OsmLuaProcessing { bool relationAccepted; // in scanRelation, whether we're using a non-MP relation std::vector> relationList; // in processNode/processWay, list of relations this entity is in, and its role + RelationScanStore::WayCursor wayRelationsCursor; // setWay's position in the relations of ways int relationSubscript = -1; // in processWay, position in the relation list int32_t lon,latp; ///< Node coordinates @@ -408,6 +698,12 @@ class OsmLuaProcessing { class LayerDefinition &layers; std::vector> outputs; // All output objects that have been created + // Memo replay, per output (may be shorter than outputs): its AttributeIndex if known already + // (UINT32_MAX: not), the layer op to tell its AttributeIndex (segmentSet, or memoSegmentSets for + // a segment with way_memo_values attributes; or nullptr), the layer op's position + std::vector outputPreset; + std::vector outputLearn; + std::vector outputSegment; // Point features in decluttered layers: the exact location we'd have indexed them at, // plus the score from Score(). Sparse - most objects add nothing here. diff --git a/include/osm_mem_tiles.h b/include/osm_mem_tiles.h index 1966bc4b..5ab90fc1 100644 --- a/include/osm_mem_tiles.h +++ b/include/osm_mem_tiles.h @@ -57,6 +57,8 @@ class OsmMemTiles : public TileDataSource { void populateLinestring(Linestring& ls, NodeID objectID) const; Linestring& getOrBuildLinestring(NodeID objectID) const; void populateMultiPolygon(MultiPolygon& dst, NodeID objectID) override; + // Polygons from ways are built from the way store, only relations are in the multipolygon store + bool storesMultiPolygon(NodeID objectID) const override { return objectID < OSM_THRESHOLD; } // Way-store polygons are only emitted by OsmLuaProcessing::Layer when // CorrectGeometry found them Valid (or Reversed: valid after reversing the ring) // on the same lon/latp coordinates. diff --git a/include/osm_store.h b/include/osm_store.h index 4f50cd5f..3eca1590 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -12,8 +12,13 @@ #include #include #include +#include +#include #include #include +#ifdef _MSC_VER +#include +#endif #include extern bool verbose; @@ -53,32 +58,76 @@ struct DataViewLessThan { class UsedWays { private: - // Sparse list of used way IDs: only member ways of accepted relations - // are inserted, which is a tiny fraction of all ways. Appended unsorted - // during the relation scan, then sorted/deduplicated by finalize(). - // If the used set is large relative to the ID range (planet-scale inputs), - // finalize() switches to a dense bitmap, which is then smaller and faster. - std::vector ids; - std::vector dense; + // Member ways of accepted relations. While the relations are scanned, each thread collects its + // own list (insertMany, no lock); finalize() turns them into the set: a bitmap if it is dense + // relative to the ID range (planet-scale inputs), else a sorted list. + struct Collected { std::vector ids; }; + std::mutex registryMutex; + std::vector> collected; // one per thread that inserted + std::vector ids; // sparse set (sorted, unique) + std::vector dense; // dense set: bit i of word i/64 bool isDense = false; - size_t dedupAt = 1 << 20; std::atomic sorted{true}; mutable std::mutex mutex; + const uint64_t instance = nextInstance(); // never reused, unlike the address - void dedupLocked() { - std::sort(ids.begin(), ids.end()); - ids.erase(std::unique(ids.begin(), ids.end()), ids.end()); + static uint64_t nextInstance() { + static std::atomic next(0); + return next++; } + std::vector &mine() { + thread_local std::vector *>> lists; + for (auto &e : lists) if (e.first == instance) return *e.second; + std::lock_guard lock(registryMutex); + collected.emplace_back(new Collected()); + lists.emplace_back(instance, &collected.back()->ids); + return collected.back()->ids; + } + + // All collected lists (and the sparse list) into the set; with mutex held void finalizeLocked() { - dedupLocked(); - if (!ids.empty() && ids.size() * 64 > ids.back()) { - dense.resize(ids.back() + 1); - for (WayID id : ids) dense[id] = true; + std::vector *> lists; + { + std::lock_guard lock(registryMutex); + for (auto &c : collected) if (!c->ids.empty()) lists.push_back(&c->ids); + } + size_t total = ids.size(); + WayID maxId = ids.empty() ? 0 : ids.back(); + for (auto *l : lists) { + total += l->size(); + for (WayID id : *l) maxId = std::max(maxId, id); + } + if (!isDense && total > 0 && total * 64 > maxId) { + // dense: set the bits on all threads (atomic OR, as lists overlap) + dense.assign(maxId / 64 + 1, 0); isDense = true; - ids.clear(); + for (WayID id : ids) dense[id / 64] |= uint64_t(1) << (id % 64); + std::vector().swap(ids); } - ids.shrink_to_fit(); + if (isDense) { + for (auto *l : lists) for (WayID id : *l) if (id / 64 >= dense.size()) dense.resize(id / 64 + 1, 0); + std::atomic next(0); + const unsigned workers = std::max(1u, std::min(std::thread::hardware_concurrency(), lists.size())); + std::vector threads; + for (unsigned t = 0; t < workers; t++) + threads.emplace_back([&]() { + for (size_t k = next++; k < lists.size(); k = next++) + for (WayID id : *lists[k]) +#ifdef _MSC_VER + _InterlockedOr64(reinterpret_cast(&dense[id / 64]), static_cast<__int64>(uint64_t(1) << (id % 64))); +#else + __atomic_fetch_or(&dense[id / 64], uint64_t(1) << (id % 64), __ATOMIC_RELAXED); +#endif + }); + for (auto &t : threads) t.join(); + } else { + for (auto *l : lists) ids.insert(ids.end(), l->begin(), l->end()); + std::sort(ids.begin(), ids.end()); + ids.erase(std::unique(ids.begin(), ids.end()), ids.end()); + ids.shrink_to_fit(); + } + for (auto *l : lists) std::vector().swap(*l); sorted.store(true, std::memory_order_release); } @@ -89,27 +138,23 @@ class UsedWays { std::lock_guard lock(mutex); if (inited) return; inited = true; - ids.reserve(1 << 16); + } + + // Mark ways as used (collected per thread until finalize) + void insertMany(const std::vector &wayids) { + if (wayids.empty()) return; + std::vector &list = mine(); + list.insert(list.end(), wayids.begin(), wayids.end()); + sorted.store(false, std::memory_order_relaxed); } // Mark a way as used void insert(WayID wayid) { - std::lock_guard lock(mutex); - if (isDense) { - if (wayid >= dense.size()) dense.resize(wayid + 1); - dense[wayid] = true; - return; - } - ids.push_back(wayid); + mine().push_back(wayid); sorted.store(false, std::memory_order_relaxed); - // Bound memory when ways are members of many relations (e.g. routes) - if (ids.size() >= dedupAt) { - dedupLocked(); - dedupAt = std::max(dedupAt, ids.size() * 2); - } } - // Sort and deduplicate; call once all inserts are done, before any at() + // Build the set; call once all inserts are done, before any at() void finalize() { std::lock_guard lock(mutex); finalizeLocked(); @@ -122,18 +167,21 @@ class UsedWays { if (!sorted.load(std::memory_order_relaxed)) finalizeLocked(); } if (isDense) - return wayid < dense.size() && dense[wayid]; + return wayid / 64 < dense.size() && ((dense[wayid / 64] >> (wayid % 64)) & 1); return std::binary_search(ids.begin(), ids.end(), wayid); } void clear() { std::lock_guard lock(mutex); + { + std::lock_guard lock2(registryMutex); + for (auto &c : collected) std::vector().swap(c->ids); + } ids.clear(); ids.shrink_to_fit(); dense.clear(); dense.shrink_to_fit(); isDense = false; - dedupAt = 1 << 20; sorted.store(true, std::memory_order_release); } }; @@ -149,27 +197,97 @@ class RelationScanStore { mutable std::vector mutex; RelationRoles relationRoles; + // Every way is looked up once while reading ways, mostly not found. relationsForWays is + // sharded by way id % 128, and each shard (a map) is ordered by way id: wayIndex holds each + // shard as a sorted array (filled during the relation scan, before; rebuilt on first use after + // relationsForWays changed). Ways are read in id order, so a reader that keeps where it was + // in each shard (WayCursor) finds the next way a few entries further on, in the same cache + // lines, instead of a random access into one big table. + using WayRelations = std::vector>; + struct WayIndexSlot { WayID id; WayRelations* relations; }; + std::vector> wayIndex; + std::atomic wayIndexStale { true }; + std::mutex wayIndexMutex; + void ensureWayIndex() { + if (wayIndexStale.load(std::memory_order_acquire)) { + std::lock_guard lock(wayIndexMutex); + if (wayIndexStale.load(std::memory_order_relaxed)) { + std::vector> index(relationsForWays.size()); + for (size_t s = 0; s < relationsForWays.size(); s++) { + index[s].reserve(relationsForWays[s].size()); + for (auto& entry : relationsForWays[s]) index[s].push_back({ entry.first, &entry.second }); + } + wayIndex.swap(index); + wayIndexStale.store(false, std::memory_order_release); + } + } + } + // The way's entry in shard `slots` (nullptr: none), searched from `pos` onwards when the way + // isn't before it (galloping), else in [0, pos); `pos` is left at the way's position + static WayRelations* findInShard(const std::vector& slots, WayID wayid, uint32_t& pos) { + const size_t n = slots.size(); + size_t lo, hi; // the way's position is in [lo, hi] + if (pos < n && slots[pos].id <= wayid) { + lo = pos; size_t step = 1; + while (lo + step < n && slots[lo + step].id < wayid) { lo += step; step *= 2; } + hi = std::min(lo + step, n); + } else { + lo = 0; hi = std::min(pos, n); + } + const auto it = std::lower_bound(slots.begin() + lo, slots.begin() + hi, wayid, + [](const WayIndexSlot& s, WayID id) { return s.id < id; }); + pos = it - slots.begin(); + return it != slots.end() && it->id == wayid ? it->relations : nullptr; + } + WayRelations* findWay(WayID wayid) { + ensureWayIndex(); + uint32_t pos = 0; + return findInShard(wayIndex[wayid % wayIndex.size()], wayid, pos); + } + +public: + // A reader's positions in wayIndex, see findWayRelations + struct WayCursor { std::vector pos; }; + // The relations way `wayid` is in (nullptr: none), as relations_for_way; for a reader going + // through ways in id order, with its own cursor + const WayRelations* findWayRelations(WayID wayid, WayCursor& cursor) { + ensureWayIndex(); + if (cursor.pos.size() != wayIndex.size()) cursor.pos.assign(wayIndex.size(), 0); + const size_t shard = wayid % wayIndex.size(); + return findInShard(wayIndex[shard], wayid, cursor.pos[shard]); + } + public: std::map>> relationsForRelations; RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + // Keep each member's parent relations ordered by relation id: relations are scanned on + // several threads, so appending would make the order NextRelation() returns depend on + // thread scheduling. Entries of the same relation keep their member order. + static void insertByRelation(std::vector>& list, RelationID relid, uint16_t roleId) { + const auto it = std::upper_bound(list.begin(), list.end(), relid, + [](RelationID id, const std::pair& entry) { return id < entry.first; }); + list.emplace(it, relid, roleId); + } + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForWays[shard][wayid], relid, roleId); + wayIndexStale.store(true, std::memory_order_release); } void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForNodes[shard][nodeId], relid, roleId); } void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); std::lock_guard lock(mutex[0]); - relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForRelations[relationId], relid, roleId); } void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); @@ -182,8 +300,7 @@ class RelationScanStore { relationTags[shard][relid][key] = value; } bool way_in_any_relations(WayID wayid) { - const size_t shard = wayid % mutex.size(); - return relationsForWays[shard].find(wayid) != relationsForWays[shard].end(); + return findWay(wayid) != nullptr; } bool node_in_any_relations(NodeID nodeId) { const size_t shard = nodeId % mutex.size(); @@ -194,6 +311,8 @@ class RelationScanStore { } std::string getRole(uint16_t roleId) const { return relationRoles.getRole(roleId); } const std::vector>& relations_for_way(WayID wayid) { + if (WayRelations* relations = findWay(wayid)) return *relations; + // Not a member of any relation (callers check first): as before, an empty list const size_t shard = wayid % mutex.size(); return relationsForWays[shard][wayid]; } @@ -347,6 +466,7 @@ class OSMStore void relations_sort(unsigned int threadNum); void mark_way_used(WayID i) { used_ways.insert(i); } + void mark_ways_used(const std::vector &ids) { used_ways.insertMany(ids); } bool way_is_used(WayID i) { return used_ways.at(i); } void ensureUsedWaysInited(); @@ -360,7 +480,51 @@ class OSMStore // Relation -> MultiPolygon or MultiLinestring MultiPolygon wayListMultiPolygon(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd, WayVec::const_iterator innerBegin, WayVec::const_iterator innerEnd) const; MultiLinestring wayListMultiLinestring(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd) const; - void mergeMultiPolygonWays(std::vector &results, std::map &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const; + // A ring or line being joined from ways: points added at either end, in one buffer with room + // in front (instead of a std::deque) + struct LatpLonChain { + std::vector buf; + size_t head = 0; // the points are buf[head...] + LatpLonChain() = default; + template LatpLonChain(It b, It e) : buf(b, e) { } + explicit LatpLonChain(std::vector &&v) : buf(std::move(v)) { } + const LatpLon &front() const { return buf[head]; } + const LatpLon &back() const { return buf.back(); } + std::vector::const_iterator begin() const { return buf.cbegin() + head; } + std::vector::const_iterator end() const { return buf.cend(); } + size_t size() const { return buf.size() - head; } + template void append(It b, It e) { buf.insert(buf.end(), b, e); } + template void prepend(It b, It e) { + const size_t n = std::distance(b, e); + if (n > head) { + // move the points back, leaving room for these and as many again as there are + const size_t count = size(), room = n + count; + std::vector grown(room + count); + std::copy(begin(), end(), grown.begin() + room); + buf.swap(grown); + head = room; + } + head -= n; + std::copy(b, e, buf.begin() + head); + } + }; + // The ways merged (mergeMultiPolygonWays): a short list, a hash set once there are many + class DoneWays { + std::vector few; + std::unordered_set many; + public: + bool has(WayID w) const { + if (!many.empty()) return many.count(w) > 0; + return std::find(few.begin(), few.end(), w) != few.end(); + } + void add(WayID w) { + if (!many.empty()) { many.insert(w); return; } + if (std::find(few.begin(), few.end(), w) != few.end()) return; + few.push_back(w); + if (few.size() > 32) many.insert(few.begin(), few.end()); + } + }; + void mergeMultiPolygonWays(std::vector &results, DoneWays &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const; ///It is not really meaningful to try using a relation as a linestring. Not normally used but included ///if Lua script attempts to do this. diff --git a/include/output_object.h b/include/output_object.h index 146e39b2..621783c9 100644 --- a/include/output_object.h +++ b/include/output_object.h @@ -100,6 +100,12 @@ class MvtAttributeMemo { public: void reset(vtzero::layer_builder& layer, char zoom); + // Is the memo set up for this layer and zoom? + bool usableFor(const vtzero::layer_builder& l, char z) const { return layer == &l && zoom == z; } + + // Returns the [key, value] index pairs for the set (count pairs) + const uint32_t* resolve(const AttributeStore& attributeStore, AttributeIndex index, uint32_t& count); + private: friend class OutputObject; @@ -114,12 +120,15 @@ class MvtAttributeMemo { void clear(); }; - // Returns the [key, value] index pairs for the set (count pairs) - const uint32_t* resolve(const AttributeStore& attributeStore, AttributeIndex index, uint32_t& count); void resolvePair(const AttributeStore& attributeStore, uint32_t pairIndex, uint32_t& key, uint32_t& value); vtzero::layer_builder* layer = nullptr; char zoom = 0; + // The pairs of an attribute set in canonical order (forEachPairIndexCanonicalUnsafe), kept across + // layers and tiles: AttributeIndex -> [begin, end) in canonPairs + const AttributeStore* canonStore = nullptr; + Table canon; + std::vector canonPairs; uint32_t layerGen = 0; // validity of keyGen entries Table sets; // AttributeIndex -> [begin, end) in entries Table pairs; // pair index -> key, value index (key NO_INDEX: not written at this zoom) diff --git a/include/pbf_processor.h b/include/pbf_processor.h index 2668b79e..c593444d 100644 --- a/include/pbf_processor.h +++ b/include/pbf_processor.h @@ -102,7 +102,12 @@ class PbfProcessor uint effectiveShards ); bool ScanWays(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys); - bool ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys); + // Relation blocks: member ways of the largest multipolygon to be processed, found by the + // relation scan (by block offset), so that the blocks with the slowest relations are read first + std::mutex relationBlockCostMutex; + std::map relationBlockCost; + + bool ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys, uint64_t& cost); bool ReadRelations( OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, @@ -110,7 +115,8 @@ class PbfProcessor const BlockMetadata& blockMetadata, const SignificantTags& wayKeys, uint shard, - uint effectiveShards + uint effectiveShards, + int onlyPass = -1 // 0: only relations with many member ways, 1: only the others ); inline bool relationIsType(const PbfReader::Relation& rel, int typeKey, int val) { diff --git a/include/pbf_reader.h b/include/pbf_reader.h index 756c9776..80030e1f 100644 --- a/include/pbf_reader.h +++ b/include/pbf_reader.h @@ -138,6 +138,27 @@ namespace PbfReader { void readDenseNodes(protozero::data_view data); }; + // DenseNodes iteration, inline (called per node) + inline bool DenseNodes::Iterator::operator!=(Iterator& other) const { + return offset != other.offset; + } + + inline void DenseNodes::Iterator::operator++() { + offset++; + + if (offset < nodes.ids.size()) { + node.id = nodes.ids[offset]; + node.lon = nodes.lons[offset]; + node.lat = nodes.lats[offset]; + node.tagStart = nodes.tagStart[offset]; + node.tagEnd = nodes.tagEnd[offset]; + } + } + + inline DenseNodes::Node& DenseNodes::Iterator::operator*() { + return node; + } + struct Way { uint64_t id; std::vector keys; diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..05e2ff6b 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -3,7 +3,13 @@ #define _PMTILES_H #include +#include +#include +#include +#include +#include #include "external/pmtiles.hpp" +#include "ordered_batches.h" struct TileOffset { uint64_t offset : 40; @@ -28,6 +34,14 @@ struct TileOffset { // PMTiles v3 tile type for MapLibre Tiles (not yet in external/pmtiles.hpp) #define TILETYPE_MLT 0x6 +// A tile waiting to be written: tiny tiles keep their raw data, so that a repeat +// can reuse the earlier copy, and are only compressed if they're written +struct PendingTile { + uint64_t tileId; + bool isTiny; + std::string data; // after writeTile: the bytes flushTile appends to the file (empty if a repeat) +}; + class PMTiles { public: @@ -37,26 +51,41 @@ class PMTiles { pmtiles::headerv3 header; bool isSparse = true; uint8_t tileType = pmtiles::TILETYPE_MVT; + int compressionLevel = 6; // for tiles; the directories and metadata use the default void open(std::string &filename); void saveTile(int zoom, int x, int y, std::string &data); void close(std::string &metadata); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + private: std::ofstream outputStream; - std::mutex fileMutex; // guards file writes, numTilesWritten - std::mutex indexMutex; // guards access to sparseIndex, denseIndex, tinyCache, numTilesAddressed + std::string filename; + std::vector> laterGarbage; // freed with the object (see close) + std::vector outputBuffer; // a large stream buffer: one thread writes all tiles + // Tile data past the page cache (O_DIRECT), when the file system allows it - see pmtiles.cpp + struct DirectWriter; + std::unique_ptr direct; + uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; uint64_t numTileEntries = 0; std::map sparseIndex; - std::vector denseIndex; + // The dense index (tile id -> offset) in blocks, allocated when their first tile is placed: one + // vector up to the highest id so far took the planet's whole id range (2.9 GB) as soon as the + // first tiles of the last Hilbert quarter were written, twice that while it grew, and an extract + // all of its ids' range. Ids from denseSize on have no tile, nor have those of a missing block. + static constexpr unsigned DENSE_BLOCK_BITS = 14; + std::vector> denseIndex; + uint64_t denseSize = 0; std::unordered_map tinyCache; + void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile + void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); - void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); - void flushEntries(std::vector &rootEntries, std::vector &entries); }; #endif //_PMTILES_H diff --git a/include/polylabel.h b/include/polylabel.h index 6ce44f16..7585674f 100644 --- a/include/polylabel.h +++ b/include/polylabel.h @@ -28,10 +28,13 @@ #include #include #include +#include // 2-wide SSE2 distance kernel. Only where the compiler cannot contract mul+add into FMA (that // would round differently from the scalar original); elsewhere the original scalar loop is used. -#if (defined(__x86_64__) || defined(_M_X64)) && !defined(__FMA__) && !defined(__FMA4__) && !defined(__AVX2__) && (!defined(__FLT_EVAL_METHOD__) || __FLT_EVAL_METHOD__ == 0) +// GCC/Clang only (the AVX2 kernels use target attributes and __builtin_cpu_supports). +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(__FMA__) && !defined(__FMA4__) && !defined(__AVX2__) && (!defined(__FLT_EVAL_METHOD__) || __FLT_EVAL_METHOD__ == 0) #include +#include #define POLYLABEL_SSE2 1 #endif @@ -145,8 +148,9 @@ double pointToPolygonDistKScalar(const Point& point, const Polygon& polygon, int #ifdef POLYLABEL_SSE2 // The edges of the polygon being labelled, one entry per (a = ring[i], b = ring[j]) exactly as // pointToPolygonDistKScalar iterates them (outer ring first, then inners in order), in blocks of two. -// An odd count is padded with a NaN entry: its distance is NaN (never selected by the min) and its -// crossing test compares false, so it changes neither the distance nor the parity. +// The entries are padded to a multiple of four with NaN entries: their distance is NaN (never +// selected by the min) and their crossing test compares false, so they change neither the +// distance nor the parity. struct EdgePair { double ax[2], ay[2], bx[2], by[2], dx[2], dy[2], den[2]; }; @@ -161,7 +165,7 @@ struct PolygonEdges { std::size_t n = p.outer().size(); for (const auto& inner : p.inners()) n += inner.size(); const double nan = std::numeric_limits::quiet_NaN(); - pairs.assign((n + 1) / 2, EdgePair { {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan} }); + pairs.assign(((n + 3) / 4) * 2, EdgePair { {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan}, {nan, nan} }); ring.resize(n); edge.resize(n); std::size_t k = 0; @@ -273,6 +277,81 @@ inline double pointToPolygonDistKSoA(const Point& point, const PolygonEdges& e, return (inside ? 1 : -1) * std::sqrt(minDistSq); } + +// lanes 0, 1 from lo, 2, 3 from hi +__attribute__((target("avx2"))) +static inline __m256d loadEdgeLanes(const double* lo, const double* hi) { + return _mm256_insertf128_pd(_mm256_castpd128_pd256(_mm_loadu_pd(lo)), _mm_loadu_pd(hi), 1); +} + +// pointToPolygonDistKSoA four edges at a time (two EdgePairs), on CPUs with AVX2. The same IEEE +// operations per lane (AVX2 without FMA: nothing can be contracted), the same minimum and parity. +__attribute__((target("avx2"))) +inline double pointToPolygonDistKAvx2(const Point& point, const PolygonEdges& e, int& kr, std::size_t& ki) { + const __m256d px = _mm256_set1_pd(point.get<0>()); + const __m256d py = _mm256_set1_pd(point.get<1>()); + const __m256d one = _mm256_set1_pd(1.0); + const __m256d zero = _mm256_setzero_pd(); + __m256d mn = _mm256_set1_pd(std::numeric_limits::infinity()); + __m256d par = _mm256_setzero_pd(); + __m256i idx = _mm256_set_epi64x(3, 2, 1, 0); + __m256i best = _mm256_setzero_si256(); + const __m256i four = _mm256_set1_epi64x(4); + // lanes 0, 1 from the first EdgePair, 2, 3 from the second + const auto load = loadEdgeLanes; + + const EdgePair* q = e.pairs.data(); + for (const EdgePair* end = q + e.pairs.size(); q != end; q += 2) { + const __m256d ax = load(q[0].ax, q[1].ax); + const __m256d ay = load(q[0].ay, q[1].ay); + const __m256d bx = load(q[0].bx, q[1].bx); + const __m256d by = load(q[0].by, q[1].by); + const __m256d dx = load(q[0].dx, q[1].dx); + const __m256d dy = load(q[0].dy, q[1].dy); + const __m256d pxa = _mm256_sub_pd(px, ax); + const __m256d pya = _mm256_sub_pd(py, ay); + + const __m256d straddle = _mm256_xor_pd(_mm256_cmp_pd(ay, py, _CMP_GT_OQ), _mm256_cmp_pd(by, py, _CMP_GT_OQ)); + const __m256d xi = _mm256_add_pd(_mm256_div_pd(_mm256_mul_pd(dx, pya), dy), ax); + par = _mm256_xor_pd(par, _mm256_and_pd(straddle, _mm256_cmp_pd(px, xi, _CMP_LT_OQ))); + + const __m256d t = _mm256_div_pd(_mm256_add_pd(_mm256_mul_pd(pxa, dx), _mm256_mul_pd(pya, dy)), load(q[0].den, q[1].den)); + const __m256d gt1 = _mm256_cmp_pd(t, one, _CMP_GT_OQ); + const __m256d gt0 = _mm256_cmp_pd(t, zero, _CMP_GT_OQ); + __m256d x = _mm256_or_pd(_mm256_and_pd(gt0, _mm256_add_pd(ax, _mm256_mul_pd(dx, t))), _mm256_andnot_pd(gt0, ax)); + __m256d y = _mm256_or_pd(_mm256_and_pd(gt0, _mm256_add_pd(ay, _mm256_mul_pd(dy, t))), _mm256_andnot_pd(gt0, ay)); + x = _mm256_or_pd(_mm256_and_pd(gt1, bx), _mm256_andnot_pd(gt1, x)); + y = _mm256_or_pd(_mm256_and_pd(gt1, by), _mm256_andnot_pd(gt1, y)); + const __m256d ex = _mm256_sub_pd(px, x); + const __m256d ey = _mm256_sub_pd(py, y); + const __m256d dd = _mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)); + + const __m256i lt = _mm256_castpd_si256(_mm256_cmp_pd(dd, mn, _CMP_LT_OQ)); + mn = _mm256_min_pd(dd, mn); + best = _mm256_blendv_epi8(best, idx, lt); + idx = _mm256_add_epi64(idx, four); + } + + double m4[4]; + _mm256_storeu_pd(m4, mn); + std::int64_t b4[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(b4), best); + const int pm = _mm256_movemask_pd(par); + const bool inside = (__builtin_popcount(pm) & 1) != 0; + + int lane = 0; + for (int l = 1; l < 4; l++) if (m4[l] < m4[lane]) lane = l; + const double minDistSq = m4[lane]; + if (e.ring.empty()) { + kr = -1; + ki = 0; + } else { + kr = e.ring[b4[lane]]; + ki = e.edge[b4[lane]]; + } + + return (inside ? 1 : -1) * std::sqrt(minDistSq); +} #endif // pointToPolygonDistKScalar, via the edge arrays when polylabel() has built them for this polygon @@ -280,6 +359,8 @@ double pointToPolygonDistK(const Point& point, const Polygon& polygon, int& kr, #ifdef POLYLABEL_SSE2 const PolygonEdges* e = activeEdges; if (e && e->polygon == &polygon) { + static const bool avx2 = __builtin_cpu_supports("avx2"); + if (avx2) return pointToPolygonDistKAvx2(point, *e, kr, ki); const double d = pointToPolygonDistKSoA(point, *e, kr, ki); return d; } @@ -329,22 +410,1103 @@ Cell getCentroidCell(const Polygon& polygon) { return Cell(area == 0 ? ring.at(0) : Point { ox + cx / area, oy + cy / area }, 0, polygon); } -// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `near.c`. The full scan +// Upper bound on pointToPolygonDist(p, polygon), from the edge nearest to `nearCell.c`. The full scan // computes min over all edges of getSegDistSq(p, a, b) with the same arguments, so its result is // <= this edge's value; the margin covers FP contraction differences between call sites. -inline double edgeDistBound(const Point& p, const Cell& near, const Polygon& polygon) { - const auto& ring = near.kr < 0 ? polygon.outer() : polygon.inners()[near.kr]; - const Point& a = ring[near.ki]; - const Point& b = ring[near.ki == 0 ? ring.size() - 1 : near.ki - 1]; +inline double edgeDistBound(const Point& p, const Cell& nearCell, const Polygon& polygon) { + const auto& ring = nearCell.kr < 0 ? polygon.outer() : polygon.inners()[nearCell.kr]; + const Point& a = ring[nearCell.ki]; + const Point& b = ring[nearCell.ki == 0 ? ring.size() - 1 : nearCell.ki - 1]; return std::sqrt(getSegDistSq(p, a, b)) * (1 + 1e-12) + 1e-11; } +#ifdef POLYLABEL_SSE2 +// --------------------------------------------------------------------------------------------- +// Fast path of polylabel() for polygons with finite coordinates of magnitude < 1e100 (all real +// input). It returns the bit-identical point: +// - every distance is the same per-edge IEEE operation sequence as getSegDistSq and the crossing +// test (see pointToPolygonDistKSoA), the minimum over the edges and the crossing parity are +// order-independent, so a distance is the same double whichever kernel or lane computes it; +// - cells are processed in the same order: the comparator is a strict total order on cells (no +// NaN can occur with such coordinates), so the pop sequence is "always the largest cell present" +// regardless of push order or heap implementation; +// - only cells that are provably no-ops when popped (neither replace bestCell nor get split, as +// bestCell.d only grows) are dropped; besides the edge bound of the original, a cell whose own +// exact d and max already satisfy that is dropped instead of being pushed; +// - once a popped cell is not split, no later cell is split either (later cells have max <= its +// max, bestCell.d only grows), so the remaining pops only pick the first cell with the largest +// d > bestCell.d in pop order: found by one scan instead of popping them all. +namespace fast { + +// growable array of doubles that is not zero-filled (every entry used is written first) +struct RawBuffer { + std::unique_ptr p; + std::size_t cap = 0; + double* get(std::size_t n) { + if (n > cap) { p.reset(new double[n]); cap = n; } + return p.get(); + } + void release(std::size_t limit) { + if (cap > limit) { p.reset(); cap = 0; } + } +}; + +struct Edges { + RawBuffer buf; + // SoA, npad entries each, 32-byte aligned; entries [n, npad) are NaN (never selected, never cross) + double *ax = nullptr, *ay = nullptr, *bx = nullptr, *by = nullptr, *dx = nullptr, *dy = nullptr, *den = nullptr; + std::size_t n = 0, npad = 0; + // without blocks: edges [nn, n) are the zero-length ones (dx == dy == 0), moved behind the others + // (the order of the edges changes neither the minimum nor the parity); for them the distance is + // the distance to a (t = 0 / 0 = NaN takes neither branch), and they never straddle (ay == by) + std::size_t nn = 0; + + // Blocks of kBlock consecutive edges with their bounding boxes (only when n >= kBlockMinEdges), + // nbpad blocks (multiple of 4; padding blocks are empty boxes at +inf / -inf). + static constexpr std::size_t kBlock = 8; + static constexpr std::size_t kBlockMinEdges = 64; + RawBuffer bbuf; + double *xmin = nullptr, *xmaxm = nullptr, *ymin = nullptr, *ymax = nullptr, *xmax = nullptr; + std::size_t nb = 0, nbpad = 0; + double margin = 0; // absolute bound on rounding excursions of computed points beyond an edge's box + + // With more than kTreeMinBlocks blocks, a 4-ary tree of boxes above them: level 0 are the + // blocks, node i of level l + 1 has the union box of nodes 4i .. 4i + 3 of level l (padding + // nodes are empty boxes); the top level has at most four nodes. A node's tests (below) are + // implied by those of each node under it, so skipping a node skips no block the flat scan + // would evaluate under the same minimum. + static constexpr std::size_t kTreeMinBlocks = 128; + struct Level { const double *xmin, *xmaxm, *ymin, *ymax, *xmax; std::size_t count; }; + static constexpr int kMaxLevels = 16; + Level lv[kMaxLevels]; + int nlv = 0; + RawBuffer tbuf; + + // false if a coordinate is non-finite or |coordinate| >= 1e100 (then the reference path runs) + bool build(const Polygon& p) { + n = p.outer().size(); + for (const auto& inner : p.inners()) n += inner.size(); + const bool blocked = n >= kBlockMinEdges; + npad = blocked ? (n + kBlock - 1) / kBlock * kBlock : (n + 3) & ~std::size_t(3); + double* base = buf.get(7 * npad + 4); + while (reinterpret_cast(base) & 31) base++; + ax = base; ay = ax + npad; bx = ay + npad; by = bx + npad; dx = by + npad; dy = dx + npad; den = dy + npad; + std::size_t k = 0; + bool ok = true; + auto add = [&](const Ring& rg) { + for (std::size_t i = 0, len = rg.size(), j = len - 1; i < len; j = i++, k++) { + const double x = rg[i].get<0>(), y = rg[i].get<1>(); + ok &= std::fabs(x) < 1e100 && std::fabs(y) < 1e100; + ax[k] = x; + ay[k] = y; + bx[k] = rg[j].get<0>(); + by[k] = rg[j].get<1>(); + dx[k] = bx[k] - ax[k]; + dy[k] = by[k] - ay[k]; + den[k] = dx[k] * dx[k] + dy[k] * dy[k]; + } + }; + add(p.outer()); + for (const auto& inner : p.inners()) add(inner); + nn = n; + if (!blocked) { + double zx[kBlockMinEdges], zy[kBlockMinEdges]; + std::size_t w = 0, z = 0; + for (std::size_t i = 0; i < n; i++) { + if (dx[i] == 0 && dy[i] == 0) { zx[z] = ax[i]; zy[z] = ay[i]; z++; continue; } + ax[w] = ax[i]; ay[w] = ay[i]; bx[w] = bx[i]; by[w] = by[i]; dx[w] = dx[i]; dy[w] = dy[i]; den[w] = den[i]; + w++; + } + nn = w; + for (std::size_t i = 0; i < z; i++, w++) { + ax[w] = bx[w] = zx[i]; ay[w] = by[w] = zy[i]; dx[w] = dy[w] = den[w] = 0; + } + } + const double nan = std::numeric_limits::quiet_NaN(); + for (; k < npad; k++) ax[k] = ay[k] = bx[k] = by[k] = dx[k] = dy[k] = den[k] = nan; + nb = 0; + if (!ok || !blocked) return ok; + + nb = npad / kBlock; + nbpad = (nb + 3) & ~std::size_t(3); + double* bb = bbuf.get(5 * nbpad + 4); + while (reinterpret_cast(bb) & 31) bb++; + xmin = bb; xmaxm = xmin + nbpad; ymin = xmaxm + nbpad; ymax = ymin + nbpad; xmax = ymax + nbpad; + double maxAbs = 1; + for (std::size_t b = 0; b < nb; b++) { + double x0 = std::numeric_limits::infinity(), x1 = -x0, y0 = x0, y1 = -x0; + for (std::size_t i = b * kBlock, end = std::min(n, i + kBlock); i < end; i++) { + x0 = std::min(x0, std::min(ax[i], bx[i])); + x1 = std::max(x1, std::max(ax[i], bx[i])); + y0 = std::min(y0, std::min(ay[i], by[i])); + y1 = std::max(y1, std::max(ay[i], by[i])); + } + xmin[b] = x0; xmax[b] = x1; ymin[b] = y0; ymax[b] = y1; + maxAbs = std::max(maxAbs, std::max(std::max(std::fabs(x0), std::fabs(x1)), std::max(std::fabs(y0), std::fabs(y1)))); + } + // A computed nearest point or crossing abscissa of an edge lies within the edge's box up to + // a few ulps of the largest coordinate magnitude (every coordinate and difference is below + // 2 * maxAbs); 1e-12 * maxAbs is > 4000 such ulps. + margin = 1e-12 * maxAbs; + for (std::size_t b = 0; b < nb; b++) xmaxm[b] = xmax[b] + margin; + const double inf = std::numeric_limits::infinity(); + for (std::size_t b = nb; b < nbpad; b++) { xmin[b] = ymin[b] = inf; xmax[b] = xmaxm[b] = ymax[b] = -inf; } + nlv = 0; + if (nbpad > kTreeMinBlocks) { + std::size_t total = 0; + for (std::size_t c = nbpad; c > 4; ) { c = (c / 4 + 3) & ~std::size_t(3); total += c; } + double* tb = tbuf.get(5 * total + 4); + while (reinterpret_cast(tb) & 31) tb++; + lv[0] = Level { xmin, xmaxm, ymin, ymax, xmax, nbpad }; + nlv = 1; + while (lv[nlv - 1].count > 4) { + const Level& c = lv[nlv - 1]; + const std::size_t cnt = c.count / 4, cpad = (cnt + 3) & ~std::size_t(3); + double *x0 = tb, *xm = x0 + cpad, *y0 = xm + cpad, *y1 = y0 + cpad, *x1 = y1 + cpad; + tb = x1 + cpad; + for (std::size_t i = 0; i < cnt; i++) { + const std::size_t f = 4 * i; + x0[i] = std::min(std::min(c.xmin[f], c.xmin[f + 1]), std::min(c.xmin[f + 2], c.xmin[f + 3])); + xm[i] = std::max(std::max(c.xmaxm[f], c.xmaxm[f + 1]), std::max(c.xmaxm[f + 2], c.xmaxm[f + 3])); + y0[i] = std::min(std::min(c.ymin[f], c.ymin[f + 1]), std::min(c.ymin[f + 2], c.ymin[f + 3])); + y1[i] = std::max(std::max(c.ymax[f], c.ymax[f + 1]), std::max(c.ymax[f + 2], c.ymax[f + 3])); + x1[i] = std::max(std::max(c.xmax[f], c.xmax[f + 1]), std::max(c.xmax[f + 2], c.xmax[f + 3])); + } + for (std::size_t i = cnt; i < cpad; i++) { x0[i] = y0[i] = inf; x1[i] = xm[i] = y1[i] = -inf; } + lv[nlv++] = Level { x0, xm, y0, y1, x1, cpad }; + } + } + return ok; + } + + void release() { + buf.release(1u << 15); + bbuf.release(1u << 13); + tbuf.release(1u << 13); + } +}; + +struct FCell { + Point c; // cell center + double h; // half the cell size + double d; // distance from cell center to polygon + double max; // max distance to polygon within a cell + std::uint32_t k; // index of an edge nearest to c +}; + +inline bool lessCell(const FCell& a, const FCell& b) { + if (a.max != b.max) return a.max < b.max; + if (a.d != b.d) return a.d < b.d; + if (a.h != b.h) return a.h > b.h; + if (a.c.get<0>() != b.c.get<0>()) return a.c.get<0>() > b.c.get<0>(); + return a.c.get<1>() > b.c.get<1>(); +} + +// Max-heap of cells under lessCell. Any correct heap pops the same sequence (lessCell is a strict +// total order on distinct cells). Entries carry max, the first key, so most comparisons stay in the +// 16-byte entries; the cells themselves stay put in the pool. +struct HeapEntry { + double max; + std::uint32_t i; +}; + +struct Heap { + // up to kLinear cells an unordered array with a linear scan per pop, then a binary heap + static constexpr std::size_t kLinear = 32; + FCell lin[kLinear + 1]; + double lm[kLinear + 1]; // lin[i].max + std::size_t nl = 0; + bool heap = false; + std::vector pool; + std::vector h; + + void clear() { nl = 0; pool.clear(); h.clear(); heap = false; } + bool empty() const { return heap ? h.empty() : nl == 0; } + bool less(const HeapEntry& a, const HeapEntry& b) const { + return a.max != b.max ? a.max < b.max : lessCell(pool[a.i], pool[b.i]); + } + void push(const FCell& c) { + if (!heap) { + lin[nl] = c; + lm[nl] = c.max; + if (++nl <= kLinear) return; + for (std::size_t i = 0; i < nl; i++) { + pool.push_back(lin[i]); + h.push_back(HeapEntry { lin[i].max, static_cast(i) }); + } + std::make_heap(h.begin(), h.end(), [this](const HeapEntry& a, const HeapEntry& b) { return less(a, b); }); + heap = true; + nl = 0; + return; + } + const HeapEntry e { c.max, static_cast(pool.size()) }; + pool.push_back(c); + std::size_t i = h.size(); + h.push_back(e); + while (i > 0) { + const std::size_t p = (i - 1) / 2; + if (!less(h[p], e)) break; + h[i] = h[p]; + i = p; + } + h[i] = e; + } + FCell pop() { + if (!heap) { + // the largest max (branch-free), then the full comparison only if another cell has it too + const std::size_t n = nl; + std::size_t bi = 0; + double bmax = lm[0]; + for (std::size_t i = 1; i < n; i++) { + const double m = lm[i]; + const bool gt = m > bmax; + bmax = gt ? m : bmax; + bi = gt ? i : bi; + } + int ties = 0; + for (std::size_t i = 0; i < n; i++) ties += lm[i] == bmax; + if (ties > 1) { + for (std::size_t i = 0; i < n; i++) if (lm[i] == bmax && lessCell(lin[bi], lin[i])) bi = i; + } + const FCell r = lin[bi]; + nl = n - 1; + lin[bi] = lin[nl]; + lm[bi] = lm[nl]; + return r; + } + const std::uint32_t top = h[0].i; + const HeapEntry e = h.back(); + h.pop_back(); + const std::size_t n = h.size(); + if (n) { + std::size_t i = 0; + for (;;) { + std::size_t c = 2 * i + 1; + if (c >= n) break; + if (c + 1 < n && less(h[c], h[c + 1])) c++; + if (!less(e, h[c])) break; + h[i] = h[c]; + i = c; + } + h[i] = e; + } + return pool[top]; + } + // the cells still queued, in no particular order + template void forEach(F f) const { + if (!heap) { for (std::size_t i = 0; i < nl; i++) f(lin[i]); return; } + for (const HeapEntry& he : h) f(pool[he.i]); + } + void release() { + if (pool.capacity() > (1u << 12)) { std::vector().swap(pool); std::vector().swap(h); } + } +}; + +// One point, edges in lanes (as pointToPolygonDistKSoA). +inline double distOneSse2(double pxs, double pys, const Edges& e, std::uint32_t& k) { + const __m128d px = _mm_set1_pd(pxs); + const __m128d py = _mm_set1_pd(pys); + const __m128d one = _mm_set1_pd(1.0); + const __m128d zero = _mm_setzero_pd(); + __m128d mn = _mm_set1_pd(std::numeric_limits::infinity()); + __m128d par = _mm_setzero_pd(); + __m128i idx = _mm_set_epi64x(1, 0); + __m128i best = _mm_setzero_si128(); + const __m128i two = _mm_set_epi64x(2, 2); + for (std::size_t i = 0; i < e.npad; i += 2) { + const __m128d ax = _mm_load_pd(e.ax + i); + const __m128d ay = _mm_load_pd(e.ay + i); + const __m128d bx = _mm_load_pd(e.bx + i); + const __m128d by = _mm_load_pd(e.by + i); + const __m128d dx = _mm_load_pd(e.dx + i); + const __m128d dy = _mm_load_pd(e.dy + i); + const __m128d pxa = _mm_sub_pd(px, ax); + const __m128d pya = _mm_sub_pd(py, ay); + const __m128d straddle = _mm_xor_pd(_mm_cmpgt_pd(ay, py), _mm_cmpgt_pd(by, py)); + const __m128d xi = _mm_add_pd(_mm_div_pd(_mm_mul_pd(dx, pya), dy), ax); + par = _mm_xor_pd(par, _mm_and_pd(straddle, _mm_cmplt_pd(px, xi))); + const __m128d t = _mm_div_pd(_mm_add_pd(_mm_mul_pd(pxa, dx), _mm_mul_pd(pya, dy)), _mm_load_pd(e.den + i)); + const __m128d gt1 = _mm_cmpgt_pd(t, one); + const __m128d gt0 = _mm_cmpgt_pd(t, zero); + __m128d x = _mm_or_pd(_mm_and_pd(gt0, _mm_add_pd(ax, _mm_mul_pd(dx, t))), _mm_andnot_pd(gt0, ax)); + __m128d y = _mm_or_pd(_mm_and_pd(gt0, _mm_add_pd(ay, _mm_mul_pd(dy, t))), _mm_andnot_pd(gt0, ay)); + x = _mm_or_pd(_mm_and_pd(gt1, bx), _mm_andnot_pd(gt1, x)); + y = _mm_or_pd(_mm_and_pd(gt1, by), _mm_andnot_pd(gt1, y)); + const __m128d ex = _mm_sub_pd(px, x); + const __m128d ey = _mm_sub_pd(py, y); + const __m128d dd = _mm_add_pd(_mm_mul_pd(ex, ex), _mm_mul_pd(ey, ey)); + const __m128i lt = _mm_castpd_si128(_mm_cmplt_pd(dd, mn)); + mn = _mm_min_pd(dd, mn); + best = _mm_or_si128(_mm_and_si128(lt, idx), _mm_andnot_si128(lt, best)); + idx = _mm_add_epi64(idx, two); + } + double m2[2]; + _mm_storeu_pd(m2, mn); + std::int64_t b2[2]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(b2), best); + const int pm = _mm_movemask_pd(par); + const bool inside = ((pm ^ (pm >> 1)) & 1) != 0; + const int lane = m2[1] < m2[0] ? 1 : 0; + k = static_cast(b2[lane]); + return (inside ? 1 : -1) * std::sqrt(m2[lane]); +} + +__attribute__((target("avx2"))) +inline double distOneAvx2(double pxs, double pys, const Edges& e, std::uint32_t& k) { + const __m256d px = _mm256_set1_pd(pxs); + const __m256d py = _mm256_set1_pd(pys); + const __m256d one = _mm256_set1_pd(1.0); + const __m256d zero = _mm256_setzero_pd(); + __m256d mn = _mm256_set1_pd(std::numeric_limits::infinity()); + __m256d par = _mm256_setzero_pd(); + __m256i idx = _mm256_set_epi64x(3, 2, 1, 0); + __m256i best = _mm256_setzero_si256(); + const __m256i four = _mm256_set1_epi64x(4); + for (std::size_t i = 0; i < e.npad; i += 4) { + const __m256d ax = _mm256_load_pd(e.ax + i); + const __m256d ay = _mm256_load_pd(e.ay + i); + const __m256d bx = _mm256_load_pd(e.bx + i); + const __m256d by = _mm256_load_pd(e.by + i); + const __m256d dx = _mm256_load_pd(e.dx + i); + const __m256d dy = _mm256_load_pd(e.dy + i); + const __m256d pxa = _mm256_sub_pd(px, ax); + const __m256d pya = _mm256_sub_pd(py, ay); + const __m256d straddle = _mm256_xor_pd(_mm256_cmp_pd(ay, py, _CMP_GT_OQ), _mm256_cmp_pd(by, py, _CMP_GT_OQ)); + const __m256d xi = _mm256_add_pd(_mm256_div_pd(_mm256_mul_pd(dx, pya), dy), ax); + par = _mm256_xor_pd(par, _mm256_and_pd(straddle, _mm256_cmp_pd(px, xi, _CMP_LT_OQ))); + const __m256d t = _mm256_div_pd(_mm256_add_pd(_mm256_mul_pd(pxa, dx), _mm256_mul_pd(pya, dy)), _mm256_load_pd(e.den + i)); + const __m256d gt1 = _mm256_cmp_pd(t, one, _CMP_GT_OQ); + const __m256d gt0 = _mm256_cmp_pd(t, zero, _CMP_GT_OQ); + __m256d x = _mm256_blendv_pd(ax, _mm256_add_pd(ax, _mm256_mul_pd(dx, t)), gt0); + __m256d y = _mm256_blendv_pd(ay, _mm256_add_pd(ay, _mm256_mul_pd(dy, t)), gt0); + x = _mm256_blendv_pd(x, bx, gt1); + y = _mm256_blendv_pd(y, by, gt1); + const __m256d ex = _mm256_sub_pd(px, x); + const __m256d ey = _mm256_sub_pd(py, y); + const __m256d dd = _mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)); + const __m256i lt = _mm256_castpd_si256(_mm256_cmp_pd(dd, mn, _CMP_LT_OQ)); + mn = _mm256_min_pd(dd, mn); + best = _mm256_blendv_epi8(best, idx, lt); + idx = _mm256_add_epi64(idx, four); + } + double m4[4]; + _mm256_storeu_pd(m4, mn); + std::int64_t b4[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(b4), best); + const int pm = _mm256_movemask_pd(par); + const bool inside = (__builtin_popcount(pm) & 1) != 0; + int lane = 0; + for (int l = 1; l < 4; l++) if (m4[l] < m4[lane]) lane = l; + k = static_cast(b4[lane]); + return (inside ? 1 : -1) * std::sqrt(m4[lane]); +} + +// distOneAvx2 over the edges of the blocks that can matter. A block is skipped when +// - none of its edges can flip the parity: py >= ymax or py < ymin (no edge straddles py), or +// px > xmax + margin (every straddling edge's computed crossing abscissa xi <= xmax + margin, +// so px < xi is false), and +// - its edges cannot lower the minimum: a conservative lower bound on every computed squared +// distance of its edges (from the gap between p and the box, less twice the margin, shrunk by +// a relative 1e-12) exceeds the minimum so far. +// The skipped edges contribute nothing to the parity and only values above the final minimum, so +// the result is the same double; the block holding the hint edge (nearest to a nearby point) is +// evaluated first to get a small minimum early. +__attribute__((target("avx2"))) +inline double distBlockedAvx2(double pxs, double pys, const Edges& e, std::uint32_t hint, std::uint32_t& k) { + const __m256d px = _mm256_set1_pd(pxs); + const __m256d py = _mm256_set1_pd(pys); + const __m256d one = _mm256_set1_pd(1.0); + const __m256d zero = _mm256_setzero_pd(); + __m256d mn = _mm256_set1_pd(std::numeric_limits::infinity()); + __m256d par = _mm256_setzero_pd(); + __m256i best = _mm256_setzero_si256(); + + auto evalBlock = [&](std::size_t b) __attribute__((target("avx2"), always_inline)) { + for (std::size_t i = b * Edges::kBlock, end = i + Edges::kBlock; i < end; i += 4) { + const __m256d ax = _mm256_load_pd(e.ax + i); + const __m256d ay = _mm256_load_pd(e.ay + i); + const __m256d bx = _mm256_load_pd(e.bx + i); + const __m256d by = _mm256_load_pd(e.by + i); + const __m256d dx = _mm256_load_pd(e.dx + i); + const __m256d dy = _mm256_load_pd(e.dy + i); + const __m256d pxa = _mm256_sub_pd(px, ax); + const __m256d pya = _mm256_sub_pd(py, ay); + const __m256d straddle = _mm256_xor_pd(_mm256_cmp_pd(ay, py, _CMP_GT_OQ), _mm256_cmp_pd(by, py, _CMP_GT_OQ)); + const __m256d xi = _mm256_add_pd(_mm256_div_pd(_mm256_mul_pd(dx, pya), dy), ax); + par = _mm256_xor_pd(par, _mm256_and_pd(straddle, _mm256_cmp_pd(px, xi, _CMP_LT_OQ))); + const __m256d t = _mm256_div_pd(_mm256_add_pd(_mm256_mul_pd(pxa, dx), _mm256_mul_pd(pya, dy)), _mm256_load_pd(e.den + i)); + const __m256d gt1 = _mm256_cmp_pd(t, one, _CMP_GT_OQ); + const __m256d gt0 = _mm256_cmp_pd(t, zero, _CMP_GT_OQ); + __m256d x = _mm256_blendv_pd(ax, _mm256_add_pd(ax, _mm256_mul_pd(dx, t)), gt0); + __m256d y = _mm256_blendv_pd(ay, _mm256_add_pd(ay, _mm256_mul_pd(dy, t)), gt0); + x = _mm256_blendv_pd(x, bx, gt1); + y = _mm256_blendv_pd(y, by, gt1); + const __m256d ex = _mm256_sub_pd(px, x); + const __m256d ey = _mm256_sub_pd(py, y); + const __m256d dd = _mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)); + const __m256i lt = _mm256_castpd_si256(_mm256_cmp_pd(dd, mn, _CMP_LT_OQ)); + mn = _mm256_min_pd(dd, mn); + best = _mm256_blendv_epi8(best, _mm256_set_epi64x(i + 3, i + 2, i + 1, i), lt); + } + }; + auto hmin = [&]() __attribute__((target("avx2"), always_inline)) { + const __m128d m2 = _mm_min_pd(_mm256_castpd256_pd128(mn), _mm256_extractf128_pd(mn, 1)); + return _mm_min_sd(m2, _mm_unpackhi_pd(m2, m2)); + }; + + const std::size_t hb = hint / Edges::kBlock; + evalBlock(hb); + __m128d mnS = hmin(); + const __m256d m2 = _mm256_set1_pd(2 * e.margin); + const __m256d shrink = _mm256_set1_pd(1 - 1e-12); + // the nodes g .. g + 3 of a level that can hold a block to evaluate + auto need = [&](const Edges::Level& L, std::size_t g) __attribute__((target("avx2"), always_inline)) { + const __m256d x0 = _mm256_load_pd(L.xmin + g); + const __m256d x1 = _mm256_load_pd(L.xmax + g); + const __m256d y0 = _mm256_load_pd(L.ymin + g); + const __m256d y1 = _mm256_load_pd(L.ymax + g); + const __m256d needPar = _mm256_and_pd(_mm256_and_pd(_mm256_cmp_pd(py, y1, _CMP_LT_OQ), _mm256_cmp_pd(py, y0, _CMP_GE_OQ)), + _mm256_cmp_pd(px, _mm256_load_pd(L.xmaxm + g), _CMP_LE_OQ)); + const __m256d gx = _mm256_max_pd(zero, _mm256_max_pd(_mm256_sub_pd(_mm256_sub_pd(x0, px), m2), _mm256_sub_pd(_mm256_sub_pd(px, x1), m2))); + const __m256d gy = _mm256_max_pd(zero, _mm256_max_pd(_mm256_sub_pd(_mm256_sub_pd(y0, py), m2), _mm256_sub_pd(_mm256_sub_pd(py, y1), m2))); + const __m256d lb = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(gx, gx), _mm256_mul_pd(gy, gy)), shrink); + const __m256d needD = _mm256_cmp_pd(lb, _mm256_broadcastsd_pd(mnS), _CMP_LE_OQ); + return _mm256_movemask_pd(_mm256_or_pd(needPar, needD)); + }; + if (e.nlv == 0) { + const Edges::Level L0 { e.xmin, e.xmaxm, e.ymin, e.ymax, e.xmax, e.nbpad }; + for (std::size_t g = 0; g < e.nbpad; g += 4) { + int mask = need(L0, g); + while (mask) { + const std::size_t b = g + __builtin_ctz(mask); + mask &= mask - 1; + if (b == hb) continue; + evalBlock(b); + mnS = hmin(); + } + } + } else { + // depth first, children in index order + struct Item { int level; std::size_t g; }; + Item st[4 * Edges::kMaxLevels]; + int sp = 0; + st[sp++] = Item { e.nlv - 1, 0 }; + while (sp) { + const Item it = st[--sp]; + int mask = need(e.lv[it.level], it.g); + if (it.level == 0) { + while (mask) { + const std::size_t b = it.g + __builtin_ctz(mask); + mask &= mask - 1; + if (b == hb) continue; + evalBlock(b); + mnS = hmin(); + } + } else { + while (mask) { + const int c = 31 - __builtin_clz(mask); + mask &= ~(1 << c); + st[sp++] = Item { it.level - 1, 4 * (it.g + c) }; + } + } + } + } + + double m4[4]; + _mm256_storeu_pd(m4, mn); + std::int64_t b4[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(b4), best); + const int pm = _mm256_movemask_pd(par); + const bool inside = (__builtin_popcount(pm) & 1) != 0; + int lane = 0; + for (int l = 1; l < 4; l++) if (m4[l] < m4[lane]) lane = l; + k = static_cast(b4[lane]); + return (inside ? 1 : -1) * std::sqrt(m4[lane]); +} + +// Two points in lanes, edges in sequence (the same per-edge operations as above per lane). +inline void distTwoSse2(const double* pxs, const double* pys, const Edges& e, double* d, std::uint32_t* k) { + const __m128d px = _mm_loadu_pd(pxs); + const __m128d py = _mm_loadu_pd(pys); + const __m128d one = _mm_set1_pd(1.0); + const __m128d zero = _mm_setzero_pd(); + __m128d mn = _mm_set1_pd(std::numeric_limits::infinity()); + __m128d par = _mm_setzero_pd(); + __m128i best = _mm_setzero_si128(); + for (std::size_t i = 0; i < e.n; i++) { + const __m128d ax = _mm_set1_pd(e.ax[i]); + const __m128d ay = _mm_set1_pd(e.ay[i]); + const __m128d bx = _mm_set1_pd(e.bx[i]); + const __m128d by = _mm_set1_pd(e.by[i]); + const __m128d dx = _mm_set1_pd(e.dx[i]); + const __m128d dy = _mm_set1_pd(e.dy[i]); + const __m128d pxa = _mm_sub_pd(px, ax); + const __m128d pya = _mm_sub_pd(py, ay); + const __m128d straddle = _mm_xor_pd(_mm_cmpgt_pd(ay, py), _mm_cmpgt_pd(by, py)); + const __m128d xi = _mm_add_pd(_mm_div_pd(_mm_mul_pd(dx, pya), dy), ax); + par = _mm_xor_pd(par, _mm_and_pd(straddle, _mm_cmplt_pd(px, xi))); + const __m128d t = _mm_div_pd(_mm_add_pd(_mm_mul_pd(pxa, dx), _mm_mul_pd(pya, dy)), _mm_set1_pd(e.den[i])); + const __m128d gt1 = _mm_cmpgt_pd(t, one); + const __m128d gt0 = _mm_cmpgt_pd(t, zero); + __m128d x = _mm_or_pd(_mm_and_pd(gt0, _mm_add_pd(ax, _mm_mul_pd(dx, t))), _mm_andnot_pd(gt0, ax)); + __m128d y = _mm_or_pd(_mm_and_pd(gt0, _mm_add_pd(ay, _mm_mul_pd(dy, t))), _mm_andnot_pd(gt0, ay)); + x = _mm_or_pd(_mm_and_pd(gt1, bx), _mm_andnot_pd(gt1, x)); + y = _mm_or_pd(_mm_and_pd(gt1, by), _mm_andnot_pd(gt1, y)); + const __m128d ex = _mm_sub_pd(px, x); + const __m128d ey = _mm_sub_pd(py, y); + const __m128d dd = _mm_add_pd(_mm_mul_pd(ex, ex), _mm_mul_pd(ey, ey)); + const __m128i lt = _mm_castpd_si128(_mm_cmplt_pd(dd, mn)); + mn = _mm_min_pd(dd, mn); + const __m128i idx = _mm_set1_epi64x(static_cast(i)); + best = _mm_or_si128(_mm_and_si128(lt, idx), _mm_andnot_si128(lt, best)); + } + double m2[2]; + _mm_storeu_pd(m2, mn); + std::int64_t b2[2]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(b2), best); + const int pm = _mm_movemask_pd(par); + for (int l = 0; l < 2; l++) { + const bool inside = ((pm >> l) & 1) != 0; + d[l] = (inside ? 1 : -1) * std::sqrt(m2[l]); + k[l] = static_cast(b2[l]); + } +} + +// Four points in lanes, edges in sequence. Also max = d + hs per lane, and returns the lanes that +// are not no-ops (as the noOp test). TrackK: report a nearest edge per lane in k (else k is not +// written). +template +__attribute__((target("avx2"))) +inline int distFourAvx2T(const double* pxs, const double* pys, const Edges& e, double* d, std::uint32_t* k, + double hs, double bestD, double precision, double* mx) { + const __m256d px = _mm256_loadu_pd(pxs); + const __m256d py = _mm256_loadu_pd(pys); + const __m256d one = _mm256_set1_pd(1.0); + const __m256d zero = _mm256_setzero_pd(); + __m256d mn = _mm256_set1_pd(std::numeric_limits::infinity()); + __m256d par = _mm256_setzero_pd(); + __m256i best = _mm256_setzero_si256(); + for (std::size_t i = 0; i < e.nn; i++) { + const __m256d ax = _mm256_broadcast_sd(e.ax + i); + const __m256d ay = _mm256_broadcast_sd(e.ay + i); + const __m256d bx = _mm256_broadcast_sd(e.bx + i); + const __m256d by = _mm256_broadcast_sd(e.by + i); + const __m256d dx = _mm256_broadcast_sd(e.dx + i); + const __m256d dy = _mm256_broadcast_sd(e.dy + i); + const __m256d pxa = _mm256_sub_pd(px, ax); + const __m256d pya = _mm256_sub_pd(py, ay); + const __m256d straddle = _mm256_xor_pd(_mm256_cmp_pd(ay, py, _CMP_GT_OQ), _mm256_cmp_pd(by, py, _CMP_GT_OQ)); + const __m256d xi = _mm256_add_pd(_mm256_div_pd(_mm256_mul_pd(dx, pya), dy), ax); + par = _mm256_xor_pd(par, _mm256_and_pd(straddle, _mm256_cmp_pd(px, xi, _CMP_LT_OQ))); + const __m256d t = _mm256_div_pd(_mm256_add_pd(_mm256_mul_pd(pxa, dx), _mm256_mul_pd(pya, dy)), _mm256_broadcast_sd(e.den + i)); + const __m256d gt1 = _mm256_cmp_pd(t, one, _CMP_GT_OQ); + const __m256d gt0 = _mm256_cmp_pd(t, zero, _CMP_GT_OQ); + __m256d x = _mm256_blendv_pd(ax, _mm256_add_pd(ax, _mm256_mul_pd(dx, t)), gt0); + __m256d y = _mm256_blendv_pd(ay, _mm256_add_pd(ay, _mm256_mul_pd(dy, t)), gt0); + x = _mm256_blendv_pd(x, bx, gt1); + y = _mm256_blendv_pd(y, by, gt1); + const __m256d ex = _mm256_sub_pd(px, x); + const __m256d ey = _mm256_sub_pd(py, y); + const __m256d dd = _mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)); + if (TrackK) { + const __m256i lt = _mm256_castpd_si256(_mm256_cmp_pd(dd, mn, _CMP_LT_OQ)); + best = _mm256_blendv_epi8(best, _mm256_set1_epi64x(static_cast(i)), lt); + } + mn = _mm256_min_pd(dd, mn); + } + for (std::size_t i = e.nn; i < e.n; i++) { + const __m256d ex = _mm256_sub_pd(px, _mm256_broadcast_sd(e.ax + i)); + const __m256d ey = _mm256_sub_pd(py, _mm256_broadcast_sd(e.ay + i)); + const __m256d dd = _mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)); + if (TrackK) { + const __m256i lt = _mm256_castpd_si256(_mm256_cmp_pd(dd, mn, _CMP_LT_OQ)); + best = _mm256_blendv_epi8(best, _mm256_set1_epi64x(static_cast(i)), lt); + } + mn = _mm256_min_pd(dd, mn); + } + // (inside ? 1 : -1) * std::sqrt(minDistSq): VSQRTPD rounds as SQRTSD, and multiplying the + // non-NaN root by -1 is exactly a sign flip + const __m256d neg = _mm256_andnot_pd(par, _mm256_set1_pd(-0.0)); + const __m256d dv = _mm256_xor_pd(_mm256_sqrt_pd(mn), neg); + const __m256d mv = _mm256_add_pd(dv, _mm256_set1_pd(hs)); + _mm256_storeu_pd(d, dv); + _mm256_storeu_pd(mx, mv); + if (TrackK) { + std::int64_t b4[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(b4), best); + for (int l = 0; l < 4; l++) k[l] = static_cast(b4[l]); + } + const __m256d bv = _mm256_set1_pd(bestD); + const __m256d noop = _mm256_and_pd(_mm256_cmp_pd(dv, bv, _CMP_LE_OQ), _mm256_cmp_pd(_mm256_sub_pd(mv, bv), _mm256_set1_pd(precision), _CMP_LE_OQ)); + return ~_mm256_movemask_pd(noop) & 15; +} + +__attribute__((target("avx2"))) +inline int distFourAvx2(const double* pxs, const double* pys, const Edges& e, double* d, std::uint32_t* k, + double hs, double bestD, double precision, double* mx) { + return distFourAvx2T(pxs, pys, e, d, k, hs, bestD, precision, mx); +} + +inline bool useAvx2() { +#ifdef POLYLABEL_TEST_NO_AVX2 + return false; +#else + static const bool avx2 = __builtin_cpu_supports("avx2"); + return avx2; +#endif +} + +// distances of m <= 4 points (lanes m..3 of the arrays are scratch) +inline void distMany(double* pxs, double* pys, int m, const Edges& e, double* d, std::uint32_t* k) { + if (m == 1) { d[0] = useAvx2() ? distOneAvx2(pxs[0], pys[0], e, k[0]) : distOneSse2(pxs[0], pys[0], e, k[0]); return; } + for (int l = m; l < 4; l++) { pxs[l] = pxs[0]; pys[l] = pys[0]; } + if (useAvx2()) { double mx[4]; distFourAvx2(pxs, pys, e, d, k, 0, 0, 0, mx); return; } + distTwoSse2(pxs, pys, e, d, k); + if (m > 2) distTwoSse2(pxs + 2, pys + 2, e, d + 2, k + 2); +} + +// Upper bound on the distance at p from the edge k (as edgeDistBound). +inline double edgeBound(const Point& p, const Edges& e, std::uint32_t k) { + return std::sqrt(getSegDistSq(p, Point { e.ax[k], e.ay[k] }, Point { e.bx[k], e.by[k] })) * (1 + 1e-12) + 1e-11; +} + +// polygons with fewer edges evaluate every candidate cell (no edge bound first) +constexpr std::size_t kBoundMinEdges = Edges::kBlockMinEdges; + +// Signed distances of four points (lanes) to the polygon: distFourAvx2T without the nearest edge, +// in registers. The nearest point of the segment is computed as +// x = t > 1 ? b.x : a.x + dx * max(t, 0) (likewise y): for t > 0 that is getSegDistSq's +// a.x + dx * t; for t <= 0 or NaN (max returns its second operand 0 for NaN) it is a.x + dx * 0, +// which equals a.x unless a.x is a zero, where only the sign of the zero can differ; p.x - x then +// differs at most in the sign of a zero, and the squares and the distance are the same. +__attribute__((target("avx2"), always_inline)) +inline __m256d distFourSmall(const __m256d px, const __m256d py, const Edges& e) { + const __m256d one = _mm256_set1_pd(1.0); + const __m256d zero = _mm256_setzero_pd(); + __m256d mn = _mm256_set1_pd(std::numeric_limits::infinity()); + __m256d par = _mm256_setzero_pd(); + for (std::size_t i = 0; i < e.nn; i++) { + const __m256d ax = _mm256_broadcast_sd(e.ax + i); + const __m256d ay = _mm256_broadcast_sd(e.ay + i); + const __m256d bx = _mm256_broadcast_sd(e.bx + i); + const __m256d by = _mm256_broadcast_sd(e.by + i); + const __m256d dx = _mm256_broadcast_sd(e.dx + i); + const __m256d dy = _mm256_broadcast_sd(e.dy + i); + const __m256d pxa = _mm256_sub_pd(px, ax); + const __m256d pya = _mm256_sub_pd(py, ay); + const __m256d straddle = _mm256_xor_pd(_mm256_cmp_pd(ay, py, _CMP_GT_OQ), _mm256_cmp_pd(by, py, _CMP_GT_OQ)); + const __m256d xi = _mm256_add_pd(_mm256_div_pd(_mm256_mul_pd(dx, pya), dy), ax); + par = _mm256_xor_pd(par, _mm256_and_pd(straddle, _mm256_cmp_pd(px, xi, _CMP_LT_OQ))); + const __m256d t = _mm256_div_pd(_mm256_add_pd(_mm256_mul_pd(pxa, dx), _mm256_mul_pd(pya, dy)), _mm256_broadcast_sd(e.den + i)); + const __m256d gt1 = _mm256_cmp_pd(t, one, _CMP_GT_OQ); + const __m256d tc = _mm256_max_pd(t, zero); + const __m256d x = _mm256_blendv_pd(_mm256_add_pd(ax, _mm256_mul_pd(dx, tc)), bx, gt1); + const __m256d y = _mm256_blendv_pd(_mm256_add_pd(ay, _mm256_mul_pd(dy, tc)), by, gt1); + const __m256d ex = _mm256_sub_pd(px, x); + const __m256d ey = _mm256_sub_pd(py, y); + mn = _mm256_min_pd(_mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)), mn); + } + for (std::size_t i = e.nn; i < e.n; i++) { + const __m256d ex = _mm256_sub_pd(px, _mm256_broadcast_sd(e.ax + i)); + const __m256d ey = _mm256_sub_pd(py, _mm256_broadcast_sd(e.ay + i)); + mn = _mm256_min_pd(_mm256_add_pd(_mm256_mul_pd(ex, ex), _mm256_mul_pd(ey, ey)), mn); + } + return _mm256_xor_pd(_mm256_sqrt_pd(mn), _mm256_andnot_pd(par, _mm256_set1_pd(-0.0))); +} + +// Small polygons on AVX2 CPUs (fewer than kBoundMinEdges edges: every candidate cell is evaluated, +// four per kernel call): the decisions of polylabelFast with the queued cells in small arrays. +// Returns false, having decided nothing, if more than kSmallQueue cells would be queued (the +// caller then runs polylabelFast's general loop from the start; evaluation is pure). +constexpr int kSmallQueue = 64; + +__attribute__((target("avx2"))) +inline bool polylabelSmallAvx2(double precision, const Box& envelope, double cellSize, const Edges& e, + const Point& centroid, const Point& bboxCenter, Point& out) { + alignas(32) double qx[kSmallQueue], qy[kSmallQueue], qh[kSmallQueue], qd[kSmallQueue], qm[kSmallQueue]; + int qn = 0; + // lessCell on queue entries + auto less = [&](int a, int b) { + if (qm[a] != qm[b]) return qm[a] < qm[b]; + if (qd[a] != qd[b]) return qd[a] < qd[b]; + if (qh[a] != qh[b]) return qh[a] > qh[b]; + if (qx[a] != qx[b]) return qx[a] > qx[b]; + return qy[a] > qy[b]; + }; + double bestX = 0, bestY = 0, bestD = 0; + + // centroid, bbox center and the initial grid, four per kernel call (lanes 0, 1 of the first + // call are the two first guesses); the grid decisions only depend on bestD after the guesses + const double h0 = cellSize / 2; + const double hs0 = h0 * std::sqrt(2); + alignas(32) double gx[4], gy[4], gd[4]; + gx[0] = centroid.get<0>(); gy[0] = centroid.get<1>(); + gx[1] = bboxCenter.get<0>(); gy[1] = bboxCenter.get<1>(); + int gm = 2; + bool first = true; + auto flushGrid = [&]() __attribute__((target("avx2"), always_inline)) { + for (int l = gm; l < 4; l++) { gx[l] = gx[0]; gy[l] = gy[0]; } + _mm256_store_pd(gd, distFourSmall(_mm256_load_pd(gx), _mm256_load_pd(gy), e)); + int l = 0; + if (first) { + // bestCell = centroidCell; if (bboxCell.d > bestCell.d) bestCell = bboxCell + bestX = gx[0]; bestY = gy[0]; bestD = gd[0]; + if (gd[1] > bestD) { bestX = gx[1]; bestY = gy[1]; bestD = gd[1]; } + first = false; + l = 2; + } + for (; l < gm; l++) { + const double d = gd[l], m = d + hs0; + if (d <= bestD && m - bestD <= precision) continue; + if (qn == kSmallQueue) return false; + qx[qn] = gx[l]; qy[qn] = gy[l]; qh[qn] = h0; qd[qn] = d; qm[qn] = m; qn++; + } + gm = 0; + return true; + }; + for (double x = envelope.min_corner().get<0>(); x < envelope.max_corner().get<0>(); x += cellSize) { + for (double y = envelope.min_corner().get<1>(); y < envelope.max_corner().get<1>(); y += cellSize) { + gx[gm] = x + h0; gy[gm] = y + h0; + if (++gm == 4 && !flushGrid()) return false; + } + } + if ((gm || first) && !flushGrid()) return false; + + // children of a cell centred at c with half size 2h: c + (-h, -h), (h, -h), (-h, h), (h, h) + const __m256d sxv = _mm256_set_pd(1, -1, 1, -1); + const __m256d syv = _mm256_set_pd(1, 1, -1, -1); + const __m256d sqrt2v = _mm256_set1_pd(std::sqrt(2)); + while (qn) { + // pop the largest cell under lessCell: largest max, the full comparison only for ties + int bi = 0; + double bmax = qm[0]; + for (int i = 1; i < qn; i++) { + const bool gt = qm[i] > bmax; + bmax = gt ? qm[i] : bmax; + bi = gt ? i : bi; + } + int ties = 0; + for (int i = 0; i < qn; i++) ties += qm[i] == bmax; + if (ties > 1) { + for (int i = 0; i < qn; i++) if (qm[i] == bmax && less(bi, i)) bi = i; + } + const double cx = qx[bi], cy = qy[bi], ch = qh[bi], cd = qd[bi], cm = qm[bi]; + qn--; + qx[bi] = qx[qn]; qy[bi] = qy[qn]; qh[bi] = qh[qn]; qd[bi] = qd[qn]; qm[bi] = qm[qn]; + + if (cd > bestD) { bestX = cx; bestY = cy; bestD = cd; } + + if (cm - bestD <= precision) { + // no remaining cell is split: the first cell in pop order with the largest d > bestD + int cand = -1; + for (int i = 0; i < qn; i++) { + if (qd[i] > bestD && (cand < 0 || qd[i] > qd[cand] || (qd[i] == qd[cand] && less(cand, i)))) cand = i; + } + if (cand >= 0) { bestX = qx[cand]; bestY = qy[cand]; } + break; + } + + const double h = ch / 2; + const __m256d hv = _mm256_set1_pd(h); + const __m256d px = _mm256_add_pd(_mm256_set1_pd(cx), _mm256_mul_pd(hv, sxv)); + const __m256d py = _mm256_add_pd(_mm256_set1_pd(cy), _mm256_mul_pd(hv, syv)); + const __m256d dv = distFourSmall(px, py, e); + const __m256d mv = _mm256_add_pd(dv, _mm256_mul_pd(hv, sqrt2v)); + const __m256d bv = _mm256_set1_pd(bestD); + const __m256d noop = _mm256_and_pd(_mm256_cmp_pd(dv, bv, _CMP_LE_OQ), _mm256_cmp_pd(_mm256_sub_pd(mv, bv), _mm256_set1_pd(precision), _CMP_LE_OQ)); + int keep = ~_mm256_movemask_pd(noop) & 15; + if (!keep) continue; + if (qn + __builtin_popcount(keep) > kSmallQueue) return false; + alignas(32) double ax[4], ay[4], ad[4], am[4]; + _mm256_store_pd(ax, px); _mm256_store_pd(ay, py); _mm256_store_pd(ad, dv); _mm256_store_pd(am, mv); + do { + const int l = __builtin_ctz(keep); + keep &= keep - 1; + qx[qn] = ax[l]; qy[qn] = ay[l]; qh[qn] = h; qd[qn] = ad[l]; qm[qn] = am[l]; qn++; + } while (keep); + } + out = Point { bestX, bestY }; + return true; +} + +inline Point polylabelFast(const Polygon& polygon, double precision, const Box& envelope, const Point& size, double cellSize, Edges& e) { + const double sqrt2 = std::sqrt(2); + const bool bound = e.n >= kBoundMinEdges; + const bool blocked = e.nb > 0 && useAvx2(); + + auto makeCell = [&](const Point& c, double h, double d, std::uint32_t k) { + return FCell { c, h, d, d + h * std::sqrt(2), k }; + }; + auto evalOne = [&](const Point& c, double h) { + std::uint32_t k; + const double d = blocked ? distBlockedAvx2(c.get<0>(), c.get<1>(), e, 0, k) + : useAvx2() ? distOneAvx2(c.get<0>(), c.get<1>(), e, k) : distOneSse2(c.get<0>(), c.get<1>(), e, k); + return makeCell(c, h, d, k); + }; + + // centroid (as getCentroidCell) + Point centroid; + { + double area = 0; + double cx = 0, cy = 0; + const auto& ring = polygon.outer(); + const Point& origin = ring.at(0); + const double ox = origin.get<0>(); + const double oy = origin.get<1>(); + for (std::size_t i = 0, len = ring.size(), j = len - 1; i < len; j = i++) { + const Point& a = ring[i]; + const Point& b = ring[j]; + const double ax = a.get<0>() - ox; + const double ay = a.get<1>() - oy; + const double bx = b.get<0>() - ox; + const double by = b.get<1>() - oy; + auto f = ax * by - bx * ay; + cx += (ax + bx) * f; + cy += (ay + by) * f; + area += f * 3; + } + centroid = area == 0 ? ring.at(0) : Point { ox + cx / area, oy + cy / area }; + } + const Point bboxCenter { + envelope.min_corner().get<0>() + size.get<0>() / 2.0, + envelope.min_corner().get<1>() + size.get<1>() / 2.0 + }; + double h = cellSize / 2; + + // Small polygons: the two first guesses and the initial grid cells are evaluated together, four + // per kernel call (evaluation is pure; the decisions below follow in the original order). + const bool combined = !bound && useAvx2(); + thread_local std::vector gridStore; + thread_local std::vector gridD; + thread_local std::vector gridK; + std::vector& grid = gridStore; + FCell centroidCell, bboxCell; + if (combined) { + grid.clear(); + grid.push_back(centroid); + grid.push_back(bboxCenter); + for (double x = envelope.min_corner().get<0>(); x < envelope.max_corner().get<0>(); x += cellSize) { + for (double y = envelope.min_corner().get<1>(); y < envelope.max_corner().get<1>(); y += cellSize) { + grid.push_back(Point {x + h, y + h}); + } + } + const std::size_t ng = grid.size(); + gridD.resize(ng + 4); + gridK.resize(ng + 4); + for (std::size_t i = 0; i < ng; i += 4) { + alignas(32) double gx[4], gy[4], mx[4]; + for (std::size_t l = 0; l < 4; l++) { + const Point& q = grid[i + l < ng ? i + l : i]; + gx[l] = q.get<0>(); + gy[l] = q.get<1>(); + } + distFourAvx2(gx, gy, e, &gridD[i], &gridK[i], 0, 0, 0, mx); + } + centroidCell = makeCell(grid[0], 0, gridD[0], gridK[0]); + bboxCell = makeCell(grid[1], 0, gridD[1], gridK[1]); + } else { + centroidCell = evalOne(centroid, 0); + bboxCell = evalOne(bboxCenter, 0); + } + FCell bestCell = centroidCell; + if (bboxCell.d > bestCell.d) bestCell = bboxCell; + + thread_local Heap heapStore; + Heap& queue = heapStore; + queue.clear(); + + // no-op when popped: bestCell.d only grows (same test as provablyNoOp, on exact or bounded values) + auto noOp = [&](double d, double max) { + return d <= bestCell.d && max - bestCell.d <= precision; + }; + auto boundNoOp = [&](double ub, double hc) { + return ub <= bestCell.d && (ub + hc * sqrt2) - bestCell.d <= precision; + }; + + alignas(32) double bxs[4], bys[4], bd[4]; + std::uint32_t bk[4] = {}; // the small path does not track nearest edges (unused there) + int bm = 0; + double bh = 0; + std::uint32_t bhint = 0; + auto flush = [&]() { + if (!bm) return; + if (!blocked && bm > 1 && useAvx2()) { + for (int l = bm; l < 4; l++) { bxs[l] = bxs[0]; bys[l] = bys[0]; } + alignas(32) double bmx[4]; + // k is only needed by the edge bound, which this path does not use + int keep = distFourAvx2T(bxs, bys, e, bd, bk, bh * std::sqrt(2), bestCell.d, precision, bmx) & ((1 << bm) - 1); + while (keep) { + const int l = __builtin_ctz(keep); + keep &= keep - 1; + queue.push(FCell { Point { bxs[l], bys[l] }, bh, bd[l], bmx[l], bk[l] }); + } + bm = 0; + return; + } + if (blocked) { + for (int l = 0; l < bm; l++) bd[l] = distBlockedAvx2(bxs[l], bys[l], e, bhint, bk[l]); + } else { + distMany(bxs, bys, bm, e, bd, bk); + } + for (int l = 0; l < bm; l++) { + const FCell c = makeCell(Point { bxs[l], bys[l] }, bh, bd[l], bk[l]); + if (noOp(c.d, c.max)) continue; + queue.push(c); + } + bm = 0; + }; + auto add = [&](double x, double y, double hc) { + bxs[bm] = x; bys[bm] = y; bh = hc; bm++; + if (bm == 4) flush(); + }; + + // cover polygon with initial cells + bhint = bboxCell.k; + if (combined) { + for (std::size_t i = 2, ng = grid.size(); i < ng; i++) { + const FCell c = makeCell(grid[i], h, gridD[i], gridK[i]); + if (!noOp(c.d, c.max)) queue.push(c); + } + if (grid.capacity() > (1u << 12)) { + std::vector().swap(grid); + std::vector().swap(gridD); + std::vector().swap(gridK); + } + } else for (double x = envelope.min_corner().get<0>(); x < envelope.max_corner().get<0>(); x += cellSize) { + for (double y = envelope.min_corner().get<1>(); y < envelope.max_corner().get<1>(); y += cellSize) { + const Point p {x + h, y + h}; + if (bound) { + const double ub = std::min(edgeBound(p, e, bboxCell.k), edgeBound(p, e, centroidCell.k)); + if (boundNoOp(ub, h)) continue; + } + add(p.get<0>(), p.get<1>(), h); + } + } + flush(); + + while (!queue.empty()) { + const FCell cell = queue.pop(); + + if (cell.d > bestCell.d) bestCell = cell; + + if (cell.max - bestCell.d <= precision) { + // no remaining cell is split: the result is the first cell in pop order with the + // largest d > bestCell.d + const FCell* cand = nullptr; + queue.forEach([&](const FCell& c) { + if (c.d > bestCell.d && (!cand || c.d > cand->d || (c.d == cand->d && lessCell(*cand, c)))) cand = &c; + }); + if (cand) bestCell = *cand; + break; + } + + h = cell.h / 2; + bhint = cell.k; + const Point children[4] = { + {cell.c.get<0>() - h, cell.c.get<1>() - h}, + {cell.c.get<0>() + h, cell.c.get<1>() - h}, + {cell.c.get<0>() - h, cell.c.get<1>() + h}, + {cell.c.get<0>() + h, cell.c.get<1>() + h} + }; + for (const Point& p : children) { + if (bound && boundNoOp(edgeBound(p, e, cell.k), h)) continue; + add(p.get<0>(), p.get<1>(), h); + } + flush(); + } + + queue.release(); + return bestCell.c; +} + +// polylabel() for polygons with fewer than kBoundMinEdges edges on AVX2 CPUs, in one pass over +// the rings: the envelope of the outer ring (as geom::envelope: the first point, then strict < and +// > per coordinate), the early return for cellSize == 0, the edge arrays of Edges::build (same +// entries; the order of the edges changes neither the minimum nor the parity) on the stack, the +// centroid (as getCentroidCell, same operations in the same order), then polylabelSmallAvx2. +// Returns false, having decided nothing, if the polygon is not small, the cell size is not finite, +// a coordinate is not finite or of magnitude >= 1e100, or the small path declines. +__attribute__((target("avx2"))) +inline bool polylabelSmall(const Polygon& polygon, double precision, Point& out) { + const auto& outer = polygon.outer(); + std::size_t n = outer.size(); + for (const auto& inner : polygon.inners()) n += inner.size(); + if (n >= kBoundMinEdges || outer.empty()) return false; + alignas(32) double buf[7][kBoundMinEdges]; + Edges e; + e.ax = buf[0]; e.ay = buf[1]; e.bx = buf[2]; e.by = buf[3]; e.dx = buf[4]; e.dy = buf[5]; e.den = buf[6]; + e.n = n; + // non-degenerate edges from the front, zero-length ones from the back + std::size_t w = 0, z = n; + bool ok = true; + auto edge = [&](double x, double y, double bx, double by) { + ok &= std::fabs(x) < 1e100 && std::fabs(y) < 1e100; + const double dx = bx - x, dy = by - y; + const std::size_t k = dx == 0 && dy == 0 ? --z : w++; + e.ax[k] = x; e.ay[k] = y; e.bx[k] = bx; e.by[k] = by; e.dx[k] = dx; e.dy[k] = dy; + e.den[k] = dx * dx + dy * dy; + }; + const double ox = outer[0].get<0>(); + const double oy = outer[0].get<1>(); + double minX = ox, minY = oy, maxX = ox, maxY = oy; + double area = 0, cx = 0, cy = 0; + for (std::size_t i = 0, len = outer.size(), j = len - 1; i < len; j = i++) { + const double x = outer[i].get<0>(), y = outer[i].get<1>(); + const double bx = outer[j].get<0>(), by = outer[j].get<1>(); + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + edge(x, y, bx, by); + const double ax = x - ox; + const double ay = y - oy; + const double bxo = bx - ox; + const double byo = by - oy; + auto f = ax * byo - bxo * ay; + cx += (ax + bxo) * f; + cy += (ay + byo) * f; + area += f * 3; + } + const Box envelope { Point { minX, minY }, Point { maxX, maxY } }; + const Point size { maxX - minX, maxY - minY }; + const double cellSize = std::min(size.get<0>(), size.get<1>()); + if (cellSize == 0) { + out = envelope.min_corner(); + return true; + } + if (!std::isfinite(cellSize)) return false; + for (const auto& inner : polygon.inners()) { + for (std::size_t i = 0, len = inner.size(), j = len - 1; i < len; j = i++) { + edge(inner[i].get<0>(), inner[i].get<1>(), inner[j].get<0>(), inner[j].get<1>()); + } + } + if (!ok) return false; + e.nn = w; + const Point centroid = area == 0 ? outer[0] : Point { ox + cx / area, oy + cy / area }; + const Point bboxCenter { + envelope.min_corner().get<0>() + size.get<0>() / 2.0, + envelope.min_corner().get<1>() + size.get<1>() / 2.0 + }; + return polylabelSmallAvx2(precision, envelope, cellSize, e, centroid, bboxCenter, out); +} + +} // namespace fast +#endif + } // namespace detail Point polylabel(const Polygon& polygon, double precision = 0.00001, bool debug = false) { using namespace detail; +#ifdef POLYLABEL_SSE2 + // small polygons on AVX2 CPUs: everything below in one pass, same result + if (!debug && fast::useAvx2()) { + Point r; + if (fast::polylabelSmall(polygon, precision, r)) return r; + } +#endif + // find the bounding box of the outer ring Box envelope; geom::envelope(polygon.outer(), envelope); @@ -370,6 +1532,19 @@ Point polylabel(const Polygon& polygon, double precision = 0.00001, bool debug = return envelope.min_corner(); } +#ifdef POLYLABEL_SSE2 + // all real input (finite coordinates of magnitude < 1e100): the fast path, same result + if (!debug && std::isfinite(cellSize)) { + thread_local fast::Edges fastEdges; + if (fastEdges.build(polygon)) { + struct Release { + ~Release() { fastEdges.release(); } + } release; + return fast::polylabelFast(polygon, precision, envelope, size, cellSize, fastEdges); + } + } +#endif + #ifdef POLYLABEL_SSE2 // flatten the polygon's edges once for all distance evaluations of this call thread_local PolygonEdges edges; diff --git a/include/ring_point_index.h b/include/ring_point_index.h new file mode 100644 index 00000000..04d165c4 --- /dev/null +++ b/include/ring_point_index.h @@ -0,0 +1,291 @@ +/*! \file */ +#ifndef _RING_POINT_INDEX_H +#define _RING_POINT_INDEX_H + +// Faster point-in-ring for Boost.Geometry on big rings (included from geom.h). +// +// Boost tests whether a point is in a ring by walking every segment of the ring +// with the cartesian winding strategy. Overlay (difference etc.) and is_valid do +// that once per hole against the outer ring, which is quadratic for a lake with +// 14000 islands and a 100000-point shore line. +// +// The winding strategy only reacts to a segment if the point's x is within the +// segment's x range (math::equals with Boost's epsilon at the ends); every other +// segment leaves the state untouched. So the same answer comes from walking only +// the segments whose x range, widened by far more than that epsilon, holds the +// point's x. The order of the walk doesn't matter: the count is a sum, and a +// point on a segment ends the walk with "on boundary" whenever it is found. +// +// The index is only used inside a RingPointIndexScope, around Boost calls whose +// rings don't change while they run (is_valid, and the ring selection of one +// overlay call). The index refers to a ring by its address; it is dropped when +// the scope ends. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ring_point_index { + +template +class Index { +public: + explicit Index(RingT const &ring) : ring(ring), size(ring.size()), data(ring.empty() ? nullptr : &ring[0]) { + usable = false; + if (size < 4 || size > std::numeric_limits::max()) return; + double minX = std::numeric_limits::max(), maxX = std::numeric_limits::lowest(); + for (auto const &p : ring) { + double const x = boost::geometry::get<0>(p); + if (!std::isfinite(x)) return; + minX = std::min(minX, x); maxX = std::max(maxX, x); + } + double const margin = 1e-9 * std::max({ 1.0, std::abs(minX), std::abs(maxX) }); + x0 = minX - margin; + std::size_t const segments = size - 1; + bands = std::max(1, std::min(segments / 4, 1 << 20)); + double const width = (maxX + margin) - x0; + if (!(width > 0) || !std::isfinite(width)) return; + scale = bands / width; + if (!std::isfinite(scale)) return; + + std::vector counts(bands + 1, 0); + auto range = [&](std::size_t i, std::size_t &lo, std::size_t &hi) { + double const a = boost::geometry::get<0>(ring[i]), b = boost::geometry::get<0>(ring[i + 1]); + lo = band(std::min(a, b) - margin); + hi = band(std::max(a, b) + margin); + }; + std::size_t lo, hi; + for (std::size_t i = 0; i < segments; i++) { + range(i, lo, hi); + if (hi - lo > 64) { wide.push_back(i); continue; } + for (std::size_t k = lo; k <= hi; k++) counts[k + 1]++; + } + for (std::size_t k = 0; k < bands; k++) counts[k + 1] += counts[k]; + start = counts; + entries.resize(counts[bands]); + for (std::size_t i = 0; i < segments; i++) { + range(i, lo, hi); + if (hi - lo > 64) continue; + for (std::size_t k = lo; k <= hi; k++) entries[counts[k]++] = i; + } + usable = true; + } + + bool matches(RingT const &r) const { + return r.size() == size && (size == 0 || &r[0] == data); + } + + // The same as boost's point_in_range over the whole ring with this winding strategy, + // or false if the index can't answer (then the caller walks the ring) + template + bool apply(PointT const &point, Strategy const &strategy, int &result) const { + if (!usable) return false; + double const px = boost::geometry::get<0>(point); + if (!std::isfinite(px)) return false; + typename Strategy::state_type state; + bool go = true; + std::size_t const k = band(px); + for (uint32_t e = start[k], end = start[k + 1]; go && e < end; e++) { + uint32_t const i = entries[e]; + go = strategy.apply(point, ring[i], ring[i + 1], state); + } + for (std::size_t w = 0; go && w < wide.size(); w++) { + uint32_t const i = wide[w]; + go = strategy.apply(point, ring[i], ring[i + 1], state); + } + result = strategy.result(state); + return true; + } + +private: + // Non-decreasing in x, so lo <= x <= hi gives band(lo) <= band(x) <= band(hi) + std::size_t band(double x) const { + double const f = std::floor((x - x0) * scale); + if (!(f > 0)) return 0; + if (f >= double(bands - 1)) return bands - 1; + return std::size_t(f); + } + + RingT const ˚ + std::size_t size; + void const *data; + bool usable; + double x0 = 0, scale = 0; + std::size_t bands = 1; + std::vector start, entries, wide; +}; + +// Rings with fewer points are walked as before +constexpr std::size_t kMinRingSize = 1024; +// A ring is indexed when it is tested for the third time in a scope +constexpr unsigned kQueriesBeforeIndex = 2; + +template +struct Cache { + struct Entry { unsigned queries = 0; std::unique_ptr> index; }; + unsigned depth = 0; + std::unordered_map entries; +}; + +template +inline Cache &cache() { + static thread_local Cache c; + return c; +} + +} // namespace ring_point_index + +// Point-in-ring tests on Ring use the index while one of these exists on this thread +class RingPointIndexScope { +public: + RingPointIndexScope() { ring_point_index::cache().depth++; } + ~RingPointIndexScope() { + auto &c = ring_point_index::cache(); + if (--c.depth == 0) c.entries.clear(); + } + RingPointIndexScope(RingPointIndexScope const &) = delete; + RingPointIndexScope &operator=(RingPointIndexScope const &) = delete; +}; + +namespace ring_point_index { + +// Boost >= 1.77 or so passes an umbrella strategy to point_in_geometry and takes the +// point/ring strategy from strategy.relate(point, ring); older Boost (1.74) passes +// the point-in-ring strategy itself +template +struct HasRelate : std::false_type {}; +template +struct HasRelate().relate(std::declval

(), std::declval()))>> + : std::true_type {}; + +template +inline auto pointInRingStrategy(S const &strategy, P const &point, R const &ring) { + if constexpr (HasRelate::value) return strategy.relate(point, ring); + else return strategy; +} + +// The index reproduces cartesian_winding with the default calculation type only +// (its point type arguments are unused, kept by Boost for backward compatibility) +template struct IsCartesianWinding : std::false_type {}; +template +struct IsCartesianWinding> : std::true_type {}; + +} // namespace ring_point_index + +namespace boost { namespace geometry { namespace detail_dispatch { namespace within { + +// Same as boost's point_in_geometry, but the walk over the ring's +// segments may use the index (see above). Ring is closed and clockwise, so the +// normalized (closed, clockwise) view boost walks is the ring itself. +template <> +struct point_in_geometry +{ + static_assert(geometry::closure::value == closed && geometry::point_order::value == clockwise, + "the ring is walked as it is"); + + template + static inline int apply(Point_ const& point, Ring const& ring, Strategy const& strategy) + { + if ( boost::size(ring) < core_detail::closure::minimum_ring_size + < + geometry::closure::value + >::value ) + { + return -1; + } + + auto const pir = ring_point_index::pointInRingStrategy(strategy, point, ring); + using pir_t = std::decay_t; + if constexpr (ring_point_index::IsCartesianWinding::value) { + if (ring.size() >= ring_point_index::kMinRingSize) { + auto &c = ring_point_index::cache(); + if (c.depth > 0) { + auto &entry = c.entries[&ring]; + if (entry.index && !entry.index->matches(ring)) { entry.index.reset(); entry.queries = 0; } + if (!entry.index && ++entry.queries > ring_point_index::kQueriesBeforeIndex) + entry.index = std::make_unique>(ring); + int result; + if (entry.index && entry.index->apply(point, pir, result)) return result; + } + } + } + + return detail::within::point_in_range(point, ring, pir); + } +}; + +}}}} // namespace boost::geometry::detail_dispatch::within + +// is_valid tests the holes of a polygon for overlaps with boost's partition, which +// hands its visitor every pair of holes in a cell, also pairs whose boxes are far +// apart. The visitor asks relate() whether their interiors or boundaries meet; for +// a big hole (an island with 15000 points) relate sectionalizes it again for each +// of hundreds of small holes. Two rings whose boxes are clearly apart can't meet: +// relate finds no turns (section boxes only grow by epsilon) and the point-in-ring +// tests put each ring outside the other, so none of the asked-for cells is set and +// the answer is false. Answer that without the work. +namespace ring_point_index { + +inline bool boxesClearlyApart(Ring const &a, Ring const &b) { + if (a.empty() || b.empty()) return false; + auto ba = boost::geometry::return_envelope(a), bb = boost::geometry::return_envelope(b); + double const c[8] = { ba.min_corner().x(), ba.min_corner().y(), ba.max_corner().x(), ba.max_corner().y(), + bb.min_corner().x(), bb.min_corner().y(), bb.max_corner().x(), bb.max_corner().y() }; + double m = 1; + for (double v : c) { if (!std::isfinite(v)) return false; m = std::max(m, std::abs(v)); } + double const gap = 1e-9 * m; + return ba.max_corner().x() + gap < bb.min_corner().x() || bb.max_corner().x() + gap < ba.min_corner().x() + || ba.max_corner().y() + gap < bb.min_corner().y() || bb.max_corner().y() + gap < ba.min_corner().y(); +} + +} // namespace ring_point_index + +// util::type_sequence doesn't exist in Boost 1.74; declaring it is harmless there +// (it is never used then) and a redeclaration where boost defines it +namespace boost { namespace geometry { namespace util { template struct type_sequence; } } } +namespace ring_point_index { + +// The handler of is_valid_polygon's nested-holes test: true if II, IB or BI is set. +// Boost 1.74 lists the masks in an mpl::vector, newer Boost in a util::type_sequence. +template class Seq> +using HolesOverlapHandlerT = boost::geometry::detail::relate::static_mask_handler< + Seq, + boost::geometry::de9im::static_mask<'*', 'T'>, + boost::geometry::de9im::static_mask<'*', '*', '*', 'T'>>, + true>; + +template +constexpr bool isHolesOverlapHandler() { + return std::is_same>::value + || std::is_same>::value; +} + +} // namespace ring_point_index + +namespace boost { namespace geometry { namespace dispatch { + +template <> +struct relate + : detail::relate::areal_areal +{ + template + static inline void apply(Ring const &ring1, Ring const &ring2, Result &result, Strategy const &strategy) + { + // The handler starts with every cell 'F': result() is false, as relate would give + if (ring_point_index::isHolesOverlapHandler() + && ring_point_index::boxesClearlyApart(ring1, ring2)) + return; + detail::relate::areal_areal::apply(ring1, ring2, result, strategy); + } +}; + +}}} // namespace boost::geometry::dispatch + +#endif diff --git a/include/shared_data.h b/include/shared_data.h index 9f1e0dd9..3bb9c4b1 100644 --- a/include/shared_data.h +++ b/include/shared_data.h @@ -94,6 +94,7 @@ class Config { uint baseZoom, startZoom, endZoom; TileFormat tileFormat; uint mvtVersion, combineBelow; + int compressionLevel; // libdeflate level for tiles (1..12) bool includeID, compress, gzip, highResolution; std::string compressOpt; bool clippingBoxFromJSON; diff --git a/include/shp_mem_tiles.h b/include/shp_mem_tiles.h index a5cf4f3f..0bab561c 100644 --- a/include/shp_mem_tiles.h +++ b/include/shp_mem_tiles.h @@ -27,7 +27,8 @@ class ShpMemTiles : public TileDataSource const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid = false // polygon: caller has checked it is valid ); std::vector QueryMatchingGeometries( diff --git a/include/sorted_way_store.h b/include/sorted_way_store.h index 2df99249..ec8ba7d3 100644 --- a/include/sorted_way_store.h +++ b/include/sorted_way_store.h @@ -118,8 +118,9 @@ class SortedWayStore: public WayStore { std::vector> allocatedMemory; // The orphanage stores nodes that come from groups that may be worked on by - // multiple threads. They'll get folded into the index during finalize() - std::map>>> orphanage; + // multiple threads. They'll get folded into the index during finalize(). + // Per group, the pieces handed in (each one thread's ways of that group, moved in whole). + std::map>>>> orphanage; std::vector>>> workerBuffers; std::atomic totalWays; @@ -128,7 +129,8 @@ class SortedWayStore: public WayStore { std::atomic totalGroupSpace; std::atomic totalChunks; - void collectOrphans(const std::vector>>& orphans); + // Moves the ways (all of one group) into the orphanage; leaves orphans empty + void collectOrphans(std::vector>>& orphans); void publishGroup(const std::vector>>& ways); }; diff --git a/include/stable_hash.h b/include/stable_hash.h new file mode 100644 index 00000000..e1af029f --- /dev/null +++ b/include/stable_hash.h @@ -0,0 +1,42 @@ +/*! \file */ +#ifndef _STABLE_HASH_H +#define _STABLE_HASH_H + +#include +#include +#include + +// Hash functions that give the same result on every platform, compiler and +// library version (unlike std::hash or boost::hash), for orderings that the +// output depends on. + +inline uint64_t stableMix(uint64_t x) { + // splitmix64 finaliser + x += 0x9e3779b97f4a7c15ull; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebull; + return x ^ (x >> 31); +} + +inline uint64_t stableCombine(uint64_t seed, uint64_t value) { + return stableMix(seed ^ stableMix(value)); +} + +inline uint64_t stableCombine(uint64_t seed, double value) { + if (value == 0) value = 0; // -0.0 and 0.0 compare equal, so hash them equally + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return stableCombine(seed, bits); +} + +inline uint64_t stableCombine(uint64_t seed, const char* data, std::size_t length) { + // FNV-1a over the bytes + uint64_t h = 0xcbf29ce484222325ull; + for (std::size_t i = 0; i < length; i++) { + h ^= static_cast(data[i]); + h *= 0x100000001b3ull; + } + return stableCombine(stableCombine(seed, static_cast(length)), h); +} + +#endif //_STABLE_HASH_H diff --git a/include/tag_map.h b/include/tag_map.h index 4f7df1d6..92d9a5d5 100644 --- a/include/tag_map.h +++ b/include/tag_map.h @@ -24,6 +24,9 @@ // This is true since the strings are owned by the protobuf block reader // 3. Max number of tag values will fit in a short // OSM limit is 5,000 tags per object +// +// The hashed form is built lazily, on the first lookup: most ways are answered by the +// way_function memo from the tags as added (see added()) and never need it. struct Tag { protozero::data_view key; protozero::data_view value; @@ -34,8 +37,14 @@ class TagMap { TagMap(); void reset(); - bool empty() const; - void addTag(const protozero::data_view& key, const protozero::data_view& value); + bool empty() const { return tags.empty(); } + void addTag(const protozero::data_view& key, const protozero::data_view& value) { + tags.push_back(Tag{key, value}); + if (built) insert(key, value); + } + + // The tags in the order they were added (a key added twice: the last value counts) + const std::vector& added() const { return tags; } // Return -1 if key not found, else return its keyLoc. int64_t getKey(const char* key, size_t size) const; @@ -71,17 +80,25 @@ class TagMap { const char* data; uint32_t size; uint32_t valueLoc; + uint64_t prefix; // the first (up to) 8 bytes, see keyPrefix }; - uint32_t ensureKey(const protozero::data_view& key); - uint32_t ensureValue(const protozero::data_view& value); + uint32_t ensureKey(const protozero::data_view& key) const; + uint32_t ensureValue(const protozero::data_view& value) const; + void insert(const protozero::data_view& key, const protozero::data_view& value) const; + void ensureBuilt() const { if (!built) build(); } + void build() const; + + std::vector tags; - std::array, SHARDS> keys; - std::array, SHARDS> values; + // The hashed form of `tags`, built on demand + mutable bool built = false; + mutable std::array, SHARDS> keys; + mutable std::array, SHARDS> values; // Bit i set => keys[i] (resp. values[i]) may be non-empty. - uint16_t usedKeyShards = 0; - uint16_t usedValueShards = 0; + mutable uint16_t usedKeyShards = 0; + mutable uint16_t usedValueShards = 0; }; #endif //_TAG_MAP_H diff --git a/include/tile_coordinates_set.h b/include/tile_coordinates_set.h index 3d45c50e..37ad06e1 100644 --- a/include/tile_coordinates_set.h +++ b/include/tile_coordinates_set.h @@ -6,6 +6,24 @@ #include #include #include "coordinates.h" +#ifdef _MSC_VER +#include +#endif + +// The index of the lowest set bit of v (v != 0) +inline unsigned lowestSetBit(uint64_t v) { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(v); +#elif defined(_MSC_VER) && defined(_M_X64) + unsigned long i; + _BitScanForward64(&i, v); + return i; +#else + unsigned n = 0; + while (!(v & 1)) { v >>= 1; n++; } + return n; +#endif +} // Interface representing a bitmap of tiles of interest at a given zoom. class TileCoordinatesSet { @@ -14,6 +32,8 @@ class TileCoordinatesSet { virtual void set(TileCoordinate x, TileCoordinate y) = 0; virtual size_t size() const = 0; virtual size_t zoom() const = 0; + // Set every tile with minX <= x <= maxX and minY <= y <= maxY. + virtual void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY); // Visit every set tile in x-major, y-minor order. virtual void forEachSet(const std::function& f) const; }; @@ -30,7 +50,28 @@ class PreciseTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) override; void forEachSet(const std::function& f) const override; + // Visit the set tiles with xBegin <= x < xEnd, x-major, y-minor + template void forEachSetInColumns(TileCoordinate xBegin, TileCoordinate xEnd, F f) const { + const uint64_t first = (uint64_t)xBegin << zoom_, last = (uint64_t)xEnd << zoom_; // bits [first, last) + for (uint64_t w = first >> 6; w < numWords && (w << 6) < last; w++) { + if (!((summary[w >> 12] >> ((w >> 6) & 63)) & 1)) { w |= 63; continue; } // skip the empty block of 64 words + uint64_t word = tiles[w]; + while (word) { + const uint64_t loc = (w << 6) + lowestSetBit(word); + word &= word - 1; + if (loc < first || loc >= last) continue; + f((TileCoordinate)(loc >> zoom_), (TileCoordinate)(loc & ((1ull << zoom_) - 1))); + } + } + } + // Add the tiles set in other, a set of the same zoom. + void merge(const PreciseTileCoordinatesSet& other); + // The same for the words under summary words [sBegin, sEnd) only: merges of disjoint ranges + // can run at the same time. + void mergeRange(const PreciseTileCoordinatesSet& other, size_t sBegin, size_t sEnd); + size_t summaryWords() const { return summary.size(); } private: unsigned int zoom_; @@ -50,6 +91,7 @@ class LossyTileCoordinatesSet : public TileCoordinatesSet { size_t size() const override; size_t zoom() const override; void set(TileCoordinate x, TileCoordinate y) override; + void forEachSet(const std::function& f) const override; private: unsigned int zoom_; diff --git a/include/tile_data.h b/include/tile_data.h index 3e8d20e9..14108fd2 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -2,10 +2,20 @@ #ifndef _TILE_DATA_H #define _TILE_DATA_H +#include +#include #include +#include #include #include +#include +#include #include +#include +#include +#include +#include +#include #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" @@ -14,15 +24,28 @@ typedef std::vector SourceList; +// The TileDataSources alive, for the store leases a thread keeps until it ends (leased_store.h): the +// main thread gives its leases back at exit, after the sources are gone. A source is also told apart +// by its number, as a new source can be at the address of a destroyed one. +uint64_t registerLeaseSource(const TileDataSource *source); +void unregisterLeaseSource(const TileDataSource *source); +std::mutex &leaseSourcesMutex(); +bool leaseSourceAlive(const TileDataSource *source, uint64_t number); // with leaseSourcesMutex() held + class TileBbox; +// lowZoom (if given): gets the objects with minZoom < CLUSTER_ZOOM, in their order before the sort template void sortOutputObjects( const unsigned int indexZoom, const size_t threadNum, typename AppendVectorNS::AppendVector::Iterator begin, - typename AppendVectorNS::AppendVector::Iterator end + typename AppendVectorNS::AppendVector::Iterator end, + std::vector* lowZoom = nullptr ); +// Frees the calling thread's buffers of sortOutputObjects (threads that end free theirs) +void releaseSortBuffers(); + template void finalizeObjects( const std::string& name, const size_t& threadNum, @@ -31,77 +54,145 @@ template void finalizeObjects( typename std::vector>::iterator end, typename std::vector>& lowZoom ) { - size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; #ifdef CLOCK_MONOTONIC timespec startTs, endTs; clock_gettime(CLOCK_MONOTONIC, &startTs); #endif - - int i = -1; - for (auto it = begin; it != end; it++) { - i++; - if (it->size() > 0 || i % 50 == 0 || i == 4095) { - std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA; - + const size_t shards = end - begin; + std::mutex progressMutex; + size_t done = 0; + auto reportDone = [&](size_t n) { + std::lock_guard lock(progressMutex); + done += n; + std::cout << "\r" << name << ": finalizing z6 tile " << done << "/" << CLUSTER_ZOOM_AREA; #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &endTs); - uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; - std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; + clock_gettime(CLOCK_MONOTONIC, &endTs); + uint64_t elapsedNs = 1e9 * (endTs.tv_sec - startTs.tv_sec) + endTs.tv_nsec - startTs.tv_nsec; + std::cout << " (" << std::to_string((uint32_t)(elapsedNs / 1e6)) << " ms)"; #endif - std::cout << std::flush; - } - if (it->size() == 0) - continue; + std::cout << std::flush; + }; + auto finalizeShard = [&](size_t i, size_t sortThreads) { + auto it = begin + i; // We track a separate copy of low zoom objects to avoid scanning large - // lists of objects that may be on slow disk storage. - for (auto objectIt = it->begin(); objectIt != it->end(); objectIt++) - if (objectIt->oo.minZoom < CLUSTER_ZOOM) - lowZoom[i].push_back(*objectIt); + // lists of objects that may be on slow disk storage. The sort collects them while it reads + // the shard anyway (on its threads). + sortOutputObjects(indexZoom, sortThreads, it->begin(), it->end(), &lowZoom[i]); + }; + + // Shards are independent. They are sorted in parallel, largest first, as many at once as + // the memory budget below allows, each with threads in proportion to its share of the + // budget: a shard holding most objects (one of the few populated z6 tiles of a small + // extract) gets all of them, and the biggest of the planet don't run one after another. + size_t total = 0, empty = 0; + for (auto it = begin; it != end; it++) total += it->size(); + std::vector big, small; + for (size_t i = 0; i < shards; i++) { + const size_t n = (begin + i)->size(); + if (n == 0) empty++; + else if (threadNum <= 1) big.push_back(i); + else small.push_back(i); // all in the pool below, the biggest with more threads + } + std::sort(small.begin(), small.end(), [&](size_t a, size_t b) { return (begin + a)->size() > (begin + b)->size(); }); + reportDone(empty); - sortOutputObjects(indexZoom, threadNum, it->begin(), it->end()); + for (size_t i : big) { + finalizeShard(i, threadNum); + reportDone(1); + } + releaseSortBuffers(); + + if (!small.empty()) { + // Each sort copies its shard into a buffer, so cap the objects being + // sorted at once (a single shard may always run). + const size_t budget = std::max(total / 2, (begin + small.front())->size()); + size_t inFlight = 0; + std::mutex budgetMutex; + std::condition_variable budgetFreed; + boost::asio::thread_pool pool(threadNum); + for (size_t i : small) { + boost::asio::post(pool, [&, i]() { + const size_t n = (begin + i)->size(); + { + std::unique_lock lock(budgetMutex); + budgetFreed.wait(lock, [&]() { return inFlight == 0 || inFlight + n <= budget; }); + inFlight += n; + } + // The budget lets only a few of the largest shards run at once: give each threads in + // proportion to its share of the budget, so that all of them are busy (the sort's + // result doesn't depend on its threads) + finalizeShard(i, std::max(1, std::min(threadNum, threadNum * n / budget))); + // Each thread kept the buffer of the biggest shard it sorted for the next sorts, which + // are smaller (biggest first): beyond the budget, every thread held one of the biggest + // shards' size. Big buffers go after their sort. + if (n * sizeof(OO) >= (64u << 20)) releaseSortBuffers(); + { + std::lock_guard lock(budgetMutex); + inFlight -= n; + } + budgetFreed.notify_all(); + reportDone(1); + }); + } + pool.join(); } std::cout << std::endl; } -template void collectTilesWithObjectsAtZoomTemplate( +// Objects per z12 tile ((x << 12) | y), counted while the tiles are collected when not empty: they +// tell which batches will be slow. A z6 tile's z12 tiles are counted by the thread doing that shard. +extern std::vector z12ObjectCounts; + +// Set the tiles holding the objects of z6 tile i (shard), and their parent tiles. +template void collectTilesWithObjectsInShardTemplate( const unsigned int& indexZoom, - const typename std::vector>::iterator objects, - const size_t size, + AppendVectorNS::AppendVector& shard, + const size_t i, std::vector>& zooms ) { size_t maxZoom = zooms.size() - 1; uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; int64_t lastX = -1; int64_t lastY = -1; - for (size_t i = 0; i < size; i++) { - const size_t z6x = i / CLUSTER_ZOOM_WIDTH; - const size_t z6y = i % CLUSTER_ZOOM_WIDTH; - - for (size_t j = 0; j < objects[i].size(); j++) { - // Compute the x, y at the base zoom level - TileCoordinate baseX = z6x * z6OffsetDivisor + objects[i][j].x; - TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; - - // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); - TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); - - if (lastX != x || lastY != y) { - lastX = x; - lastY = y; - - for (int zoom = maxZoom; zoom >= 0; zoom--) { - zooms[zoom]->set(x, y); - x /= 2; - y /= 2; - } + const size_t z6x = i / CLUSTER_ZOOM_WIDTH; + const size_t z6y = i % CLUSTER_ZOOM_WIDTH; + uint32_t *z12Counts = indexZoom >= 12 && !z12ObjectCounts.empty() ? z12ObjectCounts.data() : nullptr; + + for (size_t j = 0; j < shard.size(); j++) { + // Compute the x, y at the base zoom level + TileCoordinate baseX = z6x * z6OffsetDivisor + shard[j].x; + TileCoordinate baseY = z6y * z6OffsetDivisor + shard[j].y; + if (z12Counts) z12Counts[((size_t)(baseX >> (indexZoom - 12)) << 12) | (baseY >> (indexZoom - 12))]++; + + // Translate the x, y at the requested zoom level + TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); + + if (lastX != x || lastY != y) { + lastX = x; + lastY = y; + + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->set(x, y); + x /= 2; + y /= 2; } } } } +template void collectTilesWithObjectsAtZoomTemplate( + const unsigned int& indexZoom, + const typename std::vector>::iterator objects, + const size_t size, + std::vector>& zooms +) { + for (size_t i = 0; i < size; i++) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[i], i, zooms); +} + template OutputObjectID outputObjectWithId(const OO& input) { return OutputObjectID({ input.oo, 0 }); @@ -146,6 +237,24 @@ template void collectLowZoomObjectsForTile( } } +// The order the shards are sorted in (see sortOutputObjects) as a number: x and y (offsets in the +// z6 tile, below 2^(indexZoom - CLUSTER_ZOOM)) bit-interleaved, x above y at each level. +inline uint32_t indexSpreadBits(uint32_t v) { + v &= 0x0000FFFF; + v = (v | (v << 8)) & 0x00FF00FF; + v = (v | (v << 4)) & 0x0F0F0F0F; + v = (v | (v << 2)) & 0x33333333; + v = (v | (v << 1)) & 0x55555555; + return v; +} +inline uint32_t indexOrderKey(uint32_t x, uint32_t y) { + return (indexSpreadBits(x) << 1) | indexSpreadBits(y); +} + +// When non-zero, getObjectsForTile keeps the objects shown up to this zoom rather than the tile's +// own zoom (see the covered subtrees in tilemaker.cpp) +extern thread_local unsigned objectsUpToZoom; + template void collectObjectsForTileTemplate( const unsigned int& indexZoom, typename std::vector>::iterator objects, @@ -153,7 +262,8 @@ template void collectObjectsForTileTemplate( size_t iEnd, unsigned int zoom, TileCoordinates dstIndex, - std::vector& output + std::vector& output, + bool* nonPolygon = nullptr // if given: stop at the first object that isn't a polygon, and set it ) { if (zoom < CLUSTER_ZOOM) throw std::runtime_error("collectObjectsForTileTemplate should not be called for low zooms"); @@ -187,6 +297,31 @@ template void collectObjectsForTileTemplate( OutputObject dummyOo(POINT_, 0, 0, 0, 0); const size_t bz = indexZoom; + // (At the index zoom itself, one index tile: the search below is as fast) + if (bz >= CLUSTER_ZOOM && bz - CLUSTER_ZOOM <= 8 * sizeof(Z6Offset) && clampedZoom < bz) { + // The shard is sorted by (x, y) compared at each zoom from CLUSTER_ZOOM to indexZoom, i.e. by + // indexOrderKey (x, y < 2^(indexZoom - CLUSTER_ZOOM)). The index tiles inside the tile are those + // with keys [key(needle), key(needle) + 4^(indexZoom - clampedZoom)): one range, found with + // binary searches on the key. The same objects, in the same order, as the loop below. + const uint32_t first = indexOrderKey(needleX, needleY); + const uint32_t last = first + (uint32_t(1) << (2 * (bz - clampedZoom))); + auto keyLess = [](const OO& a, uint32_t k) { return indexOrderKey(a.x, a.y) < k; }; + auto from = std::lower_bound(objects[i].begin(), objects[i].end(), first, keyLess); + // Mostly a few objects (deep zooms): look at them one by one first + auto to = from; + unsigned steps = 0; + while (to != objects[i].end() && steps < 32 && keyLess(*to, last)) { ++to; ++steps; } + if (steps == 32) to = std::lower_bound(to, objects[i].end(), last, keyLess); + const unsigned upTo = objectsUpToZoom ? objectsUpToZoom : zoom; + for (; from != to; ++from) { + if (from->oo.minZoom <= upTo) { + if (nonPolygon && from->oo.geomType != POLYGON_) { *nonPolygon = true; return; } + output.push_back(outputObjectWithId(*from)); + } + } + continue; + } + const OO targetXY = {dummyOo, needleX, needleY }; auto iter = std::lower_bound( objects[i].begin(), @@ -226,7 +361,8 @@ template void collectObjectsForTileTemplate( TileCoordinate y = baseY / (1 << (indexZoom - clampedZoom)); if (dstIndex.x == x && dstIndex.y == y) { - if (iter->oo.minZoom <= zoom) { + if (iter->oo.minZoom <= (objectsUpToZoom ? objectsUpToZoom : zoom)) { + if (nonPolygon && iter->oo.geomType != POLYGON_) { *nonPolygon = true; return; } output.push_back(outputObjectWithId(*iter)); } } else { @@ -245,7 +381,7 @@ template void collectObjectsForTileTemplate( class TileDataSource { public: // Store for generated geometries - using point_store_t = std::vector; + using point_store_t = AppendVectorNS::AppendVector; using linestring_t = boost::geometry::model::linestring; using linestring_store_t = std::vector; @@ -300,20 +436,46 @@ class TileDataSource { std::vector multipolygonStores; // Per multipolygonStores entry: stored geometry is known to pass geom::is_valid std::vector> multipolygonValid; + // Validity checks still running (see checkValidityLater) + std::mutex deferredValidityMutex; + std::vector>> deferredValidity; ClipCache multiPolygonClipCache; ClipCache multiLinestringClipCache; - std::deque>> pendingSmallIndexObjects; + // Small index adds are staged per thread and moved to their shards in bulk (see flushStaging) + template struct IndexStaging { + std::vector entries; + std::vector shards; // z6 index of each entry + }; + struct alignas(64) IndexStagings { // a cache line of its own: each is written by its thread only + IndexStaging objects; + IndexStaging objectsWithIds; + }; + std::mutex stagingMutex; + std::deque stagings; // one per thread that added objects (guarded by stagingMutex) + uint64_t stagingEpoch; // identifies this source's stagings in threads' caches (renewed by finalize) public: TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); + ~TileDataSource(); + const uint64_t leaseNumber = registerLeaseSource(this); // see registerLeaseSource void collectTilesWithObjectsAtZoom(std::vector>& zooms); + // The same for the objects of one z6 tile (0 <= shard < CLUSTER_ZOOM_AREA) + void collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms); + // Objects in the small index of z6 tile `shard` + size_t shardObjectCount(size_t shard) const { + return (shard < objects.size() ? objects[shard].size() : 0) + (shard < objectsWithIds.size() ? objectsWithIds[shard].size() : 0); + } void collectTilesWithLargeObjectsAtZoom(std::vector>& zooms); - void collectObjectsForTile(uint zoom, TileCoordinates dstIndex, std::vector& output); + // nonPolygon: see collectObjectsForTileTemplate (z6 and above) + void collectObjectsForTile(uint zoom, TileCoordinates dstIndex, std::vector& output, bool* nonPolygon = nullptr); + // Whether the objects getObjectsForTile(zoom, dstIndex) returns are none (0), all polygons (1), + // or not (2), without sorting them: for the covered-subtree check + int objectKindsForTile(uint zoom, TileCoordinates dstIndex); void finalize(size_t threadNum); void addGeometryToIndex( @@ -335,6 +497,13 @@ class TileDataSource { void addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id); void addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id, bool needsLock); void addObjectToSmallIndexUnsafe(const TileCoordinates& index, const OutputObject& oo, uint64_t id); + // As addObjectToSmallIndex(tile, oo, id) for each tile and the outputs each(tile, add) calls + // add(oo) for, taking a z6 tile's lock once per run of tiles in it + template void addToSmallIndex(const std::vector& tiles, uint64_t id, Each each); + IndexStagings& indexStaging(); + void stageObject(IndexStagings& staging, size_t z6index, const TileCoordinates& index, const OutputObject& oo, uint64_t id); + void flushStagingIfFull(IndexStagings& staging); + template void flushStaging(IndexStaging& staging, std::vector>& shards); void addObjectToLargeIndex(const Box& envelope, const OutputObject& oo, uint64_t id) { std::lock_guard lock(mutex); @@ -348,11 +517,41 @@ class TileDataSource { std::vector getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ); + void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); + + // Large polygons are clipped once per SHARED_PIECE_ZOOM tile before the tiles are written, + // and deeper tiles clip from that piece instead of the whole polygon (see tile_data.cpp). + static constexpr unsigned SHARED_PIECE_ZOOM = 8; + // present(x, y): whether any tile below SHARED_PIECE_ZOOM tile x/y will be written + void precomputeSharedPieces(const std::function &present, unsigned threadNum); + // Compute pieces until none is left to start; the tile threads call this before their first tile + void computeSharedPieces(); + // Drop the shared pieces inside a z6 tile, once no tile in it is still to be written. Returns + // about how many bytes they took. + uint64_t releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y); + // Whether objectID's polygon is kept in the multipolygon store (retrieveMultiPolygon) + virtual bool storesMultiPolygon(NodeID objectID) const { return true; } + struct SharedPiece { + std::shared_ptr geom; // null: no usable piece, clip the whole polygon + bool valid = false; + }; + // Filled by precomputeSharedPieces, then only read while the tiles are written (no lock). + // releaseSharedPieces resets the pieces of a z6 tile no thread reads any more. + std::unordered_set pieceObjects; + std::map, std::shared_future>> sharedPieces; + struct PieceJob { + std::tuple key; + uint64_t points; + std::shared_ptr>> promise; + }; + std::vector pieceJobs; // largest polygon first + std::atomic nextPieceJob { 0 }; virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; void open() { @@ -398,8 +597,8 @@ class TileDataSource { const auto& shardId = getShard(id); const auto& shard = pointStores[shardId]; const auto offset = getId(id); - if (offset > shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); - return shard.at(offset); + if (offset >= shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); + return shard[offset]; } NodeID storeLinestring(const Linestring& src); @@ -424,6 +623,11 @@ class TileDataSource { // knownValid: src has passed geom::is_valid (see polygonKnownValid) NodeID storeMultiPolygon(const MultiPolygon& src, bool knownValid = false); + // Find out in the background whether a polygon stored with knownValid=false passes + // geom::is_valid: on a big lake with thousands of islands that takes ~10 s, which would + // otherwise hold up the end of reading. finishDeferredValidity records the results. + void checkValidityLater(NodeID objectID, MultiPolygon mp); + void finishDeferredValidity(); multi_polygon_t const &retrieveMultiPolygon(NodeID id) const { const auto& shardId = getShard(id); @@ -436,7 +640,8 @@ class TileDataSource { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum = 1 ); #endif //_TILE_DATA_H diff --git a/include/tile_encode.h b/include/tile_encode.h new file mode 100644 index 00000000..f5183445 --- /dev/null +++ b/include/tile_encode.h @@ -0,0 +1,428 @@ +/*! \file */ +#ifndef _TILE_ENCODE_H +#define _TILE_ENCODE_H + +// Encoding of one feature (geometry in tile coordinates and attributes) into a layer of the +// tile. Shared by tile_worker.cpp and the offline replay of recorded tiles (mvt_record.h). + +#include +#include +#include +#include +#include +#include +#include +#include +#include "coordinates_geom.h" +#include "output_object.h" + +// The tile is written through one of these backends. Both expose the same builder API, +// so the geometry and attribute code below is shared between the output formats. +struct MvtBackend { + using LayerBuilder = vtzero::layer_builder; + using PointFeature = vtzero::point_feature_builder; + using LinestringFeature = vtzero::linestring_feature_builder; + using PolygonFeature = vtzero::polygon_feature_builder; + static void addRing(PolygonFeature& fbuilder, uint32_t points, bool /*outer*/) { fbuilder.add_ring(points); } +}; + +// Direct encoding of MVT features: the Feature message is written into a per-thread buffer and +// added to the layer in one go, rather than through the vtzero feature builders (which write each +// varint byte by byte and reserve/erase the length of three nested messages per feature). The +// bytes are exactly those vtzero writes: fields in the order type, id, geometry, tags; the +// geometry commands, zigzag deltas and minimal length varints as vtzero/protozero encode them. +namespace mvt_direct { + +struct Buffer { + std::unique_ptr data; + std::size_t size = 0, capacity = 0; + // Room for at least n more bytes; returns the write position + char* reserve(std::size_t n) { + if (size + n > capacity) { + std::size_t c = capacity ? capacity * 2 : 4096; + while (c < size + n) c *= 2; + std::unique_ptr d(new char[c]); + if (size) std::memcpy(d.get(), data.get(), size); + data = std::move(d); + capacity = c; + } + return data.get() + size; + } +}; + +inline char* putVarint(char* p, uint64_t v) { + while (v >= 0x80U) { *p++ = char((v & 0x7fU) | 0x80U); v >>= 7U; } + *p++ = char(v); + return p; +} +inline uint32_t zigzag(int32_t v) { return protozero::encode_zigzag32(v); } + +// Geometry of the feature being encoded: command integers and zigzag deltas, as varints +struct Geometry { + Buffer buf; + vtzero::point cursor{0, 0}; + void clear() { buf.size = 0; cursor = vtzero::point{0, 0}; } + // write position with room for maxVarints varints + char* begin(std::size_t maxVarints) { return buf.reserve(maxVarints * 5); } + void end(char* p) { buf.size = p - buf.data.get(); } + char* delta(char* p, int32_t x, int32_t y) { + p = putVarint(p, zigzag(x - cursor.x)); + p = putVarint(p, zigzag(y - cursor.y)); + cursor = vtzero::point{x, y}; + return p; + } +}; + +struct Scratch { + Geometry geometry; + Buffer feature; + std::vector> points; +}; +inline Scratch& scratch() { thread_local Scratch s; return s; } + +// Writes the feature (type, id, the geometry in s.geometry, the tags kv) into the layer +inline void commit(vtzero::layer_builder& vtLayer, vtzero::GeomType type, bool withId, uint64_t id, + const uint32_t* kv, uint32_t kvCount, Scratch& s) { + const std::size_t geomSize = s.geometry.buf.size; + std::size_t tagsSize = 0; + for (uint32_t i = 0; i < 2 * kvCount; i++) { + uint32_t v = kv[i]; + tagsSize++; + while (v >= 0x80U) { tagsSize++; v >>= 7U; } + } + Buffer& f = s.feature; + f.size = 0; + char* const start = f.reserve(2 + 11 + 6 + geomSize + 6 + tagsSize); + char* p = start; + *p++ = char((uint32_t(vtzero::detail::pbf_feature::type) << 3U) | uint32_t(protozero::pbf_wire_type::varint)); + p = putVarint(p, uint64_t(int64_t(static_cast(type)))); + if (withId) { + *p++ = char((uint32_t(vtzero::detail::pbf_feature::id) << 3U) | uint32_t(protozero::pbf_wire_type::varint)); + p = putVarint(p, id); + } + *p++ = char((uint32_t(vtzero::detail::pbf_feature::geometry) << 3U) | uint32_t(protozero::pbf_wire_type::length_delimited)); + p = putVarint(p, geomSize); + std::memcpy(p, s.geometry.buf.data.get(), geomSize); + p += geomSize; + if (kvCount) { + *p++ = char((uint32_t(vtzero::detail::pbf_feature::tags) << 3U) | uint32_t(protozero::pbf_wire_type::length_delimited)); + p = putVarint(p, tagsSize); + for (uint32_t i = 0; i < 2 * kvCount; i++) p = putVarint(p, kv[i]); + } + vtLayer.add_encoded_feature(start, p - start); +} + +} // namespace mvt_direct + +// Points already scaled to the tile (and filtered to the tile extent); not empty +template +void writePointFeature( + typename Backend::LayerBuilder& vtLayer, + bool includeID, + const OutputObjectID& oo, + const std::vector>& multipoint, + const AttributeStore& attributeStore, + unsigned zoom, + MvtAttributeMemo* attrMemo +) { + if constexpr (std::is_same::value) { + if (attrMemo && attrMemo->usableFor(vtLayer, zoom)) { + mvt_direct::Scratch& s = mvt_direct::scratch(); + mvt_direct::Geometry& g = s.geometry; + g.clear(); + char* p = g.begin(1 + 2 * multipoint.size()); + p = mvt_direct::putVarint(p, vtzero::detail::command_move_to(multipoint.size())); + for (const auto &point : multipoint) p = g.delta(p, point.first, point.second); + g.end(p); + uint32_t count; + const uint32_t* kv = attrMemo->resolve(attributeStore, oo.oo.attributes, count); + mvt_direct::commit(vtLayer, vtzero::GeomType::POINT, includeID && oo.id, oo.id, kv, count, s); + return; + } + } + typename Backend::PointFeature fbuilder{vtLayer}; + if (includeID && oo.id) fbuilder.set_id(oo.id); + + fbuilder.add_points(multipoint.size()); + + for (const auto &point : multipoint) + fbuilder.set_point(point.first, point.second); + + oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); + fbuilder.commit(); +} + +// Linestrings in latp/lon (after simplification), scaled to the tile here +template +void writeLinestringFeature( + typename Backend::LayerBuilder& vtLayer, + bool includeID, + const OutputObjectID& oo, + const TileBbox& bbox, + const MultiLinestring& toWrite, + const AttributeStore& attributeStore, + unsigned zoom, + MvtAttributeMemo* attrMemo +) { + if constexpr (std::is_same::value) { + if (attrMemo && attrMemo->usableFor(vtLayer, zoom)) { + mvt_direct::Scratch& s = mvt_direct::scratch(); + mvt_direct::Geometry& g = s.geometry; + g.clear(); + bool hadLine = false; + for (const Linestring& ls : toWrite) { + if (ls.size() <= 1) + continue; + // scaled once, without zero-length segments + std::vector>& pts = s.points; + bbox.scaleLine(pts, ls); + // A line has at least 2 points. + if (pts.size() <= 1) + continue; + hadLine = true; + char* p = g.begin(2 * pts.size() + 2); + p = mvt_direct::putVarint(p, vtzero::detail::command_move_to(1)); + p = g.delta(p, pts[0].first, pts[0].second); + p = mvt_direct::putVarint(p, vtzero::detail::command_line_to(uint32_t(pts.size() - 1))); + for (std::size_t i = 1; i < pts.size(); i++) p = g.delta(p, pts[i].first, pts[i].second); + g.end(p); + } + if (hadLine) { + uint32_t count; + const uint32_t* kv = attrMemo->resolve(attributeStore, oo.oo.attributes, count); + mvt_direct::commit(vtLayer, vtzero::GeomType::LINESTRING, includeID && oo.id, oo.id, kv, count, s); + } + return; + } + } + typename Backend::LinestringFeature fbuilder{vtLayer}; + + if (includeID && oo.id) + fbuilder.set_id(oo.id); + + bool hadLine = false; + + thread_local std::vector> pts; + for (const Linestring& ls : toWrite) { + if (ls.size() <= 1) + continue; + + // vtzero dislikes linesegments that have zero-length segments, + // e.g. where p(x) == p(x + 1). So filter those out. + bbox.scaleLine(pts, ls); + + // A line has at least 2 points. + if (pts.size() <= 1) + continue; + + hadLine = true; + fbuilder.add_linestring(int(pts.size())); + for (const std::pair& xy : pts) + fbuilder.set_point(xy.first, xy.second); + } + + if (hadLine) { + // add the properties + oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); + // call commit() when you are done + fbuilder.commit(); + } +} + +template +bool writeRing( + typename Backend::PolygonFeature& fbuilder, + const Ring& ring, + bool outer +) { + // vtzero doesn't like zero-length segments in rings + std::pair lastXy = std::make_pair(0, 0); + int points = 0; + for (const Point& point : ring) { + std::pair xy = std::make_pair(point.get<0>(), point.get<1>()); + if (points == 0 || xy != lastXy) { + points++; + lastXy = xy; + } + } + + // A ring has at least 4 points. + if (points <= 3) + return false; + + bool firstPoint = true; + Backend::addRing(fbuilder, points, outer); + for (auto it = ring.rbegin(); it != ring.rend(); ++it) { + const Point& point = *it; + + std::pair xy = + std::make_pair(point.get<0>(), point.get<1>()); + + if (firstPoint || xy != lastXy) { + firstPoint = false; + lastXy = xy; + fbuilder.set_point(xy.first, xy.second); + } + } + + return true; +} + +// Polygons in tile coordinates (after simplification and repair) +template +void writePolygonFeature( + typename Backend::LayerBuilder& vtLayer, + bool includeID, + const OutputObjectID& oo, + const MultiPolygon& current, + const AttributeStore& attributeStore, + unsigned zoom, + MvtAttributeMemo* attrMemo +) { + if constexpr (std::is_same::value) { + if (attrMemo && attrMemo->usableFor(vtLayer, zoom)) { + mvt_direct::Scratch& s = mvt_direct::scratch(); + mvt_direct::Geometry& g = s.geometry; + g.clear(); + // As writeRing: in reverse order, without zero-length segments (coordinates truncated + // to int); a ring needs at least 4 points; the last one becomes ClosePath + auto ring = [&](const Ring& r) -> bool { + std::vector>& pts = s.points; + pts.clear(); + for (auto it = r.rbegin(); it != r.rend(); ++it) { + const std::pair xy(static_cast(it->get<0>()), static_cast(it->get<1>())); + if (pts.empty() || xy != pts.back()) pts.push_back(xy); + } + const std::size_t n = pts.size(); + if (n <= 3) + return false; + char* p = g.begin(2 * n + 3); + p = mvt_direct::putVarint(p, vtzero::detail::command_move_to(1)); + p = g.delta(p, pts[0].first, pts[0].second); + p = mvt_direct::putVarint(p, vtzero::detail::command_line_to(uint32_t(n - 2))); + for (std::size_t i = 1; i + 1 < n; i++) p = g.delta(p, pts[i].first, pts[i].second); + p = mvt_direct::putVarint(p, vtzero::detail::command_close_path()); + g.end(p); + return true; + }; + bool hadPoly = false; + for (const auto& poly : current) { + // If we failed to write the outer, no need to write the inners. + if (!ring(poly.outer())) + continue; + hadPoly = true; + for (const Ring& inner : poly.inners()) + ring(inner); + } + if (hadPoly) { + uint32_t count; + const uint32_t* kv = attrMemo->resolve(attributeStore, oo.oo.attributes, count); + mvt_direct::commit(vtLayer, vtzero::GeomType::POLYGON, includeID && oo.id, oo.id, kv, count, s); + } + return; + } + } + typename Backend::PolygonFeature fbuilder{vtLayer}; + + if (includeID && oo.id) + fbuilder.set_id(oo.id); + + bool hadPoly = false; + + for (const auto& p : current) { + const Ring& ring = p.outer(); + + // If we failed to write the outer, no need to write the inners. + if (!writeRing(fbuilder, ring, true)) + continue; + + hadPoly = true; + + for (const Ring& ring : p.inners()) + writeRing(fbuilder, ring, false); + } + + if (hadPoly) { + // add the properties + oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); + // call commit() when you are done + fbuilder.commit(); + } +} + +// Writes the layers of layers.layerOrder into outputdata, in order. layerFn(group, vtLayer) fills +// the layer of layerOrder[group]; attrMemo is reset for each layer. data are the tile's objects +// per source. keepEmptyLayers: when merging into an existing tile. +template +void encodeMvtTile( + const Layers& layers, + unsigned zoom, + uint32_t extent, + uint32_t mvtVersion, + const std::vector>& data, + bool keepEmptyLayers, + const bool& stop, + MvtAttributeMemo& attrMemo, + std::string& outputdata, + LayerFn&& layerFn +) { + // Layers present in the tile's objects (each source is sorted by layer first) + uint64_t present[4] = { 0, 0, 0, 0 }; + for (const auto& objects : data) { + for (auto it = objects.begin(); it != objects.end(); ) { + const unsigned layer = it->oo.layer; + present[layer >> 6] |= uint64_t(1) << (layer & 63); + it = std::upper_bound(it, objects.end(), layer, [](unsigned l, const OutputObjectID& o) { return l < o.oo.layer; }); + } + } + // Per layer group the bits of its layers, per zoom the bits of the layers shown at that zoom + // (computed once per thread; the layer definitions don't change while tiles are written) + struct LayerMasks { + const void* layers = nullptr; + std::size_t groups = 0, count = 0; + std::vector> group; + std::array, 32> zoom; + }; + thread_local LayerMasks masks; + if (masks.layers != &layers || masks.groups != layers.layerOrder.size() || masks.count != layers.layers.size()) { + masks.layers = &layers; + masks.groups = layers.layerOrder.size(); + masks.count = layers.layers.size(); + masks.group.assign(masks.groups, std::array{0, 0, 0, 0}); + for (auto& z : masks.zoom) z = std::array{0, 0, 0, 0}; + for (std::size_t g = 0; g < masks.groups; g++) + for (const auto layerNum : layers.layerOrder[g]) + masks.group[g][(layerNum >> 6) & 3] |= uint64_t(1) << (layerNum & 63); + for (std::size_t layerNum = 0; layerNum < masks.count; layerNum++) + for (unsigned z = 0; z < 32; z++) + if (z >= layers.layers[layerNum].minzoom && z <= layers.layers[layerNum].maxzoom) + masks.zoom[z][(layerNum >> 6) & 3] |= uint64_t(1) << (layerNum & 63); + } + vtzero::tile_builder tile; + for (std::size_t group = 0; group < layers.layerOrder.size(); group++) { + if (stop) break; + // A layer group without objects in its zoom range writes no features, and a layer + // without features isn't serialized: skip it (unless merging into an existing tile) + if (!keepEmptyLayers) { + bool any = false; + if (zoom < 32) { + const auto& g = masks.group[group]; + const auto& z = masks.zoom[zoom]; + any = ((present[0] & g[0] & z[0]) | (present[1] & g[1] & z[1]) | (present[2] & g[2] & z[2]) | (present[3] & g[3] & z[3])) != 0; + } else { + for (const auto layerNum : layers.layerOrder[group]) { + const auto& ld = layers.layers[layerNum]; + if (zoom >= ld.minzoom && zoom <= ld.maxzoom && (present[(layerNum >> 6) & 3] >> (layerNum & 63) & 1)) { any = true; break; } + } + } + if (!any) continue; + } + const std::string& layerName = layers.layers[layers.layerOrder[group].at(0)].name; + vtzero::layer_builder vtLayer{tile, layerName, mvtVersion, extent}; + attrMemo.reset(vtLayer, zoom); + layerFn(group, vtLayer); + } + tile.serialize(outputdata); +} + +#endif //_TILE_ENCODE_H diff --git a/include/tile_worker.h b/include/tile_worker.h index ae3cab8a..dace5174 100644 --- a/include/tile_worker.h +++ b/include/tile_worker.h @@ -12,7 +12,8 @@ void outputProc( const AttributeStore& attributeStore, const std::vector>& data, TileCoordinates coordinates, - uint zoom + uint zoom, + std::string* rawTile = nullptr // if given, receives the uncompressed tile ); #endif //_TILE_WORKER_H diff --git a/include/visvalingam.h b/include/visvalingam.h index eb43d5bc..8e1cc527 100644 --- a/include/visvalingam.h +++ b/include/visvalingam.h @@ -7,5 +7,7 @@ Linestring simplifyVis(const Linestring &ls, double max_distance); Polygon simplifyVis(const Polygon &p, double max_distance); MultiPolygon simplifyVis(const MultiPolygon &mp, double max_distance); MultiPolygon simplifyVis(const MultiPolygon &mp, double max_distance, bool &allValid); +// The same as mp = simplifyVis(mp, max_distance, allValid), in place +void simplifyVisInPlace(MultiPolygon &mp, double max_distance, bool &allValid); #endif //_VISVALINGAM_H diff --git a/include/vtzero/builder.hpp b/include/vtzero/builder.hpp index 781758e8..f229f6dd 100644 --- a/include/vtzero/builder.hpp +++ b/include/vtzero/builder.hpp @@ -83,18 +83,41 @@ namespace vtzero { */ template detail::layer_builder_impl* add_layer(TString&& name, uint32_t version, uint32_t extent) { + auto& pool = recycled(); + if (!pool.empty()) { + m_layers.push_back(std::move(pool.back())); + pool.pop_back(); + m_layers.back()->reinit(std::forward(name), version, extent); + return m_layers.back().get(); + } auto* ptr = new detail::layer_builder_impl{std::forward(name), version, extent}; m_layers.emplace_back(ptr); return ptr; } + // Layers of destroyed tile_builders of this thread, for reuse by add_layer() + static std::vector>& recycled() { + thread_local std::vector> pool; + return pool; + } + public: /// Constructor tile_builder() = default; /// Destructor - ~tile_builder() noexcept = default; + ~tile_builder() noexcept { + try { + auto& pool = recycled(); + for (auto& layer : m_layers) { + if (pool.size() >= 32) break; + if (layer && layer->recyclable()) pool.push_back(std::move(layer)); + } + } catch (...) { + // the remaining layers are freed + } + } /// Tile builders can not be copied. tile_builder(const tile_builder&) = delete; @@ -298,6 +321,15 @@ namespace vtzero { * will be copied completely over to the new layer including its * geometry and all its properties. */ + /** + * Add a feature that is already encoded as a vector tile Feature + * message (the bytes of the message, without its tag and length). + */ + void add_encoded_feature(const char* data, std::size_t size) { + m_layer->message().add_message(detail::pbf_layer::features, data, size); + m_layer->increment_feature_count(); + } + void add_feature(const feature& feature); }; // class layer_builder diff --git a/include/vtzero/builder_impl.hpp b/include/vtzero/builder_impl.hpp index 1b348df2..042fc6c8 100644 --- a/include/vtzero/builder_impl.hpp +++ b/include/vtzero/builder_impl.hpp @@ -167,6 +167,33 @@ namespace vtzero { ~layer_builder_impl() noexcept = default; + // Start again as a new, empty layer created from scratch: the + // same state as after the constructor above, but the buffers + // keep their memory. + template + void reinit(TString&& name, uint32_t version, uint32_t extent) { + m_data_view = data_view{}; + m_data.clear(); + m_keys_data.clear(); + m_values_data.clear(); + m_num_features = 0; + m_version = version; + m_num_keys = 0; + m_num_values = 0; + // clear() keeps the bucket array, which is cleared again on every reuse + if (m_keys_index.bucket_count() > 64) map_type{}.swap(m_keys_index); else m_keys_index.clear(); + if (m_values_index.bucket_count() > 64) map_type{}.swap(m_values_index); else m_values_index.clear(); + m_pbf_message_layer.add_uint32(detail::pbf_layer::version, version); + m_pbf_message_layer.add_string(detail::pbf_layer::name, std::forward(name)); + m_pbf_message_layer.add_uint32(detail::pbf_layer::extent, extent); + } + + // Worth keeping for reinit()? (not a copied layer, not much memory) + bool recyclable() const noexcept { + return !m_data_view.data() && + m_data.capacity() + m_keys_data.capacity() + m_values_data.capacity() <= (64U << 10U); + } + layer_builder_impl(const layer_builder_impl&) = delete; layer_builder_impl& operator=(const layer_builder_impl&) = delete; diff --git a/resources/process-debug.lua b/resources/process-debug.lua index 4ea5e629..5fc399f1 100644 --- a/resources/process-debug.lua +++ b/resources/process-debug.lua @@ -155,6 +155,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -504,7 +508,8 @@ function GetPOIRank(obj) local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] if list[Find(k)] then v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or v diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 74892add..c34403a6 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -3,6 +3,11 @@ -- Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors. -- Used under CC-BY 4.0 +-- Speed: tilemaker's functions and the standard library as locals (a global is a table lookup per use). +-- tilemaker registers its functions after loading this file, so they are picked up in init_function. +local tonumber, ipairs, pairs, math, string, table = tonumber, ipairs, pairs, math, string, table +local Find, Holds, Layer, LayerAsCentroid, Attribute, AttributeNumeric, AttributeInteger, AttributeBoolean, MinZoom, ZOrder, IsClosed, Area, Centroid, NextRelation, FindInRelation, Accept, MercatorArea, MinZoomByArea + -------- -- Alter these lines to control which languages are written for place/streetnames -- @@ -16,8 +21,22 @@ default_language_attribute = "name_int" additional_languages = { } -------- +-- way_function only reads tags (and IsClosed/NextRelation) and calls tilemaker's functions: tilemaker may replay it +way_memo = true +-- Tag values way_function only tests for being empty and copies unchanged into these attributes: +-- the memo replays such ways with each way's own value +-- (ref: also its length, into ref_length) +way_memo_values = { name = { "name:latin" }, ["addr:housenumber"] = { "housenumber" }, ref = { "ref", ref_length = "length" } } +-- way_function uses NextRelation() only to test for a relation, and the relations' tags only through FindInRelation +way_memo_relations = true +-- node_function, likewise (the same tags: node_function only tests them for being empty and copies them unchanged) +node_memo = true +node_memo_values = { name = { "name:latin" }, ["addr:housenumber"] = { "housenumber" } } + -- Enter/exit Tilemaker function init_function(name,is_first) + Find, Holds, Layer, LayerAsCentroid, Attribute, AttributeNumeric, AttributeInteger, AttributeBoolean, MinZoom, ZOrder, IsClosed, Area, Centroid, NextRelation, FindInRelation, Accept, MercatorArea, MinZoomByArea = + _G.Find, _G.Holds, _G.Layer, _G.LayerAsCentroid, _G.Attribute, _G.AttributeNumeric, _G.AttributeInteger, _G.AttributeBoolean, _G.MinZoom, _G.ZOrder, _G.IsClosed, _G.Area, _G.Centroid, _G.NextRelation, _G.FindInRelation, _G.Accept, _G.MercatorArea, _G.MinZoomByArea end function exit_function() end @@ -44,6 +63,25 @@ ZRES10 = 152.9 ZRES11 = 76.4 ZRES12 = 38.2 ZRES13 = 19.1 +-- Their squares, worked out once (same values as ZRESn^2 at each use) +local ZRES0SQ = ZRES0^2 +local ZRES1SQ = ZRES1^2 +local ZRES2SQ = ZRES2^2 +local ZRES3SQ = ZRES3^2 +local ZRES4SQ = ZRES4^2 +local ZRES5SQ = ZRES5^2 +local ZRES6SQ = ZRES6^2 +local ZRES7SQ = ZRES7^2 +local ZRES8SQ = ZRES8^2 +local ZRES9SQ = ZRES9^2 +local ZRES10SQ = ZRES10^2 +local ZRES11SQ = ZRES11^2 +local ZRES12SQ = ZRES12^2 +local ZRES13SQ = ZRES13^2 +-- Area thresholds for zoom 1, 2... 13 (see SetMinZoomByAreaWithLimit) +local ZRESSQ = { ZRES0SQ, ZRES1SQ, ZRES2SQ, ZRES3SQ, ZRES4SQ, ZRES5SQ, ZRES6SQ, ZRES7SQ, ZRES8SQ, ZRES9SQ, ZRES10SQ, ZRES11SQ, ZRES12SQ } +-- way_function compares MercatorArea() only with these (see way_memo) +way_memo_area = { ZRES8SQ } -- The height of one floor, in meters BUILDING_FLOOR_HEIGHT = 3.66 @@ -271,6 +309,10 @@ poiTags = { aerialway = Set { "station" }, sport = Set { "american_football", "archery", "athletics", "australian_football", "badminton", "baseball", "basketball", "beachvolleyball", "billiards", "bmx", "boules", "bowls", "boxing", "canadian_football", "canoe", "chess", "climbing", "climbing_adventure", "cricket", "cricket_nets", "croquet", "curling", "cycling", "disc_golf", "diving", "dog_racing", "equestrian", "fatsal", "field_hockey", "free_flying", "gaelic_games", "golf", "gymnastics", "handball", "hockey", "horse_racing", "horseshoes", "ice_hockey", "ice_stock", "judo", "karting", "korfball", "long_jump", "model_aerodrome", "motocross", "motor", "multi", "netball", "orienteering", "paddle_tennis", "paintball", "paragliding", "pelota", "racquet", "rc_car", "rowing", "rugby", "rugby_league", "rugby_union", "running", "sailing", "scuba_diving", "shooting", "shooting_range", "skateboard", "skating", "skiing", "soccer", "surfing", "swimming", "table_soccer", "table_tennis", "team_handball", "tennis", "toboggan", "volleyball", "water_ski", "yoga" }, tourism = Set { "alpine_hut", "aquarium", "artwork", "attraction", "bed_and_breakfast", "camp_site", "caravan_site", "chalet", "gallery", "guest_house", "hostel", "hotel", "information", "motel", "museum", "picnic_site", "theme_park", "viewpoint", "zoo" }, waterway = Set { "dock" } } +-- poiTags keys in alphabetical order: GetPOIRank tries them in this order, and the first key an object has decides its class +poiTagOrder = {} +for k in pairs(poiTags) do poiTagOrder[#poiTagOrder+1] = k end +table.sort(poiTagOrder) -- POI "class" values: based on https://github.com/openmaptiles/openmaptiles/blob/master/layers/poi/poi.yaml poiClasses = { townhall="town_hall", public_building="town_hall", courthouse="town_hall", community_centre="town_hall", @@ -742,7 +784,7 @@ function way_function() Layer("landuse", true) Attribute("class", l) if l=="residential" then - if MercatorArea()ZRES0^2 then MinZoom(1) - elseif minzoom <= 2 and area>ZRES1^2 then MinZoom(2) - elseif minzoom <= 3 and area>ZRES2^2 then MinZoom(3) - elseif minzoom <= 4 and area>ZRES3^2 then MinZoom(4) - elseif minzoom <= 5 and area>ZRES4^2 then MinZoom(5) - elseif minzoom <= 6 and area>ZRES5^2 then MinZoom(6) - elseif minzoom <= 7 and area>ZRES6^2 then MinZoom(7) - elseif minzoom <= 8 and area>ZRES7^2 then MinZoom(8) - elseif minzoom <= 9 and area>ZRES8^2 then MinZoom(9) - elseif minzoom <= 10 and area>ZRES9^2 then MinZoom(10) - elseif minzoom <= 11 and area>ZRES10^2 then MinZoom(11) - elseif minzoom <= 12 and area>ZRES11^2 then MinZoom(12) - elseif minzoom <= 13 and area>ZRES12^2 then MinZoom(13) - else MinZoom(14) end + -- The first zoom z >= minzoom whose pixel area the feature exceeds (ZRESSQ[z]), else 14: + -- worked out by tilemaker, so that way_function stays replayable (see way_memo) + MinZoomByArea(minzoom, ZRESSQ) end -- Calculate POIs (typically rank 1-4 go to 'poi' z12-14, rank 5+ to 'poi_detail' z14) @@ -912,9 +932,10 @@ function GetPOIRank() local k,list,v,class,rank -- Can we find the tag? - for k,list in pairs(poiTags) do - if list[Find(k)] then - v = Find(k) -- k/v are the OSM tag pair + for _,k in ipairs(poiTagOrder) do + list = poiTags[k] + v = Find(k) -- k/v are the OSM tag pair + if list[v] then class = poiClasses[v] or poiTagsAsClass[k] or v rank = poiClassRanks[class] or 25 subclassKey = poiSubClasses[v] diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6e661c8d..fabed0ca 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -1,4 +1,5 @@ #include "attribute_store.h" +#include "stable_hash.h" #include #include @@ -55,6 +56,15 @@ const std::string& AttributeKeyStore::getKeyUnsafe(uint16_t index) const { return keys[index]; } +void AttributeKeyStore::finalize() { + finalized = true; + // keys2index is ordered by key string + ranks.assign(keys.size(), 0); + uint16_t rank = 0; + for (const auto& it : keys2index) + ranks[it.second] = ++rank; +} + // AttributePair void AttributePair::ensureStringIsOwned() { // Before we store an AttributePair in our long-term storage, we need @@ -220,11 +230,18 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { } } - if (isHot) { + if (isHot && hotFull.load(std::memory_order_acquire)) { + // The hot shard is full, and so doesn't change any more: no lock needed + const auto& index = pairs[0].findWithHash(pair, hash); + if (index != -1) { + table.insert(&pairs[0][index], tag, index); + return index; + } + } else if (isHot) { // This might be a popular pair, worth re-using. // Have we already assigned it a hot ID? std::lock_guard lock(pairsMutex[0]); - const auto& index = pairs[0].find(pair); + const auto& index = pairs[0].findWithHash(pair, hash); if (index != -1) { table.insert(&pairs[0][index], tag, index); return index; @@ -232,11 +249,13 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { if (!pairs[0].full()) { pair.ensureStringIsOwned(); - uint32_t offset = pairs[0].add(pair); + uint32_t offset = pairs[0].addAbsentWithHash(pair, hash); uint32_t rv = (0 << (32 - SHARD_BITS)) + offset; table.insert(&pairs[0][offset], tag, rv); + if (pairs[0].full()) hotFull.store(true, std::memory_order_release); return rv; } + hotFull.store(true, std::memory_order_release); } // This is either not a hot key, or there's no room for in the hot shard. @@ -254,7 +273,7 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { if (tlsPairLookupsUncached % 1024 == 0) lookupsUncached += 1024; - const auto& index = pairs[shard].find(pair); + const auto& index = pairs[shard].findWithHash(pair, hash); if (index != -1) { const uint32_t rv = (shard << (32 - SHARD_BITS)) + index; table.insert(&pairs[shard][index], tag, rv); @@ -262,7 +281,7 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { } pair.ensureStringIsOwned(); - uint32_t offset = pairs[shard].add(pair); + uint32_t offset = pairs[shard].addAbsentWithHash(pair, hash); if (offset >= (1 << (32 - SHARD_BITS))) throw std::out_of_range("pair shard overflow"); @@ -345,26 +364,38 @@ void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const bool isHot = v >= 0 && v <= 25; attributeSet.addPair(pairStore.addPair(kv, isHot)); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { PooledString ps(&v); AttributePair kv(keyIndex, ps, minzoom); bool isHot = AttributePair::isHot(key, v); - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, bool v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = true; // All bools are eligible to be hot pairs - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, double v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = v >= 0 && v <= 25 && ceil(v) == v; // Whole numbers in 0..25 are eligible to be hot pairs - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); } -void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom) { +uint32_t AttributeStore::pairIndexWithKeyIndex(uint16_t keyIndex, int v, char minzoom) { AttributePair kv(keyIndex,v,minzoom); bool isHot = v >= 0 && v <= 25; - attributeSet.addPair(pairStore.addPair(kv, isHot)); + return pairStore.addPair(kv, isHot); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, key, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); +} +void AttributeStore::addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom) { + attributeSet.addPair(pairIndexWithKeyIndex(keyIndex, v, minzoom)); } void AttributeSet::finalize() { @@ -411,6 +442,12 @@ thread_local std::vector cachedAttributeSetIndexes(256); thread_local uint64_t tlsSetLookups = 0; thread_local uint64_t tlsSetLookupsUncached = 0; +void AttributeStore::noteLookup() { + tlsSetLookups++; + if (tlsSetLookups % 1024 == 0) { + lookups += 1024; + } +} AttributeIndex AttributeStore::add(AttributeSet &attributes) { // TODO: there's probably a way to use C++ types to distinguish a finalized // and non-finalized AttributeSet, which would make this safer. @@ -444,7 +481,7 @@ AttributeIndex AttributeStore::add(AttributeSet &attributes) { if (tlsSetLookupsUncached % 1024 == 0) lookupsUncached += 1024; - const uint32_t offset = sets[shard].add(attributes); + const uint32_t offset = sets[shard].addWithHash(attributes, hash); if (offset >= (1 << (32 - SHARD_BITS))) throw std::out_of_range("set shard overflow"); @@ -557,6 +594,52 @@ void AttributeStore::reset() { pairIndexGeneration++; } +int AttributeStore::comparePairsUnsafe(uint32_t a, uint32_t b) const { + if (a == b) return 0; + const AttributePair& p = pairStore.getPairUnsafe(a); + const AttributePair& q = pairStore.getPairUnsafe(b); + const uint16_t pRank = keyStore.getRankUnsafe(p.keyIndex), qRank = keyStore.getRankUnsafe(q.keyIndex); + if (pRank != qRank) return pRank < qRank ? -1 : 1; + if (p.minzoom != q.minzoom) return p.minzoom < q.minzoom ? -1 : 1; + if (p.valueType != q.valueType) return p.valueType < q.valueType ? -1 : 1; + if (p.hasStringValue()) { + if (p.pooledString() < q.pooledString()) return -1; + if (q.pooledString() < p.pooledString()) return 1; + return 0; + } + if (p.doubleValue_ < q.doubleValue_) return -1; + if (q.doubleValue_ < p.doubleValue_) return 1; + return 0; +} + +int AttributeStore::compareContentUnsafe(AttributeIndex a, AttributeIndex b) const { + if (a == b) return 0; + boost::container::small_vector x, y; + forEachPairIndexCanonicalUnsafe(a, [&](uint32_t pairIndex) { x.push_back(pairIndex); }); + forEachPairIndexCanonicalUnsafe(b, [&](uint32_t pairIndex) { y.push_back(pairIndex); }); + for (size_t i = 0; i < x.size() && i < y.size(); i++) { + const int c = comparePairsUnsafe(x[i], y[i]); + if (c != 0) return c; + } + if (x.size() != y.size()) return x.size() < y.size() ? -1 : 1; + return 0; +} + +uint64_t AttributeStore::contentHashUnsafe(AttributeIndex index) const { + // Sum of the pair hashes, so the (index dependent) pair order doesn't matter + uint64_t rv = 0; + forEachPairUnsafe(index, [&](const AttributePair& p) { + uint64_t h = stableCombine(keyStore.getRankUnsafe(p.keyIndex), static_cast(p.minzoom)); + h = stableCombine(h, static_cast(p.valueType)); + if (p.hasStringValue()) + h = stableCombine(h, p.pooledString().data(), p.pooledString().size()); + else + h = stableCombine(h, p.doubleValue_); + rv += h; + }); + return rv; +} + void AttributeStore::finalize() { finalized = true; keyStore.finalize(); diff --git a/src/coordinates.cpp b/src/coordinates.cpp index 4a79f83e..b3348426 100644 --- a/src/coordinates.cpp +++ b/src/coordinates.cpp @@ -1,6 +1,8 @@ #include "coordinates.h" #include #include +#include +#include TileCoordinates_::TileCoordinates_() { this->x = 0; @@ -23,6 +25,79 @@ static inline double clamp(double value, double limit) { double lat2latp(double lat) { return rad2deg(log(tan(deg2rad(clamp(lat,85.06)+90.0)/2.0))); } double latp2lat(double latp) { return rad2deg(atan(exp(deg2rad(latp)))*2.0)-90.0; } +// Fast exact int(lat2latp(lat / 1e7) * 1e7) for latitudes in 1e-7 degrees (the conversion of every +// node read from the .osm.pbf). +// +// A piecewise polynomial (degree 5, 1024 intervals over [0, 85.06] degrees, |lat|; the function is +// odd) approximates lat2latp(lat) * 1e7. Where it is more than Margin (1e-3, in 1e-7 degrees) +// away from an integer, the truncation of lat2latp's result is the same integer: the polynomial is +// within ~1e-5 of the exact function, lat2latp's double result within ~2e-6 (rounding of tan's +// argument near the clamp, log/tan within an ulp). Otherwise (0.2% of the latitudes, and beyond the +// clamp at +-85.06) lat2latp itself is called. Checked against lat2latp for every latitude up to +-86 degrees (and samples beyond). +namespace { +namespace latp_e7 { + constexpr int N = 1024; + constexpr double Limit = 85.06; // lat2latp's clamp + constexpr double Margin = 1e-3; + constexpr uint32_t MaxFast = 850599999; // |lat| up to this: below the clamp + struct Table { + double scale; // intervals per degree + double scalePerE7; // intervals per 1e-7 degree + double c[N][6]; // per interval, monomial coefficients in t in [-1, 1] + Table() { + scale = N / Limit; + scalePerE7 = scale / 10000000.0; + const double pi = 3.14159265358979323846; + for (int i = 0; i < N; i++) { + // Chebyshev interpolation at 6 nodes, then the monomial coefficients + const double lo = i / scale, hi = (i + 1) / scale; + double f[6], ch[6] = {}; + for (int j = 0; j < 6; j++) { + const double t = std::cos(pi * (j + 0.5) / 6); + f[j] = lat2latp((lo + hi) / 2 + t * (hi - lo) / 2) * 10000000.0; + } + for (int k = 0; k < 6; k++) { + double s = 0; + for (int j = 0; j < 6; j++) s += f[j] * std::cos(pi * k * (j + 0.5) / 6); + ch[k] = s * (k == 0 ? 1.0 : 2.0) / 6; + } + // T0 = 1, T1 = t, T2 = 2t^2-1, T3 = 4t^3-3t, T4 = 8t^4-8t^2+1, T5 = 16t^5-20t^3+5t + c[i][0] = ch[0] - ch[2] + ch[4]; + c[i][1] = ch[1] - 3 * ch[3] + 5 * ch[5]; + c[i][2] = 2 * ch[2] - 8 * ch[4]; + c[i][3] = 4 * ch[3] - 20 * ch[5]; + c[i][4] = 8 * ch[4]; + c[i][5] = 16 * ch[5]; + } + } + }; + inline const Table& table() { static const Table t; return t; } + + inline int32_t exact(int32_t lat) { return int(lat2latp(double(lat) / 10000000.0) * 10000000.0); } + + inline int32_t convert(int32_t lat) { + const Table& tb = table(); + const uint32_t ua = lat < 0 ? 0u - uint32_t(lat) : uint32_t(lat); + if (ua > MaxFast) return exact(lat); + const double s = double(ua) * tb.scalePerE7; + int i = int(s); + if (i >= N) i = N - 1; + const double t = 2 * (s - i) - 1; + const double* c = tb.c[i]; + const double t2 = t * t; + const double v = (c[0] + c[1] * t) + t2 * ((c[2] + c[3] * t) + t2 * (c[4] + c[5] * t)); + // the truncation of every value within Margin of v (v - Margin >= 0: truncation = floor) + const double vlo = v - Margin; + if (!(vlo >= 0)) return exact(lat); + const int64_t lo = int64_t(vlo); + if (lo != int64_t(v + Margin)) return exact(lat); + return lat < 0 ? -int32_t(lo) : int32_t(lo); + } +} +} + +int32_t lat2latpE7(int32_t lat) { return latp_e7::convert(lat); } + // Tile conversions double lon2tilexf(double lon, uint8_t z) { return scalbn((lon+180.0) * (1/360.0), (int)z); } double latp2tileyf(double latp, uint8_t z) { return scalbn((180.0-latp) * (1/360.0), (int)z); } diff --git a/src/coordinates_geom.cpp b/src/coordinates_geom.cpp index 16887856..a9e3cb2f 100644 --- a/src/coordinates_geom.cpp +++ b/src/coordinates_geom.cpp @@ -1,4 +1,7 @@ #include "coordinates_geom.h" +#ifdef __SSE2__ +#include +#endif using namespace std; namespace geom = boost::geometry; @@ -31,11 +34,81 @@ pair TileBbox::scaleLatpLon(double latp, double lon) const { return pair(x,y); } +#ifdef __SSE2__ +// scaleLatpLon() of a point (x lon, y latp), both lanes at once, as doubles (x low, y high). +// Same subtraction and division per lane; floor() by truncation, minus one where that rounded +// up. Returns false if a lane doesn't fit an int (the caller then uses scaleLatpLon(), whose +// out of range conversion this would not reproduce). +static inline bool scaleLatpLon2(__m128d v, __m128d hiMinuend, __m128d loSubtrahend, __m128d scale, __m128d &out) { + // lanes: (lon - minLon, maxLatp - latp) + const __m128d a = _mm_move_sd(hiMinuend, v); // (lon, maxLatp) + const __m128d b = _mm_move_sd(v, loSubtrahend); // (minLon, latp) + const __m128d d = _mm_div_pd(_mm_sub_pd(a, b), scale); + const __m128i t = _mm_cvttpd_epi32(d); // truncated; INT_MIN if NaN or out of range + if (_mm_movemask_epi8(_mm_cmpeq_epi32(t, _mm_set_epi32(0, 0, INT32_MIN, INT32_MIN))) & 0xff) return false; + const __m128d up = _mm_cmpgt_pd(_mm_cvtepi32_pd(t), d); + const __m128i f = _mm_add_epi32(t, _mm_shuffle_epi32(_mm_castpd_si128(up), _MM_SHUFFLE(3, 3, 2, 0))); + out = _mm_cvtepi32_pd(f); + return true; +} +#endif + +void TileBbox::scaleLine(std::vector> &pts, Linestring const &src) const { +#ifdef __SSE2__ + // scale every point first (independent divisions), then drop repeats in place + const size_t n = src.size(); + pts.resize(n); + const __m128d hiMinuend = _mm_set_pd(maxLatp, 0), loSubtrahend = _mm_set_sd(minLon), scale = _mm_set_pd(yscale, xscale); + for (size_t k = 0; k < n; k++) { + const Point &i = src[k]; + __m128d sp; + if (scaleLatpLon2(_mm_loadu_pd(reinterpret_cast(&i)), hiMinuend, loSubtrahend, scale, sp)) { + // sp holds two ints as doubles: converting back is exact + const __m128i t = _mm_cvttpd_epi32(sp); + pts[k].first = _mm_cvtsi128_si32(t); + pts[k].second = _mm_cvtsi128_si32(_mm_shuffle_epi32(t, _MM_SHUFFLE(1, 1, 1, 1))); + } else + pts[k] = scaleLatpLon(i.y(), i.x()); + } + size_t w = n ? 1 : 0; + for (size_t r = 1; r < n; r++) + if (pts[r] != pts[w - 1]) pts[w++] = pts[r]; + pts.resize(w); +#else + pts.clear(); + for (auto const &i : src) { + const std::pair xy = scaleLatpLon(i.y(), i.x()); + if (pts.empty() || xy != pts.back()) pts.push_back(xy); + } +#endif +} + // Scaling with naive self-intersection check - if we've added the new point // within the last 5 points, then backtrack to the last time we added it void TileBbox::scaleRing(Ring &points, Ring const &src) const { points.clear(); points.reserve(src.size()); +#ifdef __SSE2__ + const __m128d hiMinuend = _mm_set_pd(maxLatp, 0), loSubtrahend = _mm_set_sd(minLon), scale = _mm_set_pd(yscale, xscale); + for(auto const &i: src) { + __m128d sp; + if (!scaleLatpLon2(_mm_loadu_pd(reinterpret_cast(&i)), hiMinuend, loSubtrahend, scale, sp)) { + auto scaled = scaleLatpLon(i.y(), i.x()); + sp = _mm_set_pd(scaled.second, scaled.first); + } + const size_t n = points.size(); + const double *d = reinterpret_cast(points.data()); + size_t j = 1; + for (; j < 5 && n >= 1 + j; j++) + if (_mm_movemask_pd(_mm_cmpeq_pd(_mm_loadu_pd(d + 2 * (n - j)), sp)) == 3) break; + if (j < 5 && n >= 1 + j) points.resize(n - j + 1); + else { + Point p; + _mm_storeu_pd(reinterpret_cast(&p), sp); + points.push_back(p); + } + } +#else for(auto const &i: src) { auto scaled = scaleLatpLon(i.y(), i.x()); // -> .first is x, .second is y bool found = false; @@ -48,6 +121,25 @@ void TileBbox::scaleRing(Ring &points, Ring const &src) const { } if (!found) points.push_back(Point(scaled.first,scaled.second)); } +#endif +} + +// Scaling that only drops points repeating their immediate predecessor. Used as +// a fallback for rings that scaleRing() collapses below 4 points: its +// backtracking window is positional, so a duplicated FIRST vertex shifts every +// later vertex by one, the closing vertex then matches that duplicate at j==4, +// and resize() truncates the whole ring. Without the backtracking there is +// nothing to mis-align, and the ring survives with its shape intact. +void TileBbox::scaleRingNoBacktrack(Ring &points, Ring const &src) const { + points.clear(); + points.reserve(src.size()); + for(auto const &i: src) { + auto scaled = scaleLatpLon(i.y(), i.x()); + if (!points.empty() && + points.back().x()==scaled.first && points.back().y()==scaled.second) + continue; + points.push_back(Point(scaled.first,scaled.second)); + } } Ring TileBbox::scaleRing(Ring const &src) const { @@ -66,8 +158,15 @@ void TileBbox::scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const { // Copy the outer ring scaleRing(p.outer(), poly.outer()); - if (p.outer().size()<4) - continue; + if (p.outer().size()<4) { + // A collapsed outer ring means the whole feature disappears from the + // tile, at any size - so before giving up, retry without the + // backtracking that can truncate a ring to two points. Every ring that + // already survives is left untouched. + scaleRingNoBacktrack(p.outer(), poly.outer()); + if (p.outer().size()<4) + continue; + } // Copy the inner rings if (p.inners().size() < poly.inners().size()) @@ -211,7 +310,8 @@ static inline void pushTile(std::vector &tiles, TileCoordinate template void impl_insertIntermediateTilesVec(T const &points, uint baseZoom, std::vector &tiles) { const size_t start = tiles.size(); - const double scale = scalbn(1.0, (int)(uint8_t)baseZoom); + const unsigned zoom = (uint8_t)baseZoom; + const double scale = zoom < 32 ? double(1u << zoom) : scalbn(1.0, (int)zoom); // exactly 2^zoom TileCoordinate tileX1 = 0, tileY1 = 0; for (auto it = points.begin(); it != points.end(); ++it) { // Calculate p2 tile (same arithmetic as lon2tilexf/latp2tileyf) @@ -223,6 +323,8 @@ void impl_insertIntermediateTilesVec(T const &points, uint baseZoom, std::vector tileX1 = tileX2; tileY1 = tileY2; continue; } + // Most points are in the tile of the previous one, which is the last tile pushed: nothing to add + if (tileX2 == tileX1 && tileY2 == tileY1) continue; // Supercover line algorithm from http://eugen.dedu.free.fr/projects/bresenham/ int i; // loop counter diff --git a/src/geom.cpp b/src/geom.cpp index 43d7d079..2c5a5473 100644 --- a/src/geom.cpp +++ b/src/geom.cpp @@ -12,6 +12,9 @@ #include #include #include +#ifdef __SSE2__ +#include +#endif #include #include @@ -148,7 +151,7 @@ void make_valid(MultiPolygon &mp) for(auto const &p: mp) { geometry::correct(p, result, 1E-12); } - mp = result; + mp = std::move(result); } // --------------- @@ -166,33 +169,106 @@ namespace { const double kPrecheckEps = 1e-9; const std::size_t kPrecheckMaxWork = 1 << 26; // ring vertices walked by point-in-ring tests const std::size_t kPrecheckRetained = 1 << 16; +const double kPrecheckGridLoad = 0.25; // segments per grid cell +const std::size_t kPrecheckGridMin = 512; // segments for the grid (fewer: sweep), when the chunks overlap too much +const std::size_t kPrecheckGridEntries = 4; // grid entries per segment at most +const std::size_t kPrecheckAllPairs = 32; // up to this many segments, all pairs are tested +const std::size_t kPrecheckDirectHoles = 12; // up to this many holes, rays are tested against all segments +const std::size_t kPrecheckSweepHoles = 1; // more holes: their first vertices are located in one pass +const std::size_t kPrecheckChunkMin = 33; // segments for the chunks (fewer: all pairs) +const std::size_t kPrecheckChunk = 16; // consecutive segments per chunk +const std::size_t kPrecheckChunkPairs = 8; // overlapping chunk pairs per chunk at most (more: grid/sweep) +const std::size_t kPrecheckBridgeMin = 32; // points: up to this many, no search for clip bridges first +const std::size_t kPrecheckMonoMin = 4; // segments of a monotone chunk at least (shorter: up to kPrecheckChunk) struct PrecheckSeg { + PrecheckSeg() {} // (not zeroed when the vector grows: ringClear writes every field) double minX, maxX, minY, maxY; const Point *a; // segment a[0] -> a[1] - uint32_t ring, idx, count; // ring number, segment index, segments in the ring + uint32_t ring, idx; // ring number, segment index in the ring | kPrecheckLastSeg for the ring's last one }; +const uint32_t kPrecheckLastSeg = 1u << 31; struct PrecheckBox { double minX, maxX, minY, maxY; uint32_t id; // ring or polygon number }; +struct PrecheckSegRange { + const PrecheckSeg *b, *e; + std::size_t size() const { return std::size_t(e - b); } + const PrecheckSeg &operator[](std::size_t i) const { return b[i]; } + const PrecheckSeg *begin() const { return b; } + const PrecheckSeg *end() const { return e; } +}; +struct PrecheckSegBox { + double minX, maxXTol, minY, maxYTol; // maxXTol = maxX + tol, maxYTol = maxY + tol + uint32_t idx; // segment (sweep) + uint32_t cx0, cx1, cy0, cy1; // grid cells covered (grid) +}; +struct PrecheckChunk { + double minX, maxXTol, minY, maxYTol; // maxXTol = maxX + tol, maxYTol = maxY + tol + uint32_t b, e; // segments [b, e), of one ring + bool monotone; // no pair within the chunk needs testing +}; +struct PrecheckQuery { + double y; + uint32_t hole; +}; +struct PrecheckEdgeSeg { + uint32_t edge; // which line of the clip box the segment lies on + double lo, hi; // its extent along that line + uint32_t ring, idx; // ring number in the polygon, position among the ring's segments of nonzero length +}; struct PrecheckScratch { std::vector segs; - std::vector boxes, holeBoxes; + std::vector segBoxes; + std::vector tols; + std::vector polyStart; + std::vector bucketStart; + std::vector boxes; std::vector> pairs; + std::vector bandStart, bandFill, bandSegs, crossed; + std::vector queries; + std::vector unsure; + std::vector chunks; + std::vector> chunkPairs; + std::vector edgeSegs; void trim() { + // Every vector is trimmed after each call. With few segments, only the vectors whose + // size is not bounded by about 4x the segments (pairs of polygons) can have + // grown past kPrecheckRetained, so the others need no test. + if (segs.capacity() <= kPrecheckRetained / 8 && pairs.capacity() <= kPrecheckRetained && + edgeSegs.capacity() <= kPrecheckRetained && crossed.capacity() <= kPrecheckRetained) return; + if (chunks.capacity() > kPrecheckRetained) std::vector().swap(chunks); + if (edgeSegs.capacity() > kPrecheckRetained) std::vector().swap(edgeSegs); + if (chunkPairs.capacity() > kPrecheckRetained) std::vector>().swap(chunkPairs); + if (queries.capacity() > kPrecheckRetained) std::vector().swap(queries); + if (unsure.capacity() > kPrecheckRetained) std::vector().swap(unsure); if (segs.capacity() > kPrecheckRetained) std::vector().swap(segs); + if (segBoxes.capacity() > kPrecheckRetained) std::vector().swap(segBoxes); + if (tols.capacity() > kPrecheckRetained) std::vector().swap(tols); + if (polyStart.capacity() > kPrecheckRetained) std::vector().swap(polyStart); + if (bucketStart.capacity() > kPrecheckRetained) std::vector().swap(bucketStart); if (boxes.capacity() > kPrecheckRetained) std::vector().swap(boxes); - if (holeBoxes.capacity() > kPrecheckRetained) std::vector().swap(holeBoxes); if (pairs.capacity() > kPrecheckRetained) std::vector>().swap(pairs); + if (bandStart.capacity() > kPrecheckRetained) std::vector().swap(bandStart); + if (bandFill.capacity() > kPrecheckRetained) std::vector().swap(bandFill); + if (bandSegs.capacity() > kPrecheckRetained) std::vector().swap(bandSegs); + if (crossed.capacity() > kPrecheckRetained) std::vector().swap(crossed); } }; thread_local PrecheckScratch precheckScratch; inline double max3(double a, double b, double c) { return std::max(a, std::max(b, c)); } +// clearOrient runs in every hot loop; GCC does not inline it on its own +#if defined(__GNUC__) +#define PRECHECK_ALWAYS_INLINE __attribute__((always_inline)) inline +#else +#define PRECHECK_ALWAYS_INLINE inline +#endif + // Orientation of c relative to a->b: +1 counter-clockwise, -1 clockwise, 0 when too close to call. -inline int clearOrient(const Point &a, const Point &b, const Point &c, double tol) { +PRECHECK_ALWAYS_INLINE int clearOrient(const Point &a, const Point &b, const Point &c, double tol) { const double dx = b.x() - a.x(), dy = b.y() - a.y(); const double px = c.x() - a.x(), py = c.y() - a.y(); const double qx = c.x() - b.x(), qy = c.y() - b.y(); @@ -207,77 +283,149 @@ inline int clearOrient(const Point &a, const Point &b, const Point &c, double to return 0; } -// Coordinates finite; accumulates the largest magnitude. -inline bool ringFinite(const Ring &r, double &maxAbs) { +// Set by multiPolygonDefinitelyValidAfterClip: segment pairs are decided as Boost decides sides +thread_local bool precheckBoostSides = false; +thread_local bool precheckUsedBoostSides = false; + +// Set by the ...OrReversed entry points: the orientation every ring must have. 0 = as Boost +// expects (the plain precheck), 2 = not decided yet (the first ring decides), 1 = as Boost +// expects, -1 = reversed (every ring clearly has the opposite orientation). +thread_local int precheckOrient = 0; + +// Side of c relative to a->b as Boost's side_by_triangle decides it: the sign of the +// determinant unless it is within rounding error or Boost's zero band (0 = can't tell). +inline int boostSide(const Point &a, const Point &b, const Point &c) { + const double dx = b.x() - a.x(), dy = b.y() - a.y(); + const double px = c.x() - a.x(), py = c.y() - a.y(); + const double qx = c.x() - b.x(), qy = c.y() - b.y(); + const double t1 = dx * py, t2 = dy * px; + const double d = t1 - t2; + const double f = std::max(max3(1.0, std::fabs(dx), std::fabs(dy)), + std::max(max3(std::fabs(px), std::fabs(py), std::fabs(qx)), std::fabs(qy))); + const double bound = std::max(8 * std::numeric_limits::epsilon() * (std::fabs(t1) + std::fabs(t2)), + 64 * std::numeric_limits::epsilon() * f * f); + if (std::fabs(d) > bound) return d > 0 ? 1 : -1; + return 0; +} + +// Coordinates finite; accumulates the largest magnitude (maxAbs is only meaningful when true: +// over finite values, the maximum does not depend on the order it is taken in). +PRECHECK_ALWAYS_INLINE bool ringFinite(const Ring &r, double &maxAbs) { + const std::size_t n = r.size(); +#ifdef __SSE2__ + static_assert(sizeof(Point) == 2 * sizeof(double), "Point must be two packed doubles"); + const double *d = reinterpret_cast(r.data()); + const __m128d absMask = _mm_castsi128_pd(_mm_set1_epi64x(0x7fffffffffffffffLL)); + const __m128d big = _mm_set1_pd(std::numeric_limits::max()); + __m128d mx0 = _mm_set1_pd(maxAbs), mx1 = mx0; + __m128d ok0 = _mm_castsi128_pd(_mm_set1_epi32(-1)), ok1 = ok0; + std::size_t i = 0; + for (; i + 2 <= n; i += 2) { + const __m128d v0 = _mm_and_pd(_mm_loadu_pd(d + 2 * i), absMask); + const __m128d v1 = _mm_and_pd(_mm_loadu_pd(d + 2 * i + 2), absMask); + ok0 = _mm_and_pd(ok0, _mm_cmple_pd(v0, big)); // false for inf and NaN + ok1 = _mm_and_pd(ok1, _mm_cmple_pd(v1, big)); + mx0 = _mm_max_pd(mx0, v0); mx1 = _mm_max_pd(mx1, v1); + } + if (i < n) { + const __m128d v0 = _mm_and_pd(_mm_loadu_pd(d + 2 * i), absMask); + ok0 = _mm_and_pd(ok0, _mm_cmple_pd(v0, big)); + mx0 = _mm_max_pd(mx0, v0); + } + if (_mm_movemask_pd(_mm_and_pd(ok0, ok1)) != 3) return false; + mx0 = _mm_max_pd(mx0, mx1); + maxAbs = std::max(_mm_cvtsd_f64(mx0), _mm_cvtsd_f64(_mm_unpackhi_pd(mx0, mx0))); + return true; +#else for (const Point &p : r) { const double x = p.x(), y = p.y(); if (!std::isfinite(x) || !std::isfinite(y)) return false; maxAbs = max3(maxAbs, std::fabs(x), std::fabs(y)); } return true; +#endif +} + +// No spike at b between segments a->b and b->c: they only share b unless they fold back. +PRECHECK_ALWAYS_INLINE bool vertexClear(const Point &a, const Point &b, const Point &c, double tol) { + if (clearOrient(a, b, c, tol) != 0) return true; + // (nearly) collinear: must clearly continue forward + const double t1 = (b.x() - a.x()) * (c.x() - b.x()); + const double t2 = (b.y() - a.y()) * (c.y() - b.y()); + if (t1 + t2 > kPrecheckEps * (std::fabs(t1) + std::fabs(t2))) return true; + // Folding back at b (a needle): no spike for Boost as long as it can tell c's side of a->b + if (!precheckBoostSides || boostSide(a, b, c) == 0) return false; + precheckUsedBoostSides = true; + return true; +} + +// The area of a ring (doubled, from ringClear's sums over its n points) must clearly exceed both +// our rounding error and that of Boost's sum, with the orientation wanted (see precheckOrient). +PRECHECK_ALWAYS_INLINE bool ringAreaClear(double area, double areaScale, double boostScale, std::size_t n, bool interior) { + const double relEps = std::max(kPrecheckEps, 8.0 * (n + 4) * std::numeric_limits::epsilon()); + if (!(std::fabs(area) > relEps * areaScale && std::fabs(area) > relEps * boostScale)) return false; + const bool clockwise = boost::geometry::point_order::value == boost::geometry::clockwise; + const bool wantPositive = clockwise == interior; // clockwise polygon: CW outer, CCW inners + const bool reversed = wantPositive ? area <= 0 : area >= 0; // area is not 0 here + if (precheckOrient == 0) { + if (reversed) return false; + } else { + const int o = reversed ? -1 : 1; + if (precheckOrient == 2) precheckOrient = o; + else if (precheckOrient != o) return false; + } + return true; } // Single-ring tests: closed, >= 4 points, consecutive points either exactly // equal (Boost allows duplicate points; such zero-length segments are skipped) // or clearly distinct, area clearly non-zero with the orientation Boost -// expects, no spikes. Appends the ring's segments and returns its bbox. -bool ringClear(const Ring &r, bool interior, double tol, uint32_t ringIdx, std::vector &segs, PrecheckBox &box) { +// expects, no spikes. Appends the ring's segments (the caller has reserved a +// segment per point). +template +bool ringClear(const Ring &r, bool interior, double tol, uint32_t ringIdx, Segs &segs) { const std::size_t n = r.size(); if (n < 4 || n >= std::numeric_limits::max()) return false; const std::size_t m = n - 1; // segments: i runs r[i] -> r[i+1] if (r[0].x() != r[m].x() || r[0].y() != r[m].y()) return false; + // (segment indexes below kPrecheckLastSeg: rings of 2^31 points or more are left to Boost) + if (n > kPrecheckLastSeg) return false; const std::size_t first = segs.size(); - box.minX = box.minY = std::numeric_limits::infinity(); - box.maxX = box.maxY = -std::numeric_limits::infinity(); + segs.resize(first + m); // (callers reserve a segment per point: no reallocation) + PrecheckSeg *out = segs.data() + first; + uint32_t count = 0; + // Signed area (doubled) relative to r[0], positive = counter-clockwise, and the scales of its + // rounding error and of Boost's (x1+x2)*(y1-y2) sum, accumulated in the same pass (each sum + // in the order of i, as separate loops would) + const double x0 = r[0].x(), y0 = r[0].y(); + double area = 0, areaScale = 0, boostScale = 0; + const Point *prev = nullptr; // start of the previous segment for (std::size_t i = 0; i < m; i++) { const Point &a = r[i], &b = r[i + 1]; + boostScale += (std::fabs(a.x()) + std::fabs(b.x())) * std::fabs(a.y() - b.y()); + if (i >= 1 && i + 1 < m) { + const double t1 = (a.x() - x0) * (b.y() - y0); + const double t2 = (a.y() - y0) * (b.x() - x0); + area += t1 - t2; + areaScale += std::fabs(t1) + std::fabs(t2); + } if (a.x() == b.x() && a.y() == b.y()) continue; if (std::fabs(b.x() - a.x()) <= tol && std::fabs(b.y() - a.y()) <= tol) return false; - PrecheckSeg s; + PrecheckSeg &s = out[count]; s.minX = std::min(a.x(), b.x()); s.maxX = std::max(a.x(), b.x()); s.minY = std::min(a.y(), b.y()); s.maxY = std::max(a.y(), b.y()); s.a = &a; - s.ring = ringIdx; s.idx = static_cast(segs.size() - first); - segs.push_back(s); - box.minX = std::min(box.minX, s.minX); box.maxX = std::max(box.maxX, s.maxX); - box.minY = std::min(box.minY, s.minY); box.maxY = std::max(box.maxY, s.maxY); + s.ring = ringIdx; s.idx = count++; + // the vertex between the previous segment and this one (the first segment's start: below) + if (prev && !vertexClear(*prev, a, b, tol)) return false; + prev = &a; } - const std::size_t count = segs.size() - first; + segs.resize(first + count); if (count < 3) return false; - for (std::size_t k = first; k < segs.size(); k++) segs[k].count = static_cast(count); - - // Signed area (doubled) relative to r[0], positive = counter-clockwise. It must - // clearly exceed both our rounding error and that of Boost's (x1+x2)*(y1-y2) sum. - const double x0 = r[0].x(), y0 = r[0].y(); - double area = 0, areaScale = 0, boostScale = 0; - for (std::size_t i = 1; i + 1 < m; i++) { - const double t1 = (r[i].x() - x0) * (r[i + 1].y() - y0); - const double t2 = (r[i].y() - y0) * (r[i + 1].x() - x0); - area += t1 - t2; - areaScale += std::fabs(t1) + std::fabs(t2); - } - for (std::size_t i = 0; i < m; i++) - boostScale += (std::fabs(r[i].x()) + std::fabs(r[i + 1].x())) * std::fabs(r[i].y() - r[i + 1].y()); - const double relEps = std::max(kPrecheckEps, 8.0 * (n + 4) * std::numeric_limits::epsilon()); - if (!(std::fabs(area) > relEps * areaScale && std::fabs(area) > relEps * boostScale)) return false; - const bool clockwise = boost::geometry::point_order::value == boost::geometry::clockwise; - const bool wantPositive = clockwise == interior; // clockwise polygon: CW outer, CCW inners - if (wantPositive ? area <= 0 : area >= 0) return false; - - // Consecutive segments a->b, b->c only share b unless they fold back (spike). - for (std::size_t k = 0; k < count; k++) { - const PrecheckSeg &prev = segs[first + (k == 0 ? count - 1 : k - 1)]; - const Point &a = prev.a[0]; - const Point &b = segs[first + k].a[0]; - const Point &c = segs[first + k].a[1]; - if (clearOrient(a, b, c, tol) != 0) continue; - // (nearly) collinear: must clearly continue forward - const double t1 = (b.x() - a.x()) * (c.x() - b.x()); - const double t2 = (b.y() - a.y()) * (c.y() - b.y()); - if (!(t1 + t2 > kPrecheckEps * (std::fabs(t1) + std::fabs(t2)))) return false; - } - return true; + if (!vertexClear(*prev, out[0].a[0], out[0].a[1], tol)) return false; + out[count - 1].idx |= kPrecheckLastSeg; + return ringAreaClear(area, areaScale, boostScale, n, interior); } inline bool boxesApart(const PrecheckBox &a, const PrecheckBox &b, double tol) { @@ -297,54 +445,289 @@ bool forOverlappingBoxes(std::vector &boxes, double tol, F f) { return true; } +// The side tests of segsClear: segments p1->p2 and q1->q2 are clearly apart. +PRECHECK_ALWAYS_INLINE bool sidesClear(const Point &p1, const Point &p2, const Point &q1, const Point &q2, double tol) { + const int o1 = clearOrient(p1, p2, q1, tol), o2 = clearOrient(p1, p2, q2, tol); + if (o1 != 0 && o1 == o2) return true; // q entirely on one side of line p + const int o3 = clearOrient(q1, q2, p1, tol), o4 = clearOrient(q1, q2, p2, tol); + if (o3 != 0 && o3 == o4) return true; // p entirely on one side of line q + if (!precheckBoostSides) return false; // crossing, touching or unsure + // Apart as Boost would find them: one segment clearly on one side of the other's line + const int b1 = boostSide(p1, p2, q1), b2 = boostSide(p1, p2, q2); + const int b3 = boostSide(q1, q2, p1), b4 = boostSide(q1, q2, p2); + if ((b1 != 0 && b1 == b2) || (b3 != 0 && b3 == b4)) { precheckUsedBoostSides = true; return true; } + return false; +} + // Two segments are clearly disjoint (adjacent segments of a ring are handled by the spike test). inline bool segsClear(const PrecheckSeg &s, const PrecheckSeg &t, double tol) { if (s.ring == t.ring) { - const uint32_t lo = std::min(s.idx, t.idx), hi = std::max(s.idx, t.idx); - if (hi - lo == 1 || (lo == 0 && hi == s.count - 1)) return true; + // adjacent: consecutive, or the first and the last segment of the ring + const uint32_t si = s.idx & ~kPrecheckLastSeg, ti = t.idx & ~kPrecheckLastSeg; + const uint32_t lo = std::min(si, ti), hi = std::max(si, ti); + if (hi - lo == 1 || (lo == 0 && ((s.idx | t.idx) & kPrecheckLastSeg))) return true; } if (s.maxX + tol < t.minX || t.maxX + tol < s.minX || s.maxY + tol < t.minY || t.maxY + tol < s.minY) return true; - const Point &p1 = s.a[0], &p2 = s.a[1], &q1 = t.a[0], &q2 = t.a[1]; - const int o1 = clearOrient(p1, p2, q1, tol), o2 = clearOrient(p1, p2, q2, tol); - if (o1 != 0 && o1 == o2) return true; // q entirely on one side of line p - const int o3 = clearOrient(q1, q2, p1, tol), o4 = clearOrient(q1, q2, p2, tol); - return o3 != 0 && o3 == o4; // p entirely on one side of line q; else crossing, touching or unsure + return sidesClear(s.a[0], s.a[1], t.a[0], t.a[1], tol); } -// All pairs of segments clearly disjoint: sweep over segments sorted by minX. -// The answer does not depend on the order (every x-overlapping pair is tested). -bool segmentsClear(std::vector &segs, double tol) { +// Pairs whose boxes are clearly apart are clear whatever else segsClear tests (its box test +// comes before anything that could fail), so they can be skipped by that box test first. +inline bool boxesClearlyApart(const PrecheckSeg &s, const PrecheckSeg &t, double tol) { + return (s.maxX + tol < t.minX) | (t.maxX + tol < s.minX) | (s.maxY + tol < t.minY) | (t.maxY + tol < s.minY); +} + +// Uniform grid over the segments' boxes: each segment is entered in every cell its box +// [minX, maxX + tol] x [minY, maxY + tol] covers (cellX/cellY are monotone, so two boxes +// that are not clearly apart share a cell), and a pair is tested only in the cell at the low +// corner of the overlap of their cell ranges, so exactly once. 1 = all clear, 0 = not, +// -1 = the grid would get too big (long segments), use the sweep. +int segmentsClearGrid(const std::vector &segs, PrecheckScratch &scratch, double tol) { const std::size_t n = segs.size(); - if (n <= 32) { - for (std::size_t i = 0; i < n; i++) - for (std::size_t j = i + 1; j < n; j++) - if (!segsClear(segs[i], segs[j], tol)) return false; - return true; + std::vector &gb = scratch.segBoxes; + gb.resize(n); + double x0 = std::numeric_limits::infinity(), x1 = -x0, y0 = x0, y1 = -x0; + for (std::size_t i = 0; i < n; i++) { + const PrecheckSeg &s = segs[i]; + PrecheckSegBox &b = gb[i]; + b.minX = s.minX; b.maxXTol = s.maxX + tol; b.minY = s.minY; b.maxYTol = s.maxY + tol; + x0 = std::min(x0, b.minX); x1 = std::max(x1, b.maxXTol); + y0 = std::min(y0, b.minY); y1 = std::max(y1, b.maxYTol); } - std::sort(segs.begin(), segs.end(), [](const PrecheckSeg &a, const PrecheckSeg &b) { return a.minX < b.minX; }); + // about kPrecheckGridLoad segments per cell, cells roughly square + const double w = x1 - x0, h = y1 - y0; + const double cells = std::max(1.0, double(n) / kPrecheckGridLoad); + double fx = w > 0 && h > 0 ? std::sqrt(cells * w / h) : (w > 0 ? cells : 1.0); + fx = std::min(std::max(fx, 1.0), cells); + const std::size_t nx = static_cast(fx); + const std::size_t ny = std::max(1, static_cast(cells / double(nx))); + const double sx = w > 0 ? double(nx) / w : 0.0, sy = h > 0 ? double(ny) / h : 0.0; + const double lastX = double(nx - 1), lastY = double(ny - 1); + auto cellX = [&](double x) { + const double c = (x - x0) * sx; + return c >= lastX ? static_cast(nx - 1) : c <= 0 ? 0u : static_cast(c); + }; + auto cellY = [&](double y) { + const double c = (y - y0) * sy; + return c >= lastY ? static_cast(ny - 1) : c <= 0 ? 0u : static_cast(c); + }; + std::size_t entries = 0; + for (PrecheckSegBox &b : gb) { + b.cx0 = cellX(b.minX); b.cy0 = cellY(b.minY); + b.cx1 = cellX(b.maxXTol); b.cy1 = cellY(b.maxYTol); + entries += std::size_t(b.cx1 - b.cx0 + 1) * (b.cy1 - b.cy0 + 1); + } + if (entries > kPrecheckGridEntries * n) return -1; + + const std::size_t numCells = nx * ny; + std::vector &start = scratch.bucketStart, &fill = scratch.bandFill, &items = scratch.bandSegs; + start.assign(numCells + 1, 0); + for (const PrecheckSegBox &b : gb) + for (uint32_t cy = b.cy0; cy <= b.cy1; cy++) + for (uint32_t cx = b.cx0; cx <= b.cx1; cx++) start[cy * nx + cx + 1]++; + for (std::size_t c = 0; c < numCells; c++) start[c + 1] += start[c]; + fill.assign(start.begin(), start.end() - 1); + items.resize(entries); for (std::size_t i = 0; i < n; i++) { - const double limit = segs[i].maxX + tol; - for (std::size_t j = i + 1; j < n && segs[j].minX <= limit; j++) - if (!segsClear(segs[i], segs[j], tol)) return false; + const PrecheckSegBox &b = gb[i]; + for (uint32_t cy = b.cy0; cy <= b.cy1; cy++) + for (uint32_t cx = b.cx0; cx <= b.cx1; cx++) items[fill[cy * nx + cx]++] = static_cast(i); + } + for (std::size_t cy = 0; cy < ny; cy++) + for (std::size_t cx = 0; cx < nx; cx++) { + const std::size_t c = cy * nx + cx; + for (uint32_t a = start[c]; a < start[c + 1]; a++) { + const PrecheckSegBox bi = gb[items[a]]; + for (uint32_t k = a + 1; k < start[c + 1]; k++) { + const PrecheckSegBox &bj = gb[items[k]]; + // the box test of segsClear: s.maxX + tol < t.minX || t.maxX + tol < s.minX || (same in y) + if ((bi.maxXTol < bj.minX) | (bj.maxXTol < bi.minX) | (bi.maxYTol < bj.minY) | (bj.maxYTol < bi.minY)) continue; + if (std::max(bi.cx0, bj.cx0) != cx || std::max(bi.cy0, bj.cy0) != cy) continue; // tested in another cell + if (!segsClear(segs[items[a]], segs[items[k]], tol)) return 0; + } + } + } + return 1; +} + +// Chunks of consecutive segments (rings are spatially coherent, so a chunk's box is small): +// a pair of segments from two chunks whose boxes are clearly apart is clearly apart too +// (s.maxX <= A.maxX and t.minX >= B.minX, and adding tol is monotone), so only the pairs +// within a chunk and between chunks whose boxes are not clearly apart are tested. 1 = all +// clear, 0 = not, -1 = the chunks overlap too much (long or folded segments), use the grid/sweep. +int segmentsClearChunked(const std::vector &segs, PrecheckScratch &scratch, double tol) { + const std::size_t n = segs.size(); + const std::size_t K = kPrecheckChunk; + // up to K consecutive segments of one ring per chunk (a chunk across two rings would be large) + std::vector &cb = scratch.chunks; + cb.clear(); + double sumW = 0, sumH = 0, allX0 = std::numeric_limits::infinity(), allX1 = -allX0, allY0 = allX0, allY1 = -allX0; + for (std::size_t b = 0; b < n; ) { + std::size_t e = b + 1; + double x0 = segs[b].minX, x1 = segs[b].maxX, y0 = segs[b].minY, y1 = segs[b].maxY; + // Monotone chunk: the chunk's points p0, p1, ... (segment k runs p_k -> p_k+1) increase in x, + // p_k + tol < p_k+1 for every segment but the first, which only must not decrease. Then + // every pair of segments i < j - 1 is clearly apart in x as the box test computes it: + // s.maxX + tol = p_i+1 + tol < p_i+2 <= p_j = t.minX (rounding is monotone). Likewise + // decreasing (t.maxX + tol <= p_i+2 + tol < p_i+1 = s.minX) and in y. + const Point *pb = segs[b].a; + unsigned mono = (pb[0].x() <= pb[1].x() ? 1u : 0u) | (pb[0].x() >= pb[1].x() ? 2u : 0u) + | (pb[0].y() <= pb[1].y() ? 4u : 0u) | (pb[0].y() >= pb[1].y() ? 8u : 0u); + while (e < n && e - b < K && segs[e].ring == segs[b].ring) { + const PrecheckSeg &s = segs[e]; + const bool wideX = s.minX + tol < s.maxX, wideY = s.minY + tol < s.maxY; + const bool incX = s.a[0].x() < s.a[1].x(), incY = s.a[0].y() < s.a[1].y(); + const unsigned next = mono & ((wideX && incX ? 1u : 0u) | (wideX && !incX ? 2u : 0u) | (wideY && incY ? 4u : 0u) | (wideY && !incY ? 8u : 0u)); + // a monotone chunk of a few segments ends where it would stop being monotone + if (!next && mono && e - b >= kPrecheckMonoMin) break; + mono = next; + x0 = std::min(x0, s.minX); x1 = std::max(x1, s.maxX); + y0 = std::min(y0, s.minY); y1 = std::max(y1, s.maxY); + e++; + } + cb.push_back({ x0, x1 + tol, y0, y1 + tol, static_cast(b), static_cast(e), mono != 0 }); + sumW += x1 - x0; sumH += y1 - y0; + allX0 = std::min(allX0, x0); allX1 = std::max(allX1, x1); allY0 = std::min(allY0, y0); allY1 = std::max(allY1, y1); + b = e; + } + const std::size_t m = cb.size(); + // Sweep along the axis in which the chunks are narrower relative to the whole extent + // (fewer chunks in each sweep window). Which pairs are found does not depend on it. + const bool sweepY = sumH * (allX1 - allX0) < sumW * (allY1 - allY0); + if (sweepY) std::sort(cb.begin(), cb.end(), [](const PrecheckChunk &a, const PrecheckChunk &b) { return a.minY < b.minY; }); + else std::sort(cb.begin(), cb.end(), [](const PrecheckChunk &a, const PrecheckChunk &b) { return a.minX < b.minX; }); + // Overlapping chunk pairs first (positions in cb): bail out before testing anything if there are too many + std::vector> &pairs = scratch.chunkPairs; + pairs.clear(); + const std::size_t maxPairs = kPrecheckChunkPairs * m; + for (std::size_t i = 0; i < m; i++) { + const PrecheckChunk bi = cb[i]; + const double limit = sweepY ? bi.maxYTol : bi.maxXTol; + for (std::size_t j = i + 1; j < m && (sweepY ? cb[j].minY : cb[j].minX) <= limit; j++) { + const PrecheckChunk &bj = cb[j]; + if ((bi.maxXTol < bj.minX) | (bj.maxXTol < bi.minX) | (bi.maxYTol < bj.minY) | (bj.maxYTol < bi.minY)) continue; + pairs.emplace_back(static_cast(i), static_cast(j)); + if (pairs.size() > maxPairs) return -1; + } + } + // pairs within each chunk; consecutive segments of a ring are adjacent, which segsClear + // accepts before anything else + for (const PrecheckChunk &c : cb) { + if (c.monotone) continue; + for (std::size_t i = c.b; i < c.e; i++) + for (std::size_t j = i + 2; j < c.e; j++) + if (!boxesClearlyApart(segs[i], segs[j], tol) && !segsClear(segs[i], segs[j], tol)) return 0; + } + // pairs between overlapping chunks: only segments not clearly apart from the other chunk's box + // can be in a pair that is not clearly apart (same monotonicity argument) + uint32_t la[kPrecheckChunk], lb[kPrecheckChunk]; + for (const auto &pr : pairs) { + const PrecheckChunk &A = cb[pr.first], &B = cb[pr.second]; + std::size_t na = 0, nb = 0; + for (std::size_t i = A.b; i < A.e; i++) { + const PrecheckSeg &s = segs[i]; + // s against B: s.maxX + tol < B.minX || B.maxX + tol < s.minX || (same in y) + if (!((s.maxX + tol < B.minX) | (B.maxXTol < s.minX) | (s.maxY + tol < B.minY) | (B.maxYTol < s.minY))) la[na++] = static_cast(i); + } + if (!na) continue; + for (std::size_t j = B.b; j < B.e; j++) { + const PrecheckSeg &t = segs[j]; + if (!((t.maxX + tol < A.minX) | (A.maxXTol < t.minX) | (t.maxY + tol < A.minY) | (A.maxYTol < t.minY))) lb[nb++] = static_cast(j); + } + for (std::size_t a = 0; a < na; a++) + for (std::size_t b = 0; b < nb; b++) + if (!boxesClearlyApart(segs[la[a]], segs[lb[b]], tol) && !segsClear(segs[la[a]], segs[lb[b]], tol)) return 0; + } + return 1; +} + +// All pairs of segments clearly disjoint. segsClear is symmetric and true for every pair whose +// boxes are clearly apart, so the answer only needs every other pair to be tested, in any order: +// all pairs for a few segments, chunks of consecutive segments for many, a grid or else a sweep in x. +template +inline bool allPairsClear(const Segs &segs, double tol) { + const std::size_t n = segs.size(); + for (std::size_t i = 0; i < n; i++) + for (std::size_t j = i + 1; j < n; j++) + if (!boxesClearlyApart(segs[i], segs[j], tol) && !segsClear(segs[i], segs[j], tol)) return false; + return true; +} + +bool segmentsClear(std::vector &segs, PrecheckScratch &scratch, double tol) { + const std::size_t n = segs.size(); + if (n <= kPrecheckAllPairs) return allPairsClear(segs, tol); + if (n >= kPrecheckChunkMin && n < (std::size_t(1) << 30)) { + const int c = segmentsClearChunked(segs, scratch, tol); + if (c >= 0) return c == 1; + } + if (n >= kPrecheckGridMin && n < (std::size_t(1) << 30)) { + const int g = segmentsClearGrid(segs, scratch, tol); + if (g >= 0) return g == 1; + } + // Sweep in x without a full sort: the segments are bucketed by minX (counting sort into + // n buckets; bucketOf is monotone in x). Each segment i is paired with the segments after it + // in bucket order up to the bucket of its maxX + tol: a segment j that is not clearly apart + // from i in x has minX[j] <= maxX[i] + tol, so it lies in such a bucket, or else i lies in + // one of j's. Every pair whose boxes are not clearly apart is thus tested once. + double minX = std::numeric_limits::infinity(), maxX = -minX; + for (const PrecheckSeg &s : segs) { minX = std::min(minX, s.minX); maxX = std::max(maxX, s.minX); } + const double scale = maxX > minX ? double(n) / (maxX - minX) : 0.0; + const double lastBucket = double(n - 1); + auto bucketOf = [&](double x) { + const double b = (x - minX) * scale; + return b >= lastBucket ? n - 1 : b <= 0 ? std::size_t(0) : static_cast(b); + }; + std::vector &start = scratch.bucketStart; + start.assign(n + 1, 0); + for (const PrecheckSeg &s : segs) start[bucketOf(s.minX) + 1]++; + for (std::size_t b = 0; b < n; b++) start[b + 1] += start[b]; + std::vector &sb = scratch.segBoxes; + sb.resize(n); + { + std::vector &fill = scratch.bandFill; + fill.assign(start.begin(), start.end() - 1); + for (std::size_t i = 0; i < n; i++) { + const PrecheckSeg &s = segs[i]; + PrecheckSegBox &b = sb[fill[bucketOf(s.minX)]++]; + b.minX = s.minX; b.maxXTol = s.maxX + tol; b.minY = s.minY; b.maxYTol = s.maxY + tol; + b.idx = static_cast(i); + } + } + for (std::size_t i = 0; i < n; i++) { + const PrecheckSegBox &bi = sb[i]; + const std::size_t end = start[bucketOf(bi.maxXTol) + 1]; + for (std::size_t j = i + 1; j < end; j++) { + const PrecheckSegBox &bj = sb[j]; + // the box test of segsClear: s.maxX + tol < t.minX || t.maxX + tol < s.minX || (same in y) + if ((bi.maxXTol < bj.minX) | (bj.maxXTol < bi.minX) | (bi.maxYTol < bj.minY) | (bj.maxYTol < bi.minY)) continue; + if (!segsClear(segs[bi.idx], segs[bj.idx], tol)) return false; + } } return true; } +// Whether the ray from h to +x crosses segment a->b: 1 = yes, -1 = no, 0 = unsure. +// Vertices are classed as above (y > hy) or not with exact comparisons (the +// half-open rule), so a ray through a vertex is counted consistently and the +// crossing parity is exact; only the side of h is decided with a tolerance. +inline int rayClearlyCrosses(const Point &h, const Point &a, const Point &b, double tol) { + const double hx = h.x(), hy = h.y(); + if ((a.y() > hy) == (b.y() > hy)) return -1; + if (std::max(a.x(), b.x()) < hx) return -1; + if (std::min(a.x(), b.x()) > hx) return 1; + const int o = clearOrient(a, b, h, tol); + if (o == 0) return 0; + return (b.y() > a.y()) == (o > 0) ? 1 : -1; +} + // Crossing-number test: 1 = h clearly inside r, -1 = clearly outside, 0 = unsure. int pointClearlyInRing(const Point &h, const Ring &r, double tol) { bool inside = false; - const double hx = h.x(), hy = h.y(); for (std::size_t i = 0; i + 1 < r.size(); i++) { - const Point &a = r[i], &b = r[i + 1]; - if ((a.y() > hy + tol && b.y() > hy + tol) || (a.y() < hy - tol && b.y() < hy - tol)) continue; - if (std::fabs(a.y() - hy) <= tol || std::fabs(b.y() - hy) <= tol) return 0; - // a and b are clearly on opposite sides of the horizontal line through h - if (std::max(a.x(), b.x()) < hx - tol) continue; - if (std::min(a.x(), b.x()) > hx + tol) { inside = !inside; continue; } - const int o = clearOrient(a, b, h, tol); - if (o == 0) return 0; - if ((b.y() > a.y()) == (o > 0)) inside = !inside; // the ray to +x crosses a->b + const int c = rayClearlyCrosses(h, r[i], r[i + 1], tol); + if (c == 0) return 0; + if (c > 0) inside = !inside; } return inside ? 1 : -1; } @@ -360,12 +743,230 @@ int ringClearlyInRing(const Ring &inner, const Ring &r, double tol) { return 0; } -bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outerBox) { +// Placement test of the holes of p, given forEach(h, f): calls f(segment) for (at least) every +// segment of p whose y range reaches h.y within tol, stopping when f returns false. A ray +// from h never crosses the other segments (rayClearlyCrosses says -1 for them), so the +// answer does not depend on which of those are passed, nor on the order. +template +bool holesPlacedWith(const Polygon &p, uint32_t ringBase, std::vector &crossed, double tol, ForEach forEach) { + const auto &inners = p.inners(); + // 1 = h inside the outer ring and no other hole, -1 = not, 0 = unsure + auto locate = [&](const Point &h, uint32_t ownRing) { + crossed.clear(); + bool unsure = false; + const double hy = h.y(); + forEach(h, [&](const PrecheckSeg &s) { + // rayClearlyCrosses' first test (exactly one end above hy), on the segment's box + // (minY/maxY are its ends' y) without loading the points + if (!(s.minY <= hy && hy < s.maxY)) return true; + if (s.ring == ownRing) return true; + const int c = rayClearlyCrosses(h, s.a[0], s.a[1], tol); + if (c == 0) { unsure = true; return false; } + if (c > 0) crossed.push_back(s.ring); + return true; + }); + if (unsure) return 0; + std::sort(crossed.begin(), crossed.end()); + bool inOuter = false; + for (std::size_t i = 0; i < crossed.size(); ) { + std::size_t j = i; + while (j < crossed.size() && crossed[j] == crossed[i]) j++; + if ((j - i) % 2 == 1) { + if (crossed[i] != ringBase) return -1; // inside another hole + inOuter = true; + } + i = j; + } + return inOuter ? 1 : -1; + }; + + // A ring known not to touch the others lies wherever any of its vertices does + for (std::size_t i = 0; i < inners.size(); i++) { + const Ring &hole = inners[i]; + const std::size_t m = hole.size() - 1; + const std::size_t tries = std::min(m, 8); + int loc = 0; + for (std::size_t k = 0; k < tries && loc == 0; k++) + loc = locate(hole[k * m / tries], static_cast(ringBase + i + 1)); + if (loc != 1) return false; + } + return true; +} + +// Holes lie inside the outer ring (ring ringBase) and outside each other. `segs` holds the +// segments of all rings of p (numbered ringBase, ringBase + 1, ...), known not to touch. +// They are bucketed into horizontal bands, so that a ray from a vertex of each hole is +// tested against the segments of that vertex's band only, counting crossings per ring (odd = inside that ring). +// With few holes (few rays) the rays are tested against all segments instead. +// The first vertex of every hole located at once (what holesPlacedWith computes for its first +// try per hole): one pass over the segments, each paired with the holes whose first vertex +// lies in its half-open y range [minY, maxY), which are exactly the segments whose crossing +// rayClearlyCrosses decides (all others give -1). 1 = all holes placed, 0 = some hole is not, +// -1 = some first vertex is unsure (and none was found misplaced): run holesPlacedWith. +int holesFirstVerticesPlaced(const Polygon &p, PrecheckSegRange segs, uint32_t ringBase, PrecheckScratch &scratch, double tol) { + const auto &inners = p.inners(); + const std::size_t q = inners.size(); + std::vector &qs = scratch.queries; + qs.resize(q); + for (std::size_t i = 0; i < q; i++) qs[i] = { inners[i][0].y(), static_cast(i) }; + std::sort(qs.begin(), qs.end(), [](const PrecheckQuery &a, const PrecheckQuery &b) { return a.y < b.y; }); + // per hole: bit 0 unsure, bit 1 inside the outer ring (parity of its crossings), bit 2 parity of + // the crossings of all other holes together. That parity is odd exactly when the vertex lies in + // an odd number of other holes. Per ring parities are not needed: the holes don't touch, so the + // holes containing a vertex of hole X are nested in a chain H1 inside H2 ... inside Hk, all + // containing X. If k >= 2 is even, H1's first vertex lies in exactly H2 ... Hk, k - 1 (odd) + // holes. So when some hole lies in another hole, some hole's parity is odd: if all holes are + // sure, the answer (0) is as with per ring parities; if that hole is unsure, the answer is -1 + // instead of 0, and holesPlacedWith, which places X's first vertex just as here, says false. + // An odd parity always means that hole lies in another one (0 as before). + std::vector &unsure = scratch.unsure; + unsure.assign(q, 0); + std::size_t pos = 0; // first query with y >= minY of the current segment + for (const PrecheckSeg &s : segs) { + const double y0 = s.minY, y1 = s.maxY; + if (!(y0 < y1)) continue; // horizontal: no y is in [minY, maxY) + // walk from the previous segment's position (rings are coherent), binary search if far + int steps = 0; + while (pos > 0 && qs[pos - 1].y >= y0 && steps < 8) { pos--; steps++; } + while (pos < q && qs[pos].y < y0 && steps < 8) { pos++; steps++; } + if (steps == 8) + pos = std::lower_bound(qs.begin(), qs.end(), y0, [](const PrecheckQuery &a, double y) { return a.y < y; }) - qs.begin(); + for (std::size_t k = pos; k < q && qs[k].y < y1; k++) { + const uint32_t hole = qs[k].hole; + if (s.ring == ringBase + hole + 1) continue; + const int c = rayClearlyCrosses(inners[hole][0], s.a[0], s.a[1], tol); + if (c == 0) unsure[hole] |= 1; + else if (c > 0) unsure[hole] ^= s.ring == ringBase ? 2 : 4; + } + } + // per hole: inside the outer ring (odd crossings of ringBase) and in no other hole + bool anyUnsure = false; + for (uint32_t hole = 0; hole < q; hole++) { + if (unsure[hole] & 1) { anyUnsure = true; continue; } + if ((unsure[hole] & 6) != 2) return 0; + } + return anyUnsure ? -1 : 1; +} + +bool holesClearlyPlaced(const Polygon &p, PrecheckSegRange segs, uint32_t ringBase, PrecheckScratch &scratch, double tol) { + const auto &inners = p.inners(); + if (inners.empty()) return true; + if (segs.size() >= std::numeric_limits::max()) return false; + if (inners.size() > kPrecheckSweepHoles && segs.size() < (std::size_t(1) << 30)) { + const int r = holesFirstVerticesPlaced(p, segs, ringBase, scratch, tol); + if (r >= 0) return r == 1; + } + // (the band index below fails only with >= 2^32 entries, at most 4 per segment) + if (inners.size() <= kPrecheckDirectHoles && segs.size() < (std::size_t(1) << 30)) + return holesPlacedWith(p, ringBase, scratch.crossed, tol, [&](const Point &, auto f) { + for (const PrecheckSeg &s : segs) if (!f(s)) return; + }); + + double minY = std::numeric_limits::infinity(), maxY = -minY; + for (const PrecheckSeg &s : segs) { minY = std::min(minY, s.minY); maxY = std::max(maxY, s.maxY); } + minY -= tol; maxY += tol; + // About 4 segments per band; fewer bands while tall segments would make the index much bigger than segs + std::size_t bands = std::max(1, segs.size() / 4); + double scale = 0; + auto bandOf = [&](double y) { + const double b = (y - minY) * scale; + return b <= 0 ? std::size_t(0) : std::min(bands - 1, static_cast(b)); + }; + std::size_t entries; + for (;;) { + scale = bands / (maxY - minY); + entries = 0; + for (const PrecheckSeg &s : segs) entries += bandOf(s.maxY + tol) - bandOf(s.minY - tol) + 1; + if (entries <= 4 * segs.size() || bands == 1) break; + bands = (bands + 1) / 2; + } + if (entries >= std::numeric_limits::max()) return false; + + // Segments of band b: bandSegs[bandStart[b] .. bandStart[b+1]) + std::vector &start = scratch.bandStart, &fill = scratch.bandFill, &items = scratch.bandSegs; + start.assign(bands + 1, 0); + for (const PrecheckSeg &s : segs) + for (std::size_t b = bandOf(s.minY - tol), hi = bandOf(s.maxY + tol); b <= hi; b++) start[b + 1]++; + for (std::size_t b = 0; b < bands; b++) start[b + 1] += start[b]; + fill.assign(start.begin(), start.end() - 1); + items.resize(entries); + for (std::size_t k = 0; k < segs.size(); k++) + for (std::size_t b = bandOf(segs[k].minY - tol), hi = bandOf(segs[k].maxY + tol); b <= hi; b++) + items[fill[b]++] = static_cast(k); + + return holesPlacedWith(p, ringBase, scratch.crossed, tol, [&](const Point &h, auto f) { + const std::size_t b = bandOf(h.y()); + for (uint32_t k = start[b]; k < start[b + 1]; k++) + if (!f(segs[items[k]])) return; + }); +} + +// polygonClear of a polygon of one ring of at most kPrecheckAllPairs segments, on arrays: the +// tests of ringFinite, ringClear and smallPairsClear (all must pass, so their order does not +// matter), each on the same values. Segments of nonzero length start at r[st[j]]. +bool smallRingClear(const Ring &r) { + const std::size_t n = r.size(); + double maxAbs = 0; + if (!ringFinite(r, maxAbs)) return false; + const double tol = kPrecheckEps * std::max(1.0, maxAbs); + if (n < 4) return false; + const std::size_t m = n - 1; // segments: i runs r[i] -> r[i+1] + if (r[0].x() != r[m].x() || r[0].y() != r[m].y()) return false; + const Point *pt = r.data(); + const double x0 = pt[0].x(), y0 = pt[0].y(); + double area = 0, areaScale = 0, boostScale = 0; + uint32_t st[kPrecheckAllPairs]; + // segment boxes as segsClear's box test uses them (max + tol) + double bx0[kPrecheckAllPairs], bx1[kPrecheckAllPairs], by0[kPrecheckAllPairs], by1[kPrecheckAllPairs]; + std::size_t k = 0; + const Point *prev = nullptr; // start of the previous segment + for (std::size_t i = 0; i < m; i++) { + const Point &a = pt[i], &b = pt[i + 1]; + boostScale += (std::fabs(a.x()) + std::fabs(b.x())) * std::fabs(a.y() - b.y()); + if (i >= 1 && i + 1 < m) { + const double t1 = (a.x() - x0) * (b.y() - y0); + const double t2 = (a.y() - y0) * (b.x() - x0); + area += t1 - t2; + areaScale += std::fabs(t1) + std::fabs(t2); + } + if (a.x() == b.x() && a.y() == b.y()) continue; + if (std::fabs(b.x() - a.x()) <= tol && std::fabs(b.y() - a.y()) <= tol) return false; + if (prev && !vertexClear(*prev, a, b, tol)) return false; + prev = &a; + st[k] = static_cast(i); + bx0[k] = std::min(a.x(), b.x()); bx1[k] = std::max(a.x(), b.x()) + tol; + by0[k] = std::min(a.y(), b.y()); by1[k] = std::max(a.y(), b.y()) + tol; + k++; + } + if (k < 3) return false; + if (!vertexClear(*prev, pt[st[0]], pt[st[0] + 1], tol)) return false; + if (!ringAreaClear(area, areaScale, boostScale, n, false)) return false; + + // smallPairsClear: every pair but the adjacent ones (segsClear accepts those first), box test + // first (segsClear's), then segsClear's side tests + uint8_t cand[kPrecheckAllPairs]; + for (std::size_t i = 0; i + 2 < k; i++) { + const double ax0 = bx0[i], ax1 = bx1[i], ay0 = by0[i], ay1 = by1[i]; + std::size_t c = 0; + for (std::size_t j = i + 2, end = i == 0 ? k - 1 : k; j < end; j++) { + const bool apart = (ax1 < bx0[j]) | (bx1[j] < ax0) | (ay1 < by0[j]) | (by1[j] < ay0); + cand[c] = static_cast(j); + c += !apart; + } + for (std::size_t q = 0; q < c; q++) { + const uint32_t j = st[cand[q]]; + if (!sidesClear(pt[st[i]], pt[st[i] + 1], pt[j], pt[j + 1], tol)) return false; + } + } + return true; +} + +bool polygonClear(const Polygon &p, PrecheckScratch &scratch) { const Ring &outer = p.outer(); const auto &inners = p.inners(); + // (the small path does polygonClear's tests below for one ring: ringFinite, then smallPairsClear) + if (inners.empty() && outer.size() <= kPrecheckAllPairs + 1) return smallRingClear(outer); if (inners.size() >= std::numeric_limits::max()) return false; - // Every hole is located by walking the outer ring - if (inners.size() * outer.size() > kPrecheckMaxWork) return false; double maxAbs = 0; if (!ringFinite(outer, maxAbs)) return false; for (const auto &inner : inners) @@ -374,29 +975,20 @@ bool polygonClear(const Polygon &p, PrecheckScratch &scratch, PrecheckBox &outer std::vector &segs = scratch.segs; segs.clear(); - std::vector &holeBoxes = scratch.holeBoxes; - holeBoxes.resize(inners.size()); - if (!ringClear(outer, false, tol, 0, segs, outerBox)) return false; - for (std::size_t i = 0; i < inners.size(); i++) { - if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs, holeBoxes[i])) return false; - holeBoxes[i].id = static_cast(i); + { + std::size_t points = outer.size(); + for (const auto &inner : inners) points += inner.size(); + segs.reserve(points); // at most one segment per point: no regrowth } + if (!ringClear(outer, false, tol, 0, segs)) return false; + for (std::size_t i = 0; i < inners.size(); i++) + if (!ringClear(inners[i], true, tol, static_cast(i + 1), segs)) return false; // No two rings touch, and no ring touches itself. - if (!segmentsClear(segs, tol)) return false; + if (!segmentsClear(segs, scratch, tol)) return false; // Holes lie inside the outer ring and are not nested in each other. - for (std::size_t i = 0; i < inners.size(); i++) { - if (boxesApart(holeBoxes[i], outerBox, tol)) return false; - if (ringClearlyInRing(inners[i], outer, tol) != 1) return false; - } - std::size_t work = 0; - return forOverlappingBoxes(holeBoxes, tol, [&](uint32_t i, uint32_t j) { - work += inners[i].size() + inners[j].size(); - return work <= kPrecheckMaxWork && - ringClearlyInRing(inners[i], inners[j], tol) == -1 && - ringClearlyInRing(inners[j], inners[i], tol) == -1; - }); + return holesClearlyPlaced(p, PrecheckSegRange{ segs.data(), segs.data() + segs.size() }, 0, scratch, tol); } // a's outer ring, known not to touch any ring of b, lies outside b's interior: @@ -411,8 +1003,7 @@ bool polygonClearlyOutside(const Polygon &a, const Polygon &b, double tol) { bool multiPolygonClear(const MultiPolygon &mp, PrecheckScratch &scratch) { if (mp.empty()) return false; - PrecheckBox box; - if (mp.size() == 1) return polygonClear(mp[0], scratch, box); + if (mp.size() == 1) return polygonClear(mp[0], scratch); if (mp.size() >= std::numeric_limits::max()) return false; // Polygons whose bboxes (outer rings; holes lie inside them) overlap get the pairwise tests below. @@ -421,6 +1012,7 @@ bool multiPolygonClear(const MultiPolygon &mp, PrecheckScratch &scratch) { double maxAbs = 0; for (const auto &p : mp) { if (p.outer().empty()) return false; + PrecheckBox box; box.minX = box.minY = std::numeric_limits::infinity(); box.maxX = box.maxY = -std::numeric_limits::infinity(); for (const Point &pt : p.outer()) { @@ -445,21 +1037,65 @@ bool multiPolygonClear(const MultiPolygon &mp, PrecheckScratch &scratch) { return work <= kPrecheckMaxWork; })) return false; - for (const auto &p : mp) - if (!polygonClear(p, scratch, box)) return false; - if (pairs.empty()) return true; + if (pairs.empty()) { + for (const auto &p : mp) + if (!polygonClear(p, scratch)) return false; + return true; + } + + // No ring of one polygon touches a ring of another: all rings are tested together at the + // multipolygon's tolerance tol. That includes the rings within each polygon, which + // polygonClear tests at the polygon's own tolerance tolP. When tolP <= tol, the combined + // tests imply those (every tolerance-dependent decision only gets stricter with a larger + // tolerance, and the segments are the same), so only the holes are placed per polygon. + std::vector &tols = scratch.tols; + tols.clear(); + bool implied = true; + for (const auto &p : mp) { + if (p.inners().size() >= std::numeric_limits::max()) return false; + double polyMaxAbs = 0; + if (!ringFinite(p.outer(), polyMaxAbs)) return false; + for (const auto &inner : p.inners()) + if (!ringFinite(inner, polyMaxAbs)) return false; + const double tolP = kPrecheckEps * std::max(1.0, polyMaxAbs); + tols.push_back(tolP); + if (!(tolP <= tol)) implied = false; + } + if (!implied) + for (const auto &p : mp) + if (!polygonClear(p, scratch)) return false; - // No ring of one polygon touches a ring of another. (This re-tests the rings - // within each polygon too, at the multipolygon's possibly larger tolerance.) std::vector &segs = scratch.segs; segs.clear(); + { + std::size_t points = 0; + for (const auto &p : mp) { + points += p.outer().size(); + for (const auto &inner : p.inners()) points += inner.size(); + } + segs.reserve(points); // at most one segment per point: no regrowth + } + std::vector &polyStart = scratch.polyStart; + polyStart.clear(); uint32_t ringIdx = 0; for (const auto &p : mp) { - if (!ringClear(p.outer(), false, tol, ringIdx++, segs, box)) return false; + polyStart.push_back(segs.size()); + if (!ringClear(p.outer(), false, tol, ringIdx++, segs)) return false; for (const auto &inner : p.inners()) - if (!ringClear(inner, true, tol, ringIdx++, segs, box)) return false; + if (!ringClear(inner, true, tol, ringIdx++, segs)) return false; + } + polyStart.push_back(segs.size()); + if (!segmentsClear(segs, scratch, tol)) return false; + + // Holes lie inside their outer ring and are not nested in each other. + if (implied) { + ringIdx = 0; + for (std::size_t i = 0; i < mp.size(); i++) { + const PrecheckSegRange range{ segs.data() + polyStart[i], segs.data() + polyStart[i + 1] }; + if (!holesClearlyPlaced(mp[i], range, ringIdx, scratch, tols[i])) return false; + ringIdx += 1 + static_cast(mp[i].inners().size()); + } } - if (!segmentsClear(segs, tol)) return false; // Interiors are disjoint: neither polygon of an overlapping pair lies inside the other. for (const auto &pr : pairs) @@ -468,11 +1104,72 @@ bool multiPolygonClear(const MultiPolygon &mp, PrecheckScratch &scratch) { return true; } +// fast_clip leaves zero-width bridges along the box edge where a ring leaves and re-enters the +// box: two segments of one polygon on the same edge line of the box that overlap or touch, and +// are not adjacent in their ring. True when mp has such a pair; then multiPolygonClear is false: +// every pair of segments within a polygon gets segsClear (whichever path, at some tolerance), +// numbered as ringClear numbers them (segments of nonzero length, position in the ring, the +// last adjacent to the first). For two segments on one axis-parallel line, every determinant of +// clearOrient and boostSide is exactly 0 (a zero coordinate difference times another), and their +// boxes are not apart (the extents along the line meet, and tol >= 0 only widens them), so +// segsClear is false at any tolerance, in either mode. (A ring that ringClear rejects first +// makes the answer false as well.) +bool clipBridged(const MultiPolygon &mp, const Box &box, PrecheckScratch &scratch) { + const double x0 = box.min_corner().x(), x1 = box.max_corner().x(); + const double y0 = box.min_corner().y(), y1 = box.max_corner().y(); + std::vector &es = scratch.edgeSegs; + std::vector &counts = scratch.crossed; // segments of nonzero length per ring + for (const Polygon &p : mp) { + es.clear(); + counts.clear(); + const std::size_t rings = 1 + p.inners().size(); + for (std::size_t ri = 0; ri < rings; ri++) { + const Ring &r = ri == 0 ? p.outer() : p.inners()[ri - 1]; + uint32_t idx = 0; + for (std::size_t i = 0; i + 1 < r.size(); i++) { + const Point &a = r[i], &b = r[i + 1]; + const double ax = a.x(), ay = a.y(), bx = b.x(), by = b.y(); + if (ax == bx && ay == by) continue; + if (ax == bx && (ax == x0 || ax == x1)) + es.push_back({ ax == x0 ? 0u : 1u, std::min(ay, by), std::max(ay, by), static_cast(ri), idx }); + else if (ay == by && (ay == y0 || ay == y1)) + es.push_back({ ay == y0 ? 2u : 3u, std::min(ax, bx), std::max(ax, bx), static_cast(ri), idx }); + idx++; + } + counts.push_back(idx); + } + if (es.size() < 2) continue; + std::sort(es.begin(), es.end(), [](const PrecheckEdgeSeg &a, const PrecheckEdgeSeg &b) { + return a.edge != b.edge ? a.edge < b.edge : a.lo < b.lo; + }); + for (std::size_t i = 0; i < es.size(); i++) + for (std::size_t j = i + 1; j < es.size() && es[j].edge == es[i].edge && es[j].lo <= es[i].hi; j++) { + const PrecheckEdgeSeg &s = es[i], &t = es[j]; + if (s.ring == t.ring) { + const uint32_t lo = std::min(s.idx, t.idx), hi = std::max(s.idx, t.idx); + if (hi - lo == 1 || (lo == 0 && hi == counts[s.ring] - 1)) continue; // adjacent + } + return true; + } + } + return false; +} + } // namespace +bool multiPolygonDefinitelyValidClipped(const MultiPolygon &mp, const Box &box) { + std::size_t points = 0; + for (const Polygon &p : mp) { + points += p.outer().size(); + for (const Ring &r : p.inners()) points += r.size(); + } + const bool ok = !(points > kPrecheckBridgeMin && clipBridged(mp, box, precheckScratch)) && multiPolygonClear(mp, precheckScratch); + precheckScratch.trim(); + return ok; +} + bool polygonDefinitelyValid(const Polygon &p) { - PrecheckBox box; - const bool ok = polygonClear(p, precheckScratch, box); + const bool ok = polygonClear(p, precheckScratch); precheckScratch.trim(); return ok; } @@ -483,6 +1180,139 @@ bool multiPolygonDefinitelyValid(const MultiPolygon &mp) { return ok; } +int polygonDefinitelyValidOrReversed(const Polygon &p) { + precheckOrient = 2; + const bool ok = polygonClear(p, precheckScratch); + const int o = precheckOrient; + precheckOrient = 0; + precheckScratch.trim(); + return ok ? o : 0; +} + +int multiPolygonDefinitelyValidOrReversed(const MultiPolygon &mp) { + precheckOrient = 2; + const bool ok = multiPolygonClear(mp, precheckScratch); + const int o = precheckOrient; + precheckOrient = 0; + precheckScratch.trim(); + return ok ? o : 0; +} + +// Boost's cartesian_winding count of r around h (not 0: inside), or ok = false when it can't be +// told. Boost walks the segments whose x range holds h.x (a vertical ray): one with an end at +// h.x (math::equals) counts as touching the ray there, decided by y; one that spans h.x adds +-2 +// when h is on the matching side of it, and makes h "on the boundary" when its side is 0 within +// Boost's zero band (which a very short segment far from h can have). Here an end near h.x, or a +// side that clearOrient can't call clearly (its band is far above Boost's), is unsure. +static int clearWinding(const Point &h, const Ring &r, double tol, bool &ok) { + const double px = h.x(); + auto nearX = [px](double x) { + return std::fabs(x - px) <= 4 * std::numeric_limits::epsilon() * max3(1.0, std::fabs(x), std::fabs(px)); + }; + int w = 0; + for (std::size_t i = 0; i + 1 < r.size(); i++) { + const Point &a = r[i], &b = r[i + 1]; + if (nearX(a.x()) || nearX(b.x())) { ok = false; return 0; } + if ((a.x() < px) == (b.x() < px)) continue; + const int count = a.x() < px ? 2 : -2; + const int side = clearOrient(a, b, h, tol); + if (side == 0) { ok = false; return 0; } + if (side * count > 0) w += count; + } + return w; +} + +// Boost's covered_by(h, p) is certainly false: h is clearly outside p's outer ring, or clearly +// inside it and clearly inside one of its holes (point_in_geometry of a polygon) +static bool clearlyNotCovered(const Point &h, const Polygon &p, double tol) { + bool ok = true; + if (clearWinding(h, p.outer(), tol, ok) == 0) return ok; + if (!ok) return false; + for (const auto &inner : p.inners()) { + const int w = clearWinding(h, inner, tol, ok); + if (!ok) return false; + if (w != 0) return true; + } + return false; +} + +// Boost's intersects(a, b) is !disjoint: no turns between the rings of a and those of b +// (disjoint_linear; every pair of their segments is clearly apart, as the precheck decides it), +// and no ring of either has its first point covered by the other (rings_containing; that point +// is a vertex of a segment clear of the other's segments, so its location is clear too). +bool polygonsDefinitelyApart(const Polygon &a, const Polygon &b) { + double maxAbs = 0; + auto finite = [&](const Polygon &p) { + // (a ring Boost doesn't walk, below 4 points, is left to it) + if (p.outer().size() < 4 || !ringFinite(p.outer(), maxAbs)) return false; + for (const auto &r : p.inners()) + if (r.size() < 4 || !ringFinite(r, maxAbs)) return false; + return true; + }; + if (!finite(a) || !finite(b)) return false; + const double tol = kPrecheckEps * std::max(1.0, maxAbs); + // (of all rings: a hole of an invalid polygon can lie outside its outer ring) + auto envelope = [](const Polygon &poly) { + PrecheckBox box; + box.minX = box.minY = std::numeric_limits::infinity(); + box.maxX = box.maxY = -std::numeric_limits::infinity(); + auto add = [&](const Ring &r) { + for (const Point &p : r) { + box.minX = std::min(box.minX, p.x()); box.maxX = std::max(box.maxX, p.x()); + box.minY = std::min(box.minY, p.y()); box.maxY = std::max(box.maxY, p.y()); + } + }; + add(poly.outer()); + for (const auto &r : poly.inners()) add(r); + return box; + }; + const PrecheckBox boxA = envelope(a), boxB = envelope(b); + // The segments of p's rings whose boxes are not clearly apart from the other polygon's box + // (the rest are clearly apart from all its segments); b's rings numbered apart from a's + thread_local std::vector segsA, segsB; + auto collect = [&](const Polygon &p, const PrecheckBox &other, uint32_t ringBase, std::vector &out) { + out.clear(); + uint32_t ringIdx = ringBase; + auto ring = [&](const Ring &r) { + for (std::size_t i = 0; i + 1 < r.size(); i++) { + const Point &p0 = r[i], &p1 = r[i + 1]; + PrecheckSeg s; + s.minX = std::min(p0.x(), p1.x()); s.maxX = std::max(p0.x(), p1.x()); + s.minY = std::min(p0.y(), p1.y()); s.maxY = std::max(p0.y(), p1.y()); + if (s.maxX + tol < other.minX || other.maxX + tol < s.minX || s.maxY + tol < other.minY || other.maxY + tol < s.minY) + continue; + s.a = &p0; + s.ring = ringIdx; s.idx = 0; // (rings of a and b are numbered apart: never adjacent) + out.push_back(s); + } + ringIdx++; + }; + ring(p.outer()); + for (const auto &r : p.inners()) ring(r); + }; + if (a.inners().size() >= (1u << 30) || b.inners().size() >= (1u << 30)) return false; + collect(a, boxB, 0, segsA); + collect(b, boxA, 1u << 31, segsB); + if (segsA.size() * segsB.size() > (std::size_t(1) << 16)) return false; + for (const PrecheckSeg &s : segsA) + for (const PrecheckSeg &t : segsB) + if (!segsClear(s, t, tol)) return false; + if (!clearlyNotCovered(b.outer().front(), a, tol)) return false; + for (const auto &r : b.inners()) if (!clearlyNotCovered(r.front(), a, tol)) return false; + if (!clearlyNotCovered(a.outer().front(), b, tol)) return false; + for (const auto &r : a.inners()) if (!clearlyNotCovered(r.front(), b, tol)) return false; + return true; +} + +bool multiPolygonDefinitelyValidAfterClip(const MultiPolygon &mp) { + precheckBoostSides = true; + precheckUsedBoostSides = false; + const bool ok = multiPolygonClear(mp, precheckScratch); + precheckBoostSides = false; + precheckScratch.trim(); + return ok; +} + // Repair a single (possibly invalid) polygon in an area-preserving way and // append the resulting valid polygon(s) to `out`. Returns true on success. @@ -494,32 +1324,88 @@ bool multiPolygonDefinitelyValid(const MultiPolygon &mp) { // polygon swallow a whole island. Measured on real data, the separation is // large: genuine repairs of quantised ocean polygons change the area by // <=0.01 %, whereas filling one island hole grew it by 32 %. -static bool repair_one_polygon(const Polygon &p, double minArea, double maxArea, MultiPolygon &out) +// reversedValid: p is known to be valid once its rings are reversed. +// +// 0) Fast path: a ring without holes whose only fault is its orientation +// (e.g. after the y-flip into tile coordinates). On such a simple ring the +// dissolve finds no intersections and just reverses it, so do that directly. +// Done on p itself: returns true when the reversed p is the result, otherwise +// p is reversed back (unchanged) and repair_one_polygon_rest() goes on. +// The same for a polygon with holes that is valid once its rings are reversed +// (reversedValid): for those the dissolve below returns exactly that polygon, so +// the rings are reversed in place instead. Only with integer coordinates of +// |c| <= 2^15 and at most 2^20 points, where every computation on the way is +// exact (cross products, Boost's segment intersection ratios at shared +// endpoints, shoelace areas). Why the dissolve's result is p with every ring +// reversed (same points, same order, same first points, holes in order): +// - geometry::correct(ring) of the outer ring reverses it (wrong orientation), +// finds no self-intersection (the rings are clearly simple: only adjacent +// segments meet, at their shared point, which Boost returns exactly, so the +// scales are exactly 0 or 1 - or 0/0 for a repeated point - never both < 1) +// and returns the reversed copy; each hole is already in outer orientation +// and is returned as it is. All have area > 1E-12 (non-zero integers / 2). +// - result_combine finds no intersecting pair (the rings are clearly apart), +// so the holes stay in input order. +// - difference(outer, holes) has no turns: Boost copies the outer ring and +// each hole (clearly inside) reversed, in the order of their ring ids. +// - acceptable(): the result is valid, its area is |area(p)|. +// (Checked on all recorded planet and local cases, and fuzzed.) +static bool repair_reversed_with_holes(Polygon &p, double minArea, double maxArea) +{ + double maxAbs = 0; + std::size_t points = 0; + auto exactRing = [&](const Ring &r) { + points += r.size(); + for (const Point &q : r) { + const double ax = std::fabs(q.x()), ay = std::fabs(q.y()); + if (!(ax <= 32768.0 && ay <= 32768.0)) return false; + if (double(int32_t(ax)) != ax || double(int32_t(ay)) != ay) return false; + maxAbs = std::max(maxAbs, std::max(ax, ay)); + } + return true; + }; + if (!exactRing(p.outer())) return false; + for (const auto &r : p.inners()) if (!exactRing(r)) return false; + if (points > (std::size_t(1) << 20)) return false; + auto reverseAll = [&p]() { + std::reverse(p.outer().begin(), p.outer().end()); + for (auto &r : p.inners()) std::reverse(r.begin(), r.end()); + }; + reverseAll(); + // the dissolve drops rings of area <= 1E-12 and checks the result's area + bool ok = std::abs(geom::area(p.outer())) > 1E-12; + for (const auto &r : p.inners()) ok = ok && std::abs(geom::area(r)) > 1E-12; + if (ok) { + const double a = std::abs(geom::area(p)); + ok = a >= minArea && a <= maxArea; + } + if (!ok) reverseAll(); + return ok; +} + +static bool repair_one_polygon_reverse(Polygon &p, double minArea, double maxArea, bool reversedValid) +{ + if (!p.inners().empty()) return reversedValid && repair_reversed_with_holes(p, minArea, maxArea); + std::reverse(p.outer().begin(), p.outer().end()); + const double revArea = std::abs(geom::area(p)); + bool revValid = false; + if (revArea > 1E-12 && revArea >= minArea && revArea <= maxArea) { + const bool pre = reversedValid || polygonDefinitelyValid(p); + revValid = pre || geom::is_valid(p); + } + if (!revValid) std::reverse(p.outer().begin(), p.outer().end()); + return revValid; +} + +// Steps 1) and 2) of the repair: appends the result to out and returns true on success. +static bool repair_one_polygon_rest(const Polygon &p, double minArea, double maxArea, MultiPolygon &out) { auto acceptable = [&](const MultiPolygon &candidate) { - if (!geom::is_valid(candidate)) return false; + if (!multiPolygonDefinitelyValid(candidate) && !geom::is_valid(candidate)) return false; const double a = std::abs(geom::area(candidate)); return a >= minArea && a <= maxArea; }; - // 0) Fast path: a ring without holes whose only fault is its orientation - // (e.g. after the y-flip into tile coordinates). On such a simple ring the - // dissolve finds no intersections and just reverses it, so do that directly. - if (p.inners().empty()) { - Polygon rev = p; - std::reverse(rev.outer().begin(), rev.outer().end()); - const double revArea = std::abs(geom::area(rev)); - bool revValid = false; - if (revArea > 1E-12 && revArea >= minArea && revArea <= maxArea) { - const bool pre = polygonDefinitelyValid(rev); - revValid = pre || geom::is_valid(rev); - } - if (revValid) { - out.push_back(std::move(rev)); - return true; - } - } - // 1) Dissolve (resolves self-intersections of this single polygon). try { MultiPolygon fixed; @@ -557,9 +1443,12 @@ static bool repair_one_polygon(const Polygon &p, double minArea, double maxArea, bool repair_multi_polygon(MultiPolygon &mp, bool knownInvalid, bool *wholeValid, AreaBound areaBound) { if (wholeValid) *wholeValid = false; + // -1: mp is invalid (wrongly oriented), and valid with every ring reversed; so is each + // polygon of it on its own + int pre = 0; if (!knownInvalid) { - const bool pre = multiPolygonDefinitelyValid(mp); - if (pre || geom::is_valid(mp)) { + pre = multiPolygonDefinitelyValidOrReversed(mp); + if (pre > 0 || (pre == 0 && geom::is_valid(mp))) { if (wholeValid) *wholeValid = true; return true; } @@ -576,15 +1465,29 @@ bool repair_multi_polygon(MultiPolygon &mp, bool knownInvalid, bool *wholeValid, // O(n^2) cross-polygon union that caused the collapse. A polygon whose repair // would not preserve its area is kept as-is (invalid but complete renders; // only a tiny local artefact, never a dropped area). + // + // The result is built in mp itself while every polygon's result is that + // polygon (kept, or reversed in place): mp[0, i) is the result so far. From + // the first polygon with another result on, it goes to out (mp[0, i) moved + // there first), and replaces mp at the end. MultiPolygon out; + bool inPlace = true; + auto leavePlace = [&](std::size_t i) { + if (!inPlace) return; + inPlace = false; + out.reserve(mp.size()); + for (std::size_t j = 0; j < i; j++) out.push_back(std::move(mp[j])); + }; bool allValid = true; - for (const auto &p : mp) { + const std::size_t count = mp.size(); + for (std::size_t i = 0; i < count; i++) { + Polygon &p = mp[i]; // With a single polygon, the whole-multipolygon check above already // established it is invalid. - if (mp.size() > 1) { - const bool pre = polygonDefinitelyValid(p); - if (pre || geom::is_valid(p)) { - out.push_back(p); + if (count > 1 && pre >= 0) { + const bool prePolygon = polygonDefinitelyValid(p); + if (prePolygon || geom::is_valid(p)) { + if (!inPlace) out.push_back(std::move(p)); continue; } } @@ -601,13 +1504,18 @@ bool repair_multi_polygon(MultiPolygon &mp, bool knownInvalid, bool *wholeValid, const double maxArea = areaBound == AreaBound::Strict ? 1.01 * origArea : std::numeric_limits::infinity(); - if (!repair_one_polygon(p, 0.5 * origArea, maxArea, out)) { - out.push_back(p); + if (repair_one_polygon_reverse(p, 0.5 * origArea, maxArea, pre < 0)) { + if (!inPlace) out.push_back(std::move(p)); + continue; + } + leavePlace(i); + if (!repair_one_polygon_rest(p, 0.5 * origArea, maxArea, out)) { + out.push_back(std::move(p)); allValid = false; } } - mp = std::move(out); + if (!inPlace) mp = std::move(out); // A single valid polygon is a valid multipolygon if (wholeValid) *wholeValid = allValid && mp.size() == 1; return allValid; @@ -741,39 +1649,111 @@ char bit_code(Point const &p, Box const &bbox) { // Sutherland-Hodgeman polygon clipping algorithm // Returns true if any clipping was applied (false: points left untouched). -bool fast_clip(Ring &points, Box const &bbox, Ring &result) { - bool modified = false; - // clip against each side of the clip rectangle - for (char edge = 1; edge <= 8; edge *= 2) { - bool needsClip = false; - for (auto const &p: points) { - if (bit_code(p, bbox) & edge) { - needsClip = true; - break; - } +// +// The result is the same as clipping against each edge in turn after scanning all points for +// one outside that edge: the bit codes of all points are OR-ed in a single pass, and each clip +// pass ORs the codes of the points it outputs, which decides the later edges. + +namespace { + +static_assert(sizeof(Point) == 2 * sizeof(double), "Point must be two packed doubles"); + +// OR of bit_code() over n points +inline int bit_codes(const Point *pts, std::size_t n, Box const &bbox) { +#ifdef __SSE2__ + const __m128d lo = _mm_set_pd(bbox.min_corner().y(), bbox.min_corner().x()); + const __m128d hi = _mm_set_pd(bbox.max_corner().y(), bbox.max_corner().x()); + __m128d below0 = _mm_setzero_pd(), above0 = _mm_setzero_pd(); + __m128d below1 = _mm_setzero_pd(), above1 = _mm_setzero_pd(); + const double *d = reinterpret_cast(pts); + std::size_t i = 0; + for (; i + 2 <= n; i += 2) { + const __m128d v0 = _mm_loadu_pd(d + 2 * i), v1 = _mm_loadu_pd(d + 2 * i + 2); + const __m128d lt0 = _mm_cmplt_pd(v0, lo), lt1 = _mm_cmplt_pd(v1, lo); + below0 = _mm_or_pd(below0, lt0); below1 = _mm_or_pd(below1, lt1); + // right/top only if not left/bottom (bit_code's else-if) + above0 = _mm_or_pd(above0, _mm_andnot_pd(lt0, _mm_cmpgt_pd(v0, hi))); + above1 = _mm_or_pd(above1, _mm_andnot_pd(lt1, _mm_cmpgt_pd(v1, hi))); + } + if (i < n) { + const __m128d v0 = _mm_loadu_pd(d + 2 * i); + const __m128d lt0 = _mm_cmplt_pd(v0, lo); + below0 = _mm_or_pd(below0, lt0); + above0 = _mm_or_pd(above0, _mm_andnot_pd(lt0, _mm_cmpgt_pd(v0, hi))); + } + const int b = _mm_movemask_pd(_mm_or_pd(below0, below1)), a = _mm_movemask_pd(_mm_or_pd(above0, above1)); + return (b & 1) | ((a & 1) << 1) | ((b & 2) << 1) | ((a & 2) << 2); +#else + int code = 0; + for (std::size_t i = 0; i < n; i++) code |= bit_code(pts[i], bbox); + return code; +#endif +} + +// bit_code(p) & edge for a single edge +inline bool outside_edge(Point const &p, char edge, Box const &bbox) { + switch (edge) { + case 1: return p.x() < bbox.min_corner().x(); + case 2: return !(p.x() < bbox.min_corner().x()) && p.x() > bbox.max_corner().x(); + case 4: return p.y() < bbox.min_corner().y(); + default: return !(p.y() < bbox.min_corner().y()) && p.y() > bbox.max_corner().y(); + } +} + +// One Sutherland-Hodgman pass against `edge`; returns the OR of the output points' bit codes +template +int clip_pass(const Ring &points, Box const &bbox, Ring &result) { + result.clear(); + result.reserve(points.size() + 4); + int codes = 0; + Point prev = points[points.size() - 1]; + bool prevInside = !outside_edge(prev, edge, bbox); + + for (std::size_t i = 0; i < points.size(); i++) { + const Point &p = points[i]; + const bool inside = !outside_edge(p, edge, bbox); + + // if segment goes through the clip window, add an intersection + if (inside != prevInside) { + result.emplace_back(intersect_edge(prev, p, edge, bbox)); + codes |= bit_code(result.back(), bbox); } - if (!needsClip) continue; - - modified = true; - result.clear(); - result.reserve(points.size() + 4); - Point prev = points[points.size() - 1]; - bool prevInside = (bit_code(prev, bbox) & edge)==0; - - for (unsigned int i = 0; i(points, bbox, result); points.swap(result); - if (points.size()==0) break; + if (points.size() == 0) return true; } - return modified; + if (codes & 2) { + codes = clip_pass<2>(points, bbox, result); + points.swap(result); + if (points.size() == 0) return true; + } + if (codes & 4) { + codes = clip_pass<4>(points, bbox, result); + points.swap(result); + if (points.size() == 0) return true; + } + if (codes & 8) { + clip_pass<8>(points, bbox, result); + points.swap(result); + } + return true; } bool fast_clip(Ring &points, Box const &bbox) { @@ -781,37 +1761,157 @@ bool fast_clip(Ring &points, Box const &bbox) { return fast_clip(points, bbox, result); } +namespace { +// Scratch ring for the clip passes, kept between calls unless it grew large +struct ClipScratch { + Ring ring; + ~ClipScratch() = default; + void trim() { if (ring.capacity() > (1u << 16)) Ring().swap(ring); } +}; +thread_local ClipScratch clipScratch; +} + // Wrappers for polygon/multipolygon -bool fast_clip(Polygon &polygon, Box const &bbox) { - Ring result; +static bool fast_clip(Polygon &polygon, Box const &bbox, Ring &result) { bool modified = fast_clip(polygon.outer(), bbox, result); if (polygon.outer().empty()) { modified = modified || !polygon.inners().empty(); polygon.inners().resize(0); return modified; } + bool anyEmpty = false; for (auto &inner: polygon.inners()) { if (fast_clip(inner, bbox, result)) modified = true; + if (inner.empty()) anyEmpty = true; } - size_t innersBefore = polygon.inners().size(); + if (!anyEmpty) return modified; polygon.inners().erase(std::remove_if( - polygon.inners().begin(), polygon.inners().end(), + polygon.inners().begin(), polygon.inners().end(), [](const Ring &inner) -> bool { return inner.empty(); }), polygon.inners().end()); - return modified || polygon.inners().size() != innersBefore; + return true; +} + +bool fast_clip(Polygon &polygon, Box const &bbox) { + Ring &result = clipScratch.ring; + const bool modified = fast_clip(polygon, bbox, result); + clipScratch.trim(); + return modified; } bool fast_clip(MultiPolygon &mp, Box const &bbox) { - bool modified = false; + Ring &result = clipScratch.ring; + bool modified = false, anyEmpty = false; for (auto &polygon: mp) { - if (fast_clip(polygon, bbox)) modified = true; + if (fast_clip(polygon, bbox, result)) modified = true; + if (polygon.outer().empty()) anyEmpty = true; } - size_t sizeBefore = mp.size(); + clipScratch.trim(); + if (!anyEmpty) return modified; mp.erase(std::remove_if( - mp.begin(), mp.end(), + mp.begin(), mp.end(), [](const Polygon &poly) -> bool { return poly.outer().empty(); }), mp.end()); - return modified || mp.size() != sizeBefore; + return true; +} + +namespace { +// fast_clip of a copy of src, written to dst: the first clip pass reads src directly +bool fast_clip_ring_copy(const Ring &src, Box const &bbox, Ring &dst, Ring &scratch) { + int codes = bit_codes(src.data(), src.size(), bbox); + if (codes == 0) { dst.assign(src.begin(), src.end()); return false; } + const Ring *cur = &src; + // each pass writes to dst (first pass) or scratch, then the result is swapped into dst + auto pass = [&](auto clip) { + if (cur == &src) { codes = clip(src, dst); cur = &dst; } + else { codes = clip(dst, scratch); dst.swap(scratch); } + return !dst.empty(); + }; + if ((codes & 1) && !pass([&](const Ring &a, Ring &b) { return clip_pass<1>(a, bbox, b); })) return true; + if ((codes & 2) && !pass([&](const Ring &a, Ring &b) { return clip_pass<2>(a, bbox, b); })) return true; + if ((codes & 4) && !pass([&](const Ring &a, Ring &b) { return clip_pass<4>(a, bbox, b); })) return true; + if (codes & 8) pass([&](const Ring &a, Ring &b) { return clip_pass<8>(a, bbox, b); }); + return true; +} +} + +// Same result as `geom::assign(out, in); return fast_clip(out, bbox);`, without copying the points +// that are clipped away. out must be empty. +bool fast_clip_copy(const MultiPolygon &in, Box const &bbox, MultiPolygon &out) { + Ring &scratch = clipScratch.ring; + bool modified = false, anyEmpty = false; + out.resize(in.size()); + for (std::size_t i = 0; i < in.size(); i++) { + const Polygon &p = in[i]; + Polygon &q = out[i]; + if (fast_clip_ring_copy(p.outer(), bbox, q.outer(), scratch)) modified = true; + if (q.outer().empty()) { + // the polygon is removed below + anyEmpty = true; + continue; + } + q.inners().resize(p.inners().size()); + bool innerEmpty = false; + for (std::size_t j = 0; j < p.inners().size(); j++) { + if (fast_clip_ring_copy(p.inners()[j], bbox, q.inners()[j], scratch)) modified = true; + if (q.inners()[j].empty()) innerEmpty = true; + } + if (innerEmpty) { + q.inners().erase(std::remove_if( + q.inners().begin(), q.inners().end(), + [](const Ring &inner) -> bool { return inner.empty(); }), + q.inners().end()); + modified = true; + } + } + clipScratch.trim(); + if (!anyEmpty) return modified; + out.erase(std::remove_if( + out.begin(), out.end(), + [](const Polygon &poly) -> bool { return poly.outer().empty(); }), + out.end()); + return true; +} + +// The clip box of buildWayGeometry for a polygon at the end zoom: clippingBox, grown to take in +// both ends of every segment that crosses its border (one end strictly within, the other not), +// then limited to extBox. +Box tile_clip_box_end_zoom(const MultiPolygon &mp, Box const &clippingBox, Box const &extBox) { + Box box = clippingBox; + const double x0 = clippingBox.min_corner().x(), y0 = clippingBox.min_corner().y(); + const double x1 = clippingBox.max_corner().x(), y1 = clippingBox.max_corner().y(); + // geom::within(point, box): strictly inside in both dimensions + auto within = [x0, y0, x1, y1](const Point &p) { + return p.x() > x0 && p.x() < x1 && p.y() > y0 && p.y() < y1; + }; + auto scan = [&](const Ring &r) { + if (r.empty()) return; + bool w1 = within(r[0]); + for (std::size_t i = 0; i + 1 < r.size(); ++i) { + const bool w2 = within(r[i + 1]); + if (w1 != w2) { + const Point p1 = r[i], p2 = r[i + 1]; + box.min_corner() = Point( + std::min(box.min_corner().x(), std::min(p1.x(), p2.x())), + std::min(box.min_corner().y(), std::min(p1.y(), p2.y()))); + box.max_corner() = Point( + std::max(box.max_corner().x(), std::max(p1.x(), p2.x())), + std::max(box.max_corner().y(), std::max(p1.y(), p2.y()))); + } + w1 = w2; + } + }; + for (auto const &p : mp) { + for (auto const &inner : p.inners()) scan(inner); + scan(p.outer()); + } + box.min_corner() = Point( + std::max(box.min_corner().x(), extBox.min_corner().x()), + std::max(box.min_corner().y(), extBox.min_corner().y())); + box.max_corner() = Point( + std::min(box.max_corner().x(), extBox.max_corner().x()), + std::min(box.max_corner().y(), extBox.max_corner().y())); + return box; } // --------------- @@ -839,6 +1939,11 @@ struct RectClipper { bool inside(const Point &p) const { return p.x() >= x0 && p.x() <= x1 && p.y() >= y0 && p.y() <= y1; } + // Both end points beyond the same edge: the segment can't reach the box + bool beyondOneEdge(const Point &a, const Point &b) const { + return (a.x() < x0 && b.x() < x0) || (a.x() > x1 && b.x() > x1) || + (a.y() < y0 && b.y() < y0) || (a.y() > y1 && b.y() > y1); + } bool strictlyInside(const Point &p) const { return p.x() > x0 && p.x() < x1 && p.y() > y0 && p.y() < y1; } @@ -960,12 +2065,17 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector t0; + // Most segments of a large polygon lie far from a small tile's box: reject + // those with four comparisons before the clip computation + const bool hit = !rc.beyondOneEdge(a, b) && rc.clipSegment(a, b, t0, t1, pa, pb) && t1 > t0; if (!hit || (building && t0 > 0)) { if (building && !finish()) return RingClip::Fail; if (!hit) continue; @@ -978,8 +2088,8 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector= 2 && !rc.strictlyInside(b) && rc.throughInterior(current[current.size() - 2], b) && - rc.entersInterior(b, ring[(start + k + 2) % n]) && - rc.pinchAt(current[current.size() - 2], b, ring[(start + k + 2) % n])) { + rc.entersInterior(b, ring[ib + 1 == n ? 0 : ib + 1]) && + rc.pinchAt(current[current.size() - 2], b, ring[ib + 1 == n ? 0 : ib + 1])) { // b touches the boundary from inside (reached through the interior, left // into the interior) and the polygon lies along the boundary on both sides // of it. Kept in one chain, the boundary walk would run through b and make @@ -999,10 +2109,14 @@ RingClip clipRing(const RectClipper &rc, const Ring &ring, std::vector p.y()) != (b.y() > p.y())) { + if (std::fabs(a.y() - py) <= near && geom::equals(a, p)) return 0; + if ((a.y() > py) != (b.y() > py)) { const double x = a.x() + (p.y() - a.y()) * (b.x() - a.x()) / (b.y() - a.y()); if (x == p.x()) return 0; if (x > p.x()) in = !in; @@ -1020,6 +2134,92 @@ int ringInRing(const Ring &ring, const Ring &outer) { return 0; } +// pointInRing() for many points against one large ring: the ring's edges bucketed by y. +// pointInRing's result doesn't depend on the order of the edges (0 as soon as any vertex equals p +// or any crossing is at p.x, else the parity of the crossings right of p), so testing only the +// edges that can cross y == p.y, and the vertices whose y can be within `near` of it, with the +// very same expressions, gives the same result. +struct RingYIndex { + const Ring *ring = nullptr; + double y0 = 0, scale = 0; + int nb = 0; + std::vector eStart, edges, vStart, verts; + + int bucket(double y) const { + const double f = (y - y0) * scale; + if (!(f >= 0)) return 0; + if (f >= nb) return nb - 1; + return int(f); + } + bool build(const Ring &r) { + ring = &r; + const std::size_t n = r.size(); + if (n < 64 || n > (1u << 30)) return false; + double lo = r[0].y(), hi = lo; + for (const Point &p : r) { + if (!std::isfinite(p.x()) || !std::isfinite(p.y())) return false; + lo = std::min(lo, p.y()); hi = std::max(hi, p.y()); + } + if (!(hi > lo)) return false; + nb = int(std::min(std::max(n / 4, 1), 1u << 16)); + y0 = lo; scale = nb / (hi - lo); + if (!std::isfinite(scale)) return false; + eStart.assign(nb + 1, 0); vStart.assign(nb + 1, 0); + std::size_t total = 0; + for (std::size_t i = 0, j = n - 1; i < n; j = i++) { + const double a = r[i].y(), b = r[j].y(); + const int b0 = bucket(std::min(a, b)), b1 = bucket(std::max(a, b)); + total += b1 - b0 + 1; + if (total > 16 * n) return false; + for (int k = b0; k <= b1; k++) eStart[k + 1]++; + vStart[bucket(a) + 1]++; + } + for (int k = 0; k < nb; k++) { eStart[k + 1] += eStart[k]; vStart[k + 1] += vStart[k]; } + edges.resize(total); verts.resize(n); + std::vector ef(eStart.begin(), eStart.end() - 1), vf(vStart.begin(), vStart.end() - 1); + for (std::size_t i = 0, j = n - 1; i < n; j = i++) { + const double a = r[i].y(), b = r[j].y(); + const int b0 = bucket(std::min(a, b)), b1 = bucket(std::max(a, b)); + for (int k = b0; k <= b1; k++) edges[ef[k]++] = uint32_t(i); + verts[vf[bucket(a)]++] = uint32_t(i); + } + return true; + } + // == pointInRing(p, *ring) + int query(const Point &p) const { + const Ring &r = *ring; + const double py = p.y(); + if (!std::isfinite(py) || !std::isfinite(p.x())) return pointInRing(p, r); + const double near = 1e-9 * std::max(1.0, std::fabs(py)); + for (int k = bucket(py - 2 * near), k1 = bucket(py + 2 * near); k <= k1; k++) + for (uint32_t e = vStart[k]; e < vStart[k + 1]; e++) { + const Point &a = r[verts[e]]; + if (std::fabs(a.y() - py) <= near && geom::equals(a, p)) return 0; + } + bool in = false; + const int k = bucket(py); + const std::size_t n = r.size(); + for (uint32_t e = eStart[k]; e < eStart[k + 1]; e++) { + const std::size_t i = edges[e], j = i == 0 ? n - 1 : i - 1; + const Point &a = r[i], &b = r[j]; + if ((a.y() > py) != (b.y() > py)) { + const double x = a.x() + (p.y() - a.y()) * (b.x() - a.x()) / (b.y() - a.y()); + if (x == p.x()) return 0; + if (x > p.x()) in = !in; + } + } + return in ? 1 : -1; + } + // == ringInRing(hole, *ring) + int ringIn(const Ring &hole) const { + for (const Point &p : hole) { + const int res = query(p); + if (res != 0) return res; + } + return 0; + } +}; + bool clipPolygon(const RectClipper &rc, const Polygon &poly, MultiPolygon &out) { std::vector chains; const RingClip outerClip = clipRing(rc, poly.outer(), chains); @@ -1107,11 +2307,21 @@ bool clipPolygon(const RectClipper &rc, const Polygon &poly, MultiPolygon &out) } } - // Holes wholly inside the box go to whichever new outer contains them + // Holes wholly inside the box go to whichever new outer contains them. With many holes, + // each new outer is tested through a y index of its edges (same results as ringInRing). + const bool useIndex = insideHoles.size() >= 8; + std::vector index(useIndex ? out.size() - firstOuter : 0); + std::vector indexState(index.size(), 0); // 0 not built yet, 1 built, 2 not indexable for (const Ring *hole : insideHoles) { bool placed = false; for (std::size_t i = firstOuter; i < out.size() && !placed; i++) { - const int r = ringInRing(*hole, out[i].outer()); + int r; + if (useIndex) { + char &state = indexState[i - firstOuter]; + if (state == 0) state = index[i - firstOuter].build(out[i].outer()) ? 1 : 2; + r = state == 1 ? index[i - firstOuter].ringIn(*hole) : ringInRing(*hole, out[i].outer()); + } else + r = ringInRing(*hole, out[i].outer()); if (r == 0) return false; if (r > 0) { out[i].inners().push_back(*hole); diff --git a/src/helpers.cpp b/src/helpers.cpp index e05c3fde..1eadc631 100644 --- a/src/helpers.cpp +++ b/src/helpers.cpp @@ -11,6 +11,7 @@ #include #include "external/libdeflate/libdeflate.h" #include "helpers.h" +#include #ifdef _MSC_VER #define stat64 __stat64 @@ -109,25 +110,20 @@ std::string compress_string(const std::string& str, if (compressionlevel != compressor.level) compressor.setLevel(compressionlevel); - std::string rv; - if (asGzip) { - size_t maxSize = libdeflate_gzip_compress_bound(compressor.compressor, str.size()); - rv.resize(maxSize); - - size_t compressedSize = libdeflate_gzip_compress(compressor.compressor, str.data(), str.size(), &rv[0], maxSize); - if (compressedSize == 0) - throw std::runtime_error("libdeflate_gzip_compress failed"); - rv.resize(compressedSize); - } else { - size_t maxSize = libdeflate_zlib_compress_bound(compressor.compressor, str.size()); - rv.resize(maxSize); - - size_t compressedSize = libdeflate_zlib_compress(compressor.compressor, str.data(), str.size(), &rv[0], maxSize); - if (compressedSize == 0) - throw std::runtime_error("libdeflate_zlib_compress failed"); - rv.resize(compressedSize); - } - + // Compress into a scratch buffer and return a string of exactly the compressed size. A string + // resized down from the compression bound keeps the bound (about the uncompressed size) as its + // capacity: the tiles held for the ordered writer then took several times the memory counted. + thread_local std::vector scratch; + const size_t maxSize = asGzip ? libdeflate_gzip_compress_bound(compressor.compressor, str.size()) + : libdeflate_zlib_compress_bound(compressor.compressor, str.size()); + if (scratch.size() < maxSize) scratch.resize(maxSize); + const size_t compressedSize = asGzip + ? libdeflate_gzip_compress(compressor.compressor, str.data(), str.size(), scratch.data(), maxSize) + : libdeflate_zlib_compress(compressor.compressor, str.data(), str.size(), scratch.data(), maxSize); + if (compressedSize == 0) + throw std::runtime_error(asGzip ? "libdeflate_gzip_compress failed" : "libdeflate_zlib_compress failed"); + std::string rv(scratch.data(), compressedSize); + if (scratch.size() > (64u << 20)) std::vector().swap(scratch); // e.g. a huge metadata or leaf return rv; } diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp index c056d3a6..1d2e653f 100644 --- a/src/mbtiles.cpp +++ b/src/mbtiles.cpp @@ -2,15 +2,30 @@ #include "helpers.h" #include #include +#include using namespace sqlite; using namespace std; MBTiles::MBTiles(): pendingStatements1(std::make_shared>()), - pendingStatements2(std::make_shared>()) + pendingStatements2(std::make_shared>()), + pendingBytes(0), + batches([this](PendingStatement& stmt) { + const std::lock_guard lock(m); + insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + }, + [](const PendingStatement& stmt) { return sizeof(PendingStatement) + stmt.data.size(); }) {} +// At most this much is buffered while SQLite is busy; beyond it, workers wait +// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB). +static const size_t MAX_PENDING_BYTES = 256 << 20; + +static size_t pendingCost(const std::string& data) { + return sizeof(PendingStatement) + data.size(); +} + MBTiles::~MBTiles() { if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open } @@ -71,6 +86,7 @@ void MBTiles::flushPendingStatements() { while(!pendingStatements2->empty()) { const PendingStatement& stmt = pendingStatements2->back(); insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + pendingBytes -= pendingCost(stmt.data); pendingStatements2->pop_back(); } @@ -80,19 +96,37 @@ void MBTiles::flushPendingStatements() { } void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) { + if (batches.inBatch()) { + batches.add({zoom, x, y, *data, isMerge}); + return; + } + // If the lock is available, write directly to SQLite. if (m.try_lock()) { insertOrReplace(zoom, x, y, *data, isMerge); flushPendingStatements(); m.unlock(); - } else { + return; + } + + { // Else buffer the write for later, copying its binary blob. const std::lock_guard lock(pendingStatementsMutex); - pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + if (pendingBytes < MAX_PENDING_BYTES) { + pendingBytes += pendingCost(*data); + pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + return; + } } + + // The buffer is full: wait for SQLite, then help drain the buffer. + const std::lock_guard lock(m); + insertOrReplace(zoom, x, y, *data, isMerge); + flushPendingStatements(); } void MBTiles::closeForWriting() { + if (!batches.allWritten()) throw std::runtime_error("MBTiles: a batch of tiles was never written - please file an issue"); flushPendingStatements(); preparedStatements[0].used(true); preparedStatements[1].used(true); diff --git a/src/mmap_allocator.cpp b/src/mmap_allocator.cpp index 65818873..33ced307 100644 --- a/src/mmap_allocator.cpp +++ b/src/mmap_allocator.cpp @@ -1,9 +1,13 @@ #include "mmap_allocator.h" +#include +#include #include #include #include #include #include +#include +#include #include #include @@ -94,13 +98,15 @@ static mmap_dir_t mmap_dir; std::vector< mmap_shm_ptr > mmap_shm_regions; size_t mmap_shm::mmap_file_size = 0; -std::mutex mmap_allocator_mutex; +// Never destroyed (nor the two lists below): thread_regions_t's destructor uses them, and the +// main thread's thread-locals are destroyed after the statics on some platforms (macOS) +std::mutex &mmap_allocator_mutex = *new std::mutex; // Regions of threads that have exited, for reuse by new threads. Every thread // pool (one per .shp/GeoJSON layer, per .pbf read phase) starts new threads, and // a new region (or 1GB store file) for each of them would never be released. -std::vector free_mmap_files; -std::vector free_mmap_shm_regions; +std::vector &free_mmap_files = *new std::vector(); +std::vector &free_mmap_shm_regions = *new std::vector(); // The regions the current thread allocates from struct thread_regions_t @@ -191,6 +197,31 @@ bool mmap_dir_t::is_open() { if(!mmap_dir_filename.empty()) { try { + // The files of exited threads, kept for reuse: let them go first, so that they are + // deleted below (free_mmap_files is never destroyed) + { + std::lock_guard lock(mmap_allocator_mutex); + free_mmap_files.clear(); + } + // Unmapping and deleting a big store (planet: ~140 GB in ~300 files) + // takes half a minute on one thread, so release the files from several. + const size_t threads = std::min(files.size(), std::min(16u, std::max(1u, std::thread::hardware_concurrency()))); + if (threads > 1) { + std::atomic next(0); + std::vector workers; + for (size_t t = 0; t < threads; t++) { + try { + workers.emplace_back([this, &next]() { + for (size_t i; (i = next++) < files.size(); ) + files[i].reset(); + }); + } catch (const std::system_error &) { + break; // fewer threads; files.clear() below releases the rest + } + } + for (auto &worker : workers) + worker.join(); + } files.clear(); if(mmap_dir_created) { diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index ba34304e..d718d21e 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -142,9 +142,9 @@ kaguya::LuaTable getAllTags(kaguya::State& luaState, const boost::container::fla return tagsTable; } -std::string rawId() { return osmLuaProcessing->Id(); } -std::string rawOsmType() { return osmLuaProcessing->OsmType(); } -kaguya::LuaTable rawAllKeys() { +std::string rawId() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Id(); } +std::string rawOsmType() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->OsmType(); } +kaguya::LuaTable rawAllKeys() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); if (osmLuaProcessing->isPostScanRelation) { return osmLuaProcessing->AllKeys(*g_luaState); } @@ -152,7 +152,7 @@ kaguya::LuaTable rawAllKeys() { auto tags = osmLuaProcessing->currentTags->exportToBoostMap(); return getAllKeys(*g_luaState, &tags); -}kaguya::LuaTable rawAllTags() { +}kaguya::LuaTable rawAllTags() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); if (osmLuaProcessing->isPostScanRelation) { return osmLuaProcessing->AllTags(*g_luaState); } @@ -161,8 +161,8 @@ kaguya::LuaTable rawAllKeys() { return getAllTags(*g_luaState, &tags); } -bool rawHasTags() { return osmLuaProcessing->HasTags(); } -void rawSetTag(const std::string &key, const std::string &value) { return osmLuaProcessing->SetTag(key, value); } +bool rawHasTags() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->HasTags(); } +void rawSetTag(const std::string &key, const std::string &value) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->SetTag(key, value); } // Native lua_CFunction versions of Holds/Find, bypassing kaguya's invoker. // Semantics match the former kaguya-bound versions (KnownTagKey): kaguya does // no type check for a single (non-overloaded) function, so the argument is @@ -185,7 +185,9 @@ static int luaHolds(lua_State* L) { return 1; } - lua_pushboolean(L, osmLuaProcessing->currentTags->getKey(buffer, size) >= 0); + const bool held = osmLuaProcessing->currentTags->getKey(buffer, size) >= 0; + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Holds, buffer, size, held ? "1" : "0"); + lua_pushboolean(L, held); return 1; } @@ -211,8 +213,16 @@ static int luaFind(lua_State* L) { int64_t tagLoc = tags->getKey(buffer, size); if (tagLoc >= 0) { const protozero::data_view* value = tags->getValueFromKey(tagLoc); + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoValueTag(buffer, size)) osmLuaProcessing->memoReadValue(buffer, size, std::string(value->data(), value->size())); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Find, buffer, size, std::string(value->data(), value->size())); + } lua_pushlstring(L, value->data(), value->size()); } else { + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoValueTag(buffer, size)) osmLuaProcessing->memoReadValue(buffer, size, std::string()); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::Find, buffer, size, std::string()); + } lua_pushvalue(L, lua_upvalueindex(1)); } return 1; @@ -279,36 +289,70 @@ static void wrapKaguyaBinding(lua_State* L, const char* name, lua_CFunction fn) lua_pop(L, 1); } -std::vector rawFindIntersecting(const std::string &layerName) { return osmLuaProcessing->FindIntersecting(layerName); } -bool rawIntersects(const std::string& layerName) { return osmLuaProcessing->Intersects(layerName); } -std::vector rawFindCovering(const std::string& layerName) { return osmLuaProcessing->FindCovering(layerName); } -bool rawCoveredBy(const std::string& layerName) { return osmLuaProcessing->CoveredBy(layerName); } -bool rawIsClosed() { return osmLuaProcessing->IsClosed(); } -bool rawIsMultiPolygon() { return osmLuaProcessing->IsMultiPolygon(); } -double rawArea() { return osmLuaProcessing->Area(); } -double rawLength() { return osmLuaProcessing->Length(); } -kaguya::optional> rawCentroid(kaguya::VariadicArgType algorithm) { return osmLuaProcessing->Centroid(algorithm); } +std::vector rawFindIntersecting(const std::string &layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->FindIntersecting(layerName); } +bool rawIntersects(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Intersects(layerName); } +std::vector rawFindCovering(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->FindCovering(layerName); } +bool rawCoveredBy(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->CoveredBy(layerName); } +bool rawIsClosed() { + const bool closed = osmLuaProcessing->IsClosed(); + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::IsClosed, "", 0, closed ? "1" : "0"); + return closed; +} +bool rawIsMultiPolygon() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->IsMultiPolygon(); } +double rawMercatorArea() { + const double area = osmLuaProcessing->MercatorArea(); + if (osmLuaProcessing->memoTrace.active) { + if (osmLuaProcessing->memoArea.empty()) osmLuaProcessing->memoLuaOnly(); + else osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::MercatorArea, "", 0, osmLuaProcessing->memoAreaResult(area)); + } + return area; +} +void rawMinZoomByArea(double limit, std::vector thresholds) { return osmLuaProcessing->MinZoomByArea(limit, thresholds); } +double rawArea() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Area(); } +double rawLength() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Length(); } +kaguya::optional> rawCentroid(kaguya::VariadicArgType algorithm) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Centroid(algorithm); } void rawModifyId(const int newId) { return osmLuaProcessing->ModifyId(newId); } void rawLayer(const std::string& layerName, bool area) { return osmLuaProcessing->Layer(layerName, area); } void rawLayerAsCentroid(const std::string &layerName, kaguya::VariadicArgType nodeSources) { return osmLuaProcessing->LayerAsCentroid(layerName, nodeSources); } void rawMinZoom(const double z) { return osmLuaProcessing->MinZoom(z); } void rawZOrder(const double z) { return osmLuaProcessing->ZOrder(z); } void rawScore(const double score) { return osmLuaProcessing->Score(score); } -OsmLuaProcessing::OptionalRelation rawNextRelation() { return osmLuaProcessing->NextRelation(); } -void rawRestartRelations() { return osmLuaProcessing->RestartRelations(); } -std::string rawFindInRelation(const std::string& key) { return osmLuaProcessing->FindInRelation(key); } -void rawAccept() { return osmLuaProcessing->Accept(); } -double rawAreaIntersecting(const std::string& layerName) { return osmLuaProcessing->AreaIntersecting(layerName); } - -void rawSetData(const std::string &key, const std::string &value) { +OsmLuaProcessing::OptionalRelation rawNextRelation() { + OsmLuaProcessing::OptionalRelation r = osmLuaProcessing->NextRelation(); + if (osmLuaProcessing->memoTrace.active) { + // way_memo_relations: the read is whether there is a relation, and its role + osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::NextRelation, "", 0, r.done ? "0" : osmLuaProcessing->memoRelations ? "1" + r.role : "1"); + if (!r.done && !osmLuaProcessing->memoRelations) osmLuaProcessing->memoLuaOnly(); + } + return r; +} +void rawRestartRelations() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->RestartRelations(); } +std::string rawFindInRelation(const std::string& key) { + if (osmLuaProcessing->memoTrace.active && !(osmLuaProcessing->memoRelations && osmLuaProcessing->relationCurrent())) osmLuaProcessing->memoLuaOnly(); + std::string value = osmLuaProcessing->FindInRelation(key); + if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoRead(OsmLuaProcessing::MemoNode::FindInRelation, key.data(), key.size(), value); + return value; +} +void rawAccept() { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->Accept(); } +double rawAreaIntersecting(const std::string& layerName) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); return osmLuaProcessing->AreaIntersecting(layerName); } + +void rawSetData(const std::string &key, const std::string &value) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); std::lock_guard lock(osmLuaProcessing->dataStoreMutex); osmLuaProcessing->dataStore[key] = value; } -std::string rawGetData(const std::string &key) { +std::string rawGetData(const std::string &key) { if (osmLuaProcessing->memoTrace.active) osmLuaProcessing->memoLuaOnly(); auto r = osmLuaProcessing->dataStore.find(key); return r==osmLuaProcessing->dataStore.end() ? "" : r->second; } +static std::unordered_map> readMemoValueTags(lua_State *L, const char *global, std::unordered_map &lengths); +static std::vector readMemoArea(lua_State *L, const char *global); + +// Memo of way_function: the processors that use it (for printMemoStats) +static std::mutex memoInstancesMutex; +static std::vector memoInstances; +static uint64_t memoDone[OsmLuaProcessing::MEMO_KINDS][4] = {}; // of processors gone, by MemoKind: hits, recorded, through Lua, tree nodes + bool supportsRemappingShapefiles = false; int lua_error_handler(int errCode, const char *errMessage) @@ -453,6 +497,8 @@ OsmLuaProcessing::OsmLuaProcessing( luaState["IsClosed"] = &rawIsClosed; luaState["IsMultiPolygon"] = &rawIsMultiPolygon; luaState["Area"] = &rawArea; + luaState["MercatorArea"] = &rawMercatorArea; + luaState["MinZoomByArea"] = &rawMinZoomByArea; luaState["AreaIntersecting"] = &rawAreaIntersecting; luaState["Length"] = &rawLength; luaState["Centroid"] = &rawCentroid; @@ -497,6 +543,31 @@ OsmLuaProcessing::OsmLuaProcessing( supportsWritingNodes = !!luaState["node_function"]; supportsWritingWays = !!luaState["way_function"]; supportsWritingRelations = !!luaState["relation_function"]; + memoEnabled = !!luaState["way_memo"]; + if (memoEnabled) { + memoValueTags = readMemoValueTags(luaState.state(), "way_memo_values", memoValueLengths); + memoArea = readMemoArea(luaState.state(), "way_memo_area"); + memoRelations = !!luaState["way_memo_relations"]; + } + nodeMemoEnabled = supportsWritingNodes && !!luaState["node_memo"]; + MemoState &nodeMemo = memoSaved[MemoNodes]; + nodeMemo.enabled = nodeMemoEnabled; + if (nodeMemoEnabled) { + nodeMemo.valueTags = readMemoValueTags(luaState.state(), "node_memo_values", nodeMemo.valueLengths); + nodeMemo.area = readMemoArea(luaState.state(), "node_memo_area"); + nodeMemo.relations = !!luaState["node_memo_relations"]; + } + // Multipolygon relations run way_function: its memo, with a tree of their own + MemoState &relationMemo = memoSaved[MemoRelations]; + relationMemo.enabled = memoEnabled; + relationMemo.valueTags = memoValueTags; + relationMemo.valueLengths = memoValueLengths; + relationMemo.area = memoArea; + relationMemo.relations = memoRelations; + if (memoEnabled || nodeMemoEnabled) { + std::lock_guard lock(memoInstancesMutex); + memoInstances.push_back(this); + } // ---- Call init_function of Lua logic @@ -506,6 +577,14 @@ OsmLuaProcessing::OsmLuaProcessing( } OsmLuaProcessing::~OsmLuaProcessing() { + { + std::lock_guard lock(memoInstancesMutex); + auto it = std::find(memoInstances.begin(), memoInstances.end(), this); + if (it != memoInstances.end()) { + memoInstances.erase(it); + memoCounts(memoDone); + } + } // Call exit_function of Lua logic luaState("if exit_function~=nil then exit_function() end"); } @@ -731,6 +810,34 @@ double OsmLuaProcessing::projectedPolygonArea(const Polygon &p) { } #endif +// Geometries are kept in Web Mercator (lon, latp in degrees), so the planar area only needs scaling +double OsmLuaProcessing::MercatorArea() { + if (!IsClosed()) return 0; + const double metresPerDegree = 6378137.0 * M_PI / 180.0; // Web Mercator, at the equator + double area = 0; + if (isRelation) { + for (const Polygon &p : multiPolygonCached()) area += std::abs(geom::area(p)); + } else if (isWay) { + area = std::abs(geom::area(polygonCached())); + } + return area * metresPerDegree * metresPerDegree; +} + +void OsmLuaProcessing::MinZoomByArea(double limit, const std::vector &thresholds) { + if (memoTrace.active) { + MemoOp op{ MemoOp::MinZoomByArea, false, 0, limit, std::string(), std::string() }; + op.numbers = thresholds; + memoOp(std::move(op)); + } + if (outputs.size()==0) { ProcessingError("Can't set minimum zoom if no Layer set"); return; } + const double area = MercatorArea(); + double z = thresholds.size() + 1; + for (std::size_t i = 0; i < thresholds.size(); i++) { + if (limit <= i + 1 && area > thresholds[i]) { z = i + 1; break; } + } + outputs.back().first.setMinZoom(z); +} + // Returns area double OsmLuaProcessing::Area() { if (!IsClosed()) return 0; @@ -788,10 +895,12 @@ const Linestring &OsmLuaProcessing::linestringCached() { if (!linestringInited) { linestringInited = true; // Same as osmStore.llListLinestring(), refilling the cache in place - linestringCache.clear(); - linestringCache.reserve(llVecPtr->size()); - for (const LatpLon &ll : *llVecPtr) - linestringCache.push_back(geom::make(ll.lon/10000000.0, ll.latp/10000000.0)); + const size_t n = llVecPtr->size(); + linestringCache.resize(n); + const LatpLon *src = llVecPtr->data(); + Point *dst = linestringCache.data(); + for (size_t i = 0; i < n; i++) + dst[i] = geom::make(src[i].lon/10000000.0, src[i].latp/10000000.0); } return linestringCache; } @@ -807,7 +916,18 @@ const MultiLinestring &OsmLuaProcessing::multiLinestringCached() { const Polygon &OsmLuaProcessing::polygonCached() { if (!polygonInited) { polygonInited = true; - polygonCache = osmStore.llListPolygon(llVecPtr->cbegin(), llVecPtr->cend()); + // As osmStore.llListPolygon(), refilling the cache in place: the points of + // linestringCached(), then corrected + auto &outer = polygonCache.outer(); + polygonCache.inners().clear(); + if (linestringInited) outer.assign(linestringCache.begin(), linestringCache.end()); + else { + outer.clear(); + outer.reserve(llVecPtr->size()); + for (const LatpLon &ll : *llVecPtr) + outer.push_back(geom::make(ll.lon/10000000.0, ll.latp/10000000.0)); + } + geom::correct(polygonCache); } return polygonCache; } @@ -826,12 +946,19 @@ const MultiPolygon &OsmLuaProcessing::multiPolygonCached() { // Add object to specified layer from Lua void OsmLuaProcessing::Layer(const string &layerName, bool area) { + if (memoTrace.active) memoOp({ MemoOp::Layer, area, 0, 0, layerName, std::string() }); outputKeys.clear(); - if (layers.layerMap.count(layerName) == 0) { + const auto layer = layers.layerMap.find(layerName); + if (layer == layers.layerMap.end()) { throw out_of_range("ERROR: Layer(): a layer named as \"" + layerName + "\" doesn't exist."); } + layerAt(layer->second, area); +} - uint layerMinZoom = layers.layers[layers.layerMap[layerName]].minzoom; +// Layer(), with the layer's index +void OsmLuaProcessing::layerAt(unsigned layerIndex, bool area) { + outputKeys.clear(); + uint layerMinZoom = layers.layers[layerIndex].minzoom; AttributeSet attributes; OutputGeometryType geomType = isRelation ? (area ? POLYGON_ : MULTILINESTRING_ ) : isWay ? (area ? POLYGON_ : LINESTRING_) : POINT_; @@ -840,7 +967,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { // a highway and its name. Avoid duplicating geometry processing and storage // when this occurs. if (lastStoredGeometryId != 0 && lastStoredGeometryType == geomType) { - OutputObject oo(geomType, layers.layerMap[layerName], lastStoredGeometryId, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, lastStoredGeometryId, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); return; } @@ -853,7 +980,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { NodeID id = USE_NODE_STORE | originalOsmID; if (materializeGeometries) id = osmMemTiles.storePoint(p); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); noteIfDecluttered(LatpLon { latp, lon }); return; @@ -863,15 +990,28 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { if (isRelation) { try { - MultiPolygon mp = multiPolygonCached(); - auto correctionResult = CorrectGeometry(mp); + // Checked in the cache, copied only when it needs correcting (as CorrectGeometry) + const MultiPolygon &cached = multiPolygonCached(); + bool reversedValid = false; + const geom::validity_failure_type failure = CheckGeometry(cached, &reversedValid); + if (failure == geom::validity_failure_type::no_failure) { + NodeID id = osmMemTiles.storeMultiPolygon(cached, true); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); + outputs.push_back(std::make_pair(std::move(oo), attributes)); + return; + } + MultiPolygon mp = cached; + auto correctionResult = CorrectGeometryAfterCheck(mp, failure, false, reversedValid); if(correctionResult == CorrectGeometryResult::Invalid) return; // make_valid usually succeeds; checking that once here saves // re-validating the relation in every tile it is clipped to. - const bool knownValid = correctionResult == CorrectGeometryResult::Valid || - PrecheckValid(mp) || geom::is_valid(mp); + // A big polygon is checked in the background until the tiles are written. + const bool checked = correctionResult == CorrectGeometryResult::Valid || PrecheckValid(mp); + const bool checkLater = !checked && geom::num_points(mp) >= 100000; + const bool knownValid = checked || (!checkLater && IsValidIndexed(mp)); NodeID id = osmMemTiles.storeMultiPolygon(mp, knownValid); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + if (checkLater) osmMemTiles.checkValidityLater(id, std::move(mp)); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } catch(std::out_of_range &err) { cout << "In relation " << originalOsmID << ": " << err.what() << endl; @@ -899,7 +1039,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { wayEmitted = true; } else id = osmMemTiles.storeMultiPolygon(mp, correctionResult != CorrectGeometryResult::Corrected); - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } @@ -918,7 +1058,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { NodeID id = osmMemTiles.storeMultiLinestring(mls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } else if (geomType==LINESTRING_) { @@ -944,13 +1084,13 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { id = osmMemTiles.storeLinestring(ls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } else { NodeID id = osmMemTiles.storeLinestring(ls); lastStoredGeometryId = id; lastStoredGeometryType = geomType; - OutputObject oo(geomType, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(geomType, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); } } @@ -969,24 +1109,40 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { // When called for a relation, you can pass a list of roles. The point of a node // with that role will be used if available. void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::VariadicArgType varargs) { - outputKeys.clear(); if (layers.layerMap.count(layerName) == 0) { throw out_of_range("ERROR: LayerAsCentroid(): a layer named as \"" + layerName + "\" doesn't exist."); - } - + } + // The first argument is the algorithm, the others relation roles to look for CentroidAlgorithm algorithm = defaultCentroidAlgorithm(); - + std::vector relationRoles; + bool first = true; for (auto needleRef : varargs) { const std::string needle = needleRef.get(); - algorithm = parseCentroidAlgorithm(needle); - break; + if (first) algorithm = parseCentroidAlgorithm(needle); + else relationRoles.push_back(needle); + first = false; + } + if (memoTrace.active) { + if (isRelation && !relationRoles.empty()) memoLuaOnly(); + else memoOp({ MemoOp::Centroid, false, 0, static_cast(static_cast(algorithm)), layerName, std::string() }); } + layerAsCentroid(layerName, algorithm, relationRoles); +} + +void OsmLuaProcessing::layerAsCentroid(const std::string &layerName, CentroidAlgorithm algorithm, const std::vector &relationRoles) { + outputKeys.clear(); + layerAsCentroidAt(layers.layerMap[layerName], algorithm, relationRoles); +} + +// layerAsCentroid(), with the layer's index +void OsmLuaProcessing::layerAsCentroidAt(unsigned layerIndex, CentroidAlgorithm algorithm, const std::vector &relationRoles) { + outputKeys.clear(); // This will be non-zero if we ultimately used a node from a relation to // label the point. NodeID relationNode = 0; - uint layerMinZoom = layers.layers[layers.layerMap[layerName]].minzoom; + uint layerMinZoom = layers.layers[layerIndex].minzoom; AttributeSet attributes; Point geomp; bool centroidFound = false; @@ -994,12 +1150,7 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::Variadic // If we're a relation, see if the user would prefer we use one of its members // to label the point. if (isRelation) { - int i = -1; - for (auto needleRef : varargs) { - i++; - // Skip the first vararg, it's the algorithm. - if (i == 0) continue; - const std::string needle = needleRef.get(); + for (const std::string &needle : relationRoles) { // We do a linear search of the relation's members. This is not very efficient // for relations like Tongass National Park (ID 6535292, 29,000 members), @@ -1071,7 +1222,7 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::Variadic id = USE_WAY_STORE | originalOsmID; wayEmitted = true; } - OutputObject oo(POINT_, layers.layerMap[layerName], id, 0, layerMinZoom); + OutputObject oo(POINT_, layerIndex, id, 0, layerMinZoom); outputs.push_back(std::make_pair(std::move(oo), attributes)); noteIfDecluttered(LatpLon { (int32_t)geomp.y(), (int32_t)geomp.x() }); } @@ -1200,11 +1351,13 @@ void OsmLuaProcessing::removeAttributeIfNeeded(uint16_t keyIndex) { // Force a new ID void OsmLuaProcessing::ModifyId(const int newId) { + if (memoTrace.active) memoOp({ MemoOp::ModifyId, false, 0, static_cast(newId), std::string(), std::string() }); originalOsmID = static_cast(newId); } // Set attributes in a vector tile's Attributes table void OsmLuaProcessing::Attribute(const string &key, const protozero::data_view val, const char minzoom) { + if (memoTrace.active) memoStringAttribute({ MemoOp::Attr, false, minzoom, 0, key, std::string(val.data(), val.size()) }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1213,6 +1366,7 @@ void OsmLuaProcessing::Attribute(const string &key, const protozero::data_view v } void OsmLuaProcessing::AttributeNumeric(const string &key, const double val, const char minzoom) { + if (memoTrace.active) memoNumberAttribute({ MemoOp::AttrNumeric, false, minzoom, static_cast(val), key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1221,6 +1375,7 @@ void OsmLuaProcessing::AttributeNumeric(const string &key, const double val, con } void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val, const char minzoom) { + if (memoTrace.active) memoOp({ MemoOp::AttrBoolean, val, minzoom, 0, key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1229,6 +1384,7 @@ void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val, const } void OsmLuaProcessing::AttributeInteger(const string &key, const int val, const char minzoom) { + if (memoTrace.active) memoNumberAttribute({ MemoOp::AttrInteger, false, minzoom, static_cast(val), key, std::string() }); if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } const uint16_t keyIndex = attributeStore.keyStore.key2index(key); removeAttributeIfNeeded(keyIndex); @@ -1238,12 +1394,14 @@ void OsmLuaProcessing::AttributeInteger(const string &key, const int val, const // Set minimum zoom void OsmLuaProcessing::MinZoom(const double z) { + if (memoTrace.active) memoOp({ MemoOp::MinZoom, false, 0, z, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set minimum zoom if no Layer set"); return; } outputs.back().first.setMinZoom(z); } // Set the score used to rank this feature against its neighbours when decluttering void OsmLuaProcessing::Score(const double score) { + if (memoTrace.active) memoOp({ MemoOp::Score, false, 0, score, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set score if no Layer set"); return; } if (declutterOutputs.empty() || declutterOutputs.back().outputIndex != outputs.size()-1) { ProcessingError("Score() only applies to point features in a layer with declutter_below set"); @@ -1254,6 +1412,7 @@ void OsmLuaProcessing::Score(const double score) { // Set z_order void OsmLuaProcessing::ZOrder(const double z) { + if (memoTrace.active) memoOp({ MemoOp::ZOrder, false, 0, z, std::string(), std::string() }); if (outputs.size()==0) { ProcessingError("Can't set z_order if no Layer set"); return; } outputs.back().first.setZOrder(z); } @@ -1369,6 +1528,15 @@ template inline void addAttributeByIndex(AttributeStore& store, Attr // the key index and vector_layers bookkeeping served from the key cache. template void OsmLuaProcessing::attributeFast(const char *key, size_t keyLen, T val, char minzoom, uint type) { + if (memoTrace.active) { + MemoOp op{ static_cast(MemoOp::FastStr + type), false, minzoom, 0, std::string(key, keyLen), std::string() }; + if constexpr (std::is_same::value) op.str.assign(val.data(), val.size()); + else if constexpr (std::is_same::value) op.flag = val; + else op.number = static_cast(val); + if constexpr (std::is_same::value) memoStringAttribute(std::move(op)); + else if constexpr (std::is_same::value) memoOp(std::move(op)); + else memoNumberAttribute(std::move(op)); + } if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } AttrKeyCacheEntry& e = lookupAttrKey(attributeStore.keyStore, key, keyLen); removeAttributeIfNeeded(e.keyIndex); @@ -1460,7 +1628,8 @@ bool OsmLuaProcessing::setNode(NodeID id, LatpLon node, const TagMap& tags) { //Start Lua processing for node try { - if (!callLuaGlobal("node_function", true)) luaState["node_function"](); + if (nodeMemoEnabled) { memoSelect(true); runWayFunction(); } + else if (!callLuaGlobal("node_function", true)) luaState["node_function"](); } catch(luaProcessingException &e) { std::cerr << "Lua error on node " << originalOsmID << std::endl; exit(1); @@ -1491,8 +1660,9 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap linestringInited = polygonInited = multiPolygonInited = false; centroidMemoValid[0] = centroidMemoValid[1] = false; - if (supportsReadingRelations && osmStore.scannedRelations.way_in_any_relations(wayId)) { - relationList = osmStore.scannedRelations.relations_for_way(wayId); + if (supportsReadingRelations) { + if (const auto *relations = osmStore.scannedRelations.findWayRelations(wayId, wayRelationsCursor)) + relationList = *relations; } try { @@ -1510,11 +1680,8 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap if (ok) { //Start Lua processing for way try { - if (!callLuaGlobal("way_function", false)) { - kaguya::LuaFunction way_function = luaState["way_function"]; - kaguya::LuaRef ret = way_function(); - assert(!ret); - } + memoSelect(false); + runWayFunction(); } catch(luaProcessingException &e) { std::cerr << "Lua error on way " << originalOsmID << std::endl; exit(1); @@ -1561,7 +1728,8 @@ void OsmLuaProcessing::setRelation( if (!isNativeMP && !supportsWritingRelations) return; try { if (isNativeMP && supportsWritingWays) { - if (!callLuaGlobal("way_function", true)) luaState["way_function"](); + memoSelect(MemoRelations); + runWayFunction(); } else if (!isNativeMP && supportsWritingRelations) { if (!callLuaGlobal("relation_function", true)) luaState["relation_function"](); } @@ -1610,7 +1778,22 @@ std::vector &OsmLuaProcessing::finalizeOutputs() { size_t nextDeclutter = 0; for (size_t i = 0; i < this->outputs.size(); i++) { auto& output = this->outputs[i]; - output.first.setAttributeSet(attributeStore.add(output.second)); + if (i < outputPreset.size() && outputPreset[i] != UINT32_MAX) { + output.first.setAttributeSet(outputPreset[i]); + attributeStore.noteLookup(); + } else { + const AttributeIndex index = attributeStore.add(output.second); + output.first.setAttributeSet(index); + if (i < outputLearn.size() && outputLearn[i] != nullptr) { + MemoOp &learn = *outputLearn[i]; + if (learn.segment & MemoOp::SegmentConstant) learn.segmentSet = index; + else { + char sig[MemoSigCache::SIG_MAX]; + const size_t len = memoSegmentSig(learn, sig); + memoSegmentSets.put(MemoSigCache::hashOf(sig, len), sig, len, index); + } + } + } if (nextDeclutter < declutterOutputs.size() && declutterOutputs[nextDeclutter].outputIndex == i) { // Park it: Declutter::apply() will set its minimum zoom and index it later const auto& pending = declutterOutputs[nextDeclutter++]; @@ -1621,3 +1804,522 @@ std::vector &OsmLuaProcessing::finalizeOutputs() { } return list; } + +// ---- Memo of way_function (see osm_lua_processing.h) + +// A node branching more often than this reads something that is different for most ways (a name, +// a house number): from there on the ways run through Lua. Values such as building heights or +// rare building types are many, but each comes often enough to pay for its path (planet: 256 +// replays ~60% of the ways 64 left to Lua; more gains little). At most 65535. +static const size_t MEMO_MAX_CHILDREN = 256; +// per thread +static const size_t MEMO_MAX_NODES = 1 << 18; +// A full tree is compacted (memoCompact) after this many objects it had no room for +static const size_t MEMO_COMPACT_MISSES = std::max(MEMO_MAX_NODES / 64, 16); + +void OsmLuaProcessing::printMemoStats() { + std::lock_guard lock(memoInstancesMutex); + uint64_t counts[MEMO_KINDS][4]; + for (int k = 0; k < MEMO_KINDS; k++) for (int i = 0; i < 4; i++) counts[k][i] = memoDone[k][i]; + for (auto *p : memoInstances) p->memoCounts(counts); + const uint64_t *way = counts[MemoWays], *node = counts[MemoNodes], *rel = counts[MemoRelations]; + if (way[0] + way[1] + way[2] != 0) + std::cout << "way_function memo: " << way[0] << " replayed, " << way[1] << " recorded, " << way[2] << " through Lua, " << way[3] << " nodes" << std::endl; + if (node[0] + node[1] + node[2] != 0) + std::cout << "node_function memo: " << node[0] << " replayed, " << node[1] << " recorded, " << node[2] << " through Lua, " << node[3] << " nodes" << std::endl; + if (rel[0] + rel[1] + rel[2] != 0) + std::cout << "way_function memo (multipolygons): " << rel[0] << " replayed, " << rel[1] << " recorded, " << rel[2] << " through Lua, " << rel[3] << " nodes" << std::endl; +} + +void OsmLuaProcessing::runWayFunctionInLua(bool record) { + if (record) { + memoTrace.luaOnly = false; + memoTrace.ambiguous = false; + memoTrace.reads.clear(); + memoTrace.ops.clear(); + memoTrace.emptyValues.clear(); + memoTrace.values.clear(); + } + memoTrace.active = record; + if (memoForNodes) { + // as setNode called it before the memo + if (!callLuaGlobal("node_function", true)) luaState["node_function"](); + } else if (!callLuaGlobal("way_function", false)) { + kaguya::LuaFunction way_function = luaState["way_function"]; + kaguya::LuaRef ret = way_function(); + assert(!ret); + } + memoTrace.active = false; + if (record) memoInsert(); +} + +// The walk reads the way's tags through memoSlots (filled once per way from the tags as added, +// see memo_keys.h), which answers as currentTags->getKey/getValueFromKey would. +void OsmLuaProcessing::runWayFunction() { + if (!memoEnabled) { runWayFunctionInLua(false); return; } + if (memoNodes.empty()) memoNodes.emplace_back(); + const bool canRecord = memoNodes.size() < MEMO_MAX_NODES; + for (size_t id = memoKeys.valueOnly.size(); id < memoKeys.size(); id++) + memoKeys.valueOnly.push_back(memoValueTag(memoKeys.name(id).data(), memoKeys.name(id).size())); + // IsClosed() as Lua sees it: false for a node + const bool closed = IsClosed(), hasRelations = !relationList.empty(); + char sig[MemoSigCache::SIG_MAX]; + const size_t sigLen = memoSlots.fill(memoKeys, currentTags->added(), uint8_t(closed) | uint8_t(hasRelations) << 1, sig, sizeof(sig)); + const uint64_t sigHash = sigLen ? MemoSigCache::hashOf(sig, sigLen) : 0; + uint32_t last = 0; + uint32_t n = sigLen ? memoWalkCache.find(sigHash, sig, sigLen) : UINT32_MAX; + if (n == UINT32_MAX) { + // The reads that aren't tags, as rawNextRelation, rawMercatorArea, rawFindInRelation record + // them. A walk that made one isn't a function of the signature: it isn't cached. + struct Extra { + OsmLuaProcessing &p; + bool uncached = false; + int relation = -1; // as relationSubscript + std::string result; + std::string_view nextRelation() { + if (!p.memoRelations) return "1"; // such a way runs through Lua: no reads after this one + uncached = true; + if (size_t(++relation) >= p.relationList.size()) return "0"; + result = "1" + p.osmStore.scannedRelations.getRole(p.relationList[relation].second); + return result; + } + std::string_view area() { + uncached = true; + result = p.memoAreaResult(p.MercatorArea()); + return result; + } + std::string_view findInRelation(uint32_t key) { + uncached = true; + if (relation < 0 || size_t(relation) >= p.relationList.size()) return "\xff"; // not recorded: no child + result = p.osmStore.scannedRelations.get_relation_tag(p.relationList[relation].first, p.memoKeys.name(key)); + return result; + } + } extra{ *this }; + n = memoWalk(memoNodes.data(), memoSlots, closed, hasRelations, [this](uint32_t k, std::string_view r) { + const MemoHotNode &node = memoNodes[k]; + return node.more == UINT32_MAX ? UINT32_MAX : memoMore[node.more].child(r); + }, extra, last); + if (sigLen && !extra.uncached && n != UINT32_MAX && (memoNodes[n].kind == MemoNode::Leaf || memoNodes[n].kind == MemoNode::Lua)) + memoWalkCache.put(sigHash, sig, sigLen, n); + } + if (n == UINT32_MAX && memoNodes[last].overflow) { memoLua++; runWayFunctionInLua(false); return; } + if (n != UINT32_MAX && memoNodes[n].kind == MemoNode::Leaf) { + MemoMore &leaf = memoMore[memoNodes[n].more]; + if (leaf.uses != UINT32_MAX) leaf.uses++; + memoHits++; memoReplay(leaf.ops); return; + } + if (n != UINT32_MAX && memoNodes[n].kind == MemoNode::Lua) { + MemoMore &end = memoMoreOf(n); + if (end.uses != UINT32_MAX) end.uses++; + memoLua++; runWayFunctionInLua(false); return; + } + // no child for the result read, or Unset: record, if the tree has room + memoRecorded++; + if (!canRecord && ++memoFullMisses >= MEMO_COMPACT_MISSES) { + memoCompact(); + runWayFunctionInLua(memoNodes.size() < MEMO_MAX_NODES); + return; + } + runWayFunctionInLua(canRecord); +} + +// The tree is full, and objects keep taking paths it hasn't got: keep only the paths that objects +// have taken since the last time (Leaf and Lua nodes used, and the reads to them), so that the +// paths of the objects to come can be recorded. The tree then is as if those paths had been +// recorded alone (in the same order). +void OsmLuaProcessing::memoCompact() { + const size_t size = memoNodes.size(); + std::vector remap(size, UINT32_MAX); + auto keptChild = [&](uint32_t c) { return c != UINT32_MAX && remap[c] != UINT32_MAX; }; + // A child has a higher index than its parent: going backwards, the children are decided + std::vector keep(size, 0); + for (size_t i = size; i-- > 0; ) { + const MemoHotNode &node = memoNodes[i]; + bool k = i == 0; + if (node.kind == MemoNode::Leaf || node.kind == MemoNode::Lua) k = k || (node.more != UINT32_MAX && memoMore[node.more].uses > 0); + else if (node.kind != MemoNode::Unset) { + for (int q = 0; q < 4 && !k; q++) k = node.quick[q] != UINT32_MAX && keep[node.quick[q]]; + if (!k && node.more != UINT32_MAX) + for (const auto &c : memoMore[node.more].children) if (keep[c.second]) { k = true; break; } + } + keep[i] = k; + } + uint32_t next = 0; + for (size_t i = 0; i < size; i++) if (keep[i]) remap[i] = next++; + std::vector nodes; + std::vector more; + nodes.reserve(next); + for (size_t i = 0; i < size; i++) { + if (!keep[i]) continue; + const MemoHotNode &old = memoNodes[i]; + MemoHotNode node; + node.kind = old.kind; node.valueOnly = old.valueOnly; node.overflow = old.overflow; node.key = old.key; + node.children = 0; + for (int q = 0; q < 4; q++) + if (keptChild(old.quick[q])) { node.quick[q] = remap[old.quick[q]]; node.children++; } + if (old.more != UINT32_MAX) { + MemoMore &om = memoMore[old.more]; + MemoMore m; + for (const auto &c : om.children) + if (keptChild(c.second)) { m.addChild(c.first, remap[c.second]); node.children++; } + m.ops = std::move(om.ops); + if (!m.children.empty() || !m.ops.empty() || node.kind == MemoNode::Leaf || node.kind == MemoNode::Lua) { + node.more = more.size(); + more.push_back(std::move(m)); + } + } + // An overflowed node that lost children takes new results again (the results used often + // stay: their paths were used, so kept) + if (node.children < MEMO_MAX_CHILDREN) node.overflow = false; + nodes.push_back(node); + } + memoNodes.swap(nodes); + memoMore.swap(more); + memoWalkCache.clear(); + memoComputeJumps(memoNodes.data(), memoNodes.size()); + memoJumpsAt = memoNodes.size(); + memoFullMisses = 0; +} + +// Add the trace just recorded to the tree +void OsmLuaProcessing::memoInsert() { + if (memoTrace.ambiguous && !memoTrace.luaOnly) return; + auto nondeterministic = [&]() { + memoEnabled = false; + if (memoForNodes) std::cerr << "node_memo: node_function read differently for the same input (node " << originalOsmID << "); memo switched off for this thread" << std::endl; + else if (memoKind == MemoRelations) std::cerr << "way_memo: way_function read differently for the same input (relation " << originalOsmID << "); memo switched off for relations on this thread" << std::endl; + else std::cerr << "way_memo: way_function read differently for the same input (way " << originalOsmID << "); memo switched off for this thread" << std::endl; + }; + uint32_t n = 0; + const size_t sizeBefore = memoNodes.size(); + for (const MemoRead &r : memoTrace.reads) { + // key ids: equal ids <=> equal keys (IsClosed, NextRelation, MercatorArea: no key) + const uint32_t key = r.kind == MemoNode::Find || r.kind == MemoNode::Holds || r.kind == MemoNode::FindInRelation ? memoKeys.add(r.key) : MemoKeyTable::NONE; + if (memoNodes[n].kind == MemoNode::Unset) { memoNodes[n].kind = r.kind; memoNodes[n].key = key; memoNodes[n].valueOnly = r.valueOnly; } + else if (memoNodes[n].kind == MemoNode::Lua) return; + else if (memoNodes[n].kind != r.kind || memoNodes[n].key != key || memoNodes[n].valueOnly != r.valueOnly) { nondeterministic(); return; } + uint32_t next = memoChild(n, r.result); + if (next == UINT32_MAX) { + if (memoNodes[n].children >= MEMO_MAX_CHILDREN) { + memoNodes[n].overflow = true; + return; + } + next = memoNodes.size(); + memoNodes.emplace_back(); + memoAddChild(n, r.result, next); + } + n = next; + } + MemoHotNode &end = memoNodes[n]; + if (end.kind == MemoNode::Unset) { + if (memoTrace.luaOnly) end.kind = MemoNode::Lua; + else { + end.kind = MemoNode::Leaf; + std::vector &ops = memoMoreOf(n).ops; + ops = memoTrace.ops; + for (MemoOp &op : ops) if (!op.valueKey.empty()) op.valueKeyId = memoKeys.add(op.valueKey); + } + } else if (end.kind != MemoNode::Lua && (memoTrace.luaOnly || end.kind != MemoNode::Leaf)) { + nondeterministic(); + } + // The jumps of the nodes just added (their children are newer: none yet), and those of all + // nodes when the tree has doubled since the last time (geometrically: O(nodes) in all) and once + // more when it is full, so that older nodes' jumps reach into the paths added since + if (memoNodes.size() >= memoJumpsAt * 2 + 64 || (memoNodes.size() >= MEMO_MAX_NODES && memoJumpsAt < memoNodes.size())) { + memoComputeJumps(memoNodes.data(), memoNodes.size()); + memoJumpsAt = memoNodes.size(); + } else if (memoNodes.size() > sizeBefore) memoComputeJumps(memoNodes.data(), memoNodes.size(), sizeBefore); +} + +// Make the calls way_function made +// +// The calls of one leaf are the same every time, but for the values of way_memo_values tags +// (valueKey) and whether a Layer/Centroid call adds an output (that depends on the geometry). So the +// layer index, key index and pair of an attribute op without a valueKey are worked out once and +// kept in the op (the pair index for the same pair never changes). An output gets the attribute +// ops after its layer op up to the next layer op (its segment), and those of following layer ops +// that add no output. If its segment has no valueKey ops, and it gets no others, its attribute +// set is always the same, and so its AttributeIndex (one per distinct set): the first time it is +// kept in the layer op (segmentSet), and later replays skip the segment's attribute ops and the +// set lookup. Should a later layer op add no output but have attribute ops, the skipped ops are +// made after all (as they were, with outputKeys as Layer left it), and then the later ones. +static inline bool memoIsLayerOp(OsmLuaProcessing::MemoOp::Kind k) { return k == OsmLuaProcessing::MemoOp::Layer || k == OsmLuaProcessing::MemoOp::Centroid; } +static inline bool memoIsAttributeOp(OsmLuaProcessing::MemoOp::Kind k) { return k >= OsmLuaProcessing::MemoOp::Attr && k <= OsmLuaProcessing::MemoOp::FastInteger; } + +void OsmLuaProcessing::memoReplay(std::vector &ops) { + bool skipping = false; // the attribute ops are those of outputs.back()'s segment, whose AttributeIndex is known + for (size_t i = 0; i < ops.size(); i++) { + MemoOp &op = ops[i]; + if (memoIsLayerOp(op.kind)) { + if (op.layerIndex < 0) { + const auto layer = layers.layerMap.find(op.key); + if (layer == layers.layerMap.end()) { Layer(op.key, op.flag); continue; } // throws + op.layerIndex = layer->second; + } + if (!(op.segment & MemoOp::SegmentKnown)) { + uint8_t s = MemoOp::SegmentKnown | MemoOp::SegmentConstant; + size_t j = i + 1; + for (; j < ops.size() && !memoIsLayerOp(ops[j].kind); j++) + if (memoIsAttributeOp(ops[j].kind)) { + s |= MemoOp::SegmentAttributes; + if (!ops[j].valueKey.empty()) s &= ~MemoOp::SegmentConstant; + } + op.segment = s; + op.segmentOps = j - i - 1; + op.segmentId = ++memoSegmentIds; + } + const size_t before = outputs.size(); + if (op.kind == MemoOp::Layer) layerAt(op.layerIndex, op.flag); + else layerAsCentroidAt(op.layerIndex, static_cast(static_cast(op.number)), {}); + if (outputs.size() > before) { + outputPreset.resize(before, UINT32_MAX); outputLearn.resize(before, nullptr); outputSegment.resize(before, UINT32_MAX); + const bool constant = op.segment & MemoOp::SegmentConstant; + AttributeIndex preset = constant ? op.segmentSet : UINT32_MAX; + bool learn = constant && preset == UINT32_MAX; + if (!constant && (op.segment & MemoOp::SegmentAttributes) && op.segmentId != 0) { + // its set is a function of the segment and the values of its way_memo_values tags + char sig[MemoSigCache::SIG_MAX]; + const size_t len = memoSegmentSig(op, sig); + if (len) { + preset = memoSegmentSets.find(MemoSigCache::hashOf(sig, len), sig, len); + learn = preset == UINT32_MAX; + } + } + skipping = preset != UINT32_MAX; + outputPreset.push_back(preset); + outputLearn.push_back(learn ? &op : nullptr); + outputSegment.push_back(i); + } else { + // This segment's attribute ops go to the output before, if any + if ((op.segment & MemoOp::SegmentAttributes) && !outputs.empty() && outputPreset.size() == outputs.size()) { + if (outputPreset.back() != UINT32_MAX) memoSegmentAttributes(ops, outputSegment.back()); + outputPreset.back() = UINT32_MAX; + outputLearn.back() = nullptr; + } + skipping = false; + } + continue; + } + if (memoIsAttributeOp(op.kind)) { + if (skipping) continue; + memoAttribute(op, memoOpValue(op)); + continue; + } + switch (op.kind) { + case MemoOp::MinZoom: MinZoom(op.number); break; + case MemoOp::ZOrder: ZOrder(op.number); break; + case MemoOp::Score: Score(op.number); break; + case MemoOp::ModifyId: ModifyId(static_cast(op.number)); break; + case MemoOp::MinZoomByArea: MinZoomByArea(op.number, op.numbers); break; + default: break; + } + } +} + +// The skipped attribute ops of the segment of layer op `layerOp`, onto outputs.back(), as made then +// (Layer had cleared outputKeys), and outputKeys as the layer op since has left it +void OsmLuaProcessing::memoSegmentAttributes(std::vector &ops, size_t layerOp) { + outputKeys.clear(); + for (size_t j = layerOp + 1; j < ops.size() && !memoIsLayerOp(ops[j].kind); j++) + if (memoIsAttributeOp(ops[j].kind)) memoAttribute(ops[j], memoOpValue(ops[j])); + outputKeys.clear(); +} + +// The value of an attribute op: its string, or for a valueKey op the current object's value of the +// tag, as currentTags->getKey/getValueFromKey (the leaf's value keys are in memoKeys since before +// the walk). It is copied if the attribute store keeps it. +protozero::data_view OsmLuaProcessing::memoOpValue(const MemoOp &op) const { + if (op.valueKey.empty()) return protozero::data_view(op.str.data(), op.str.size()); + const std::string_view v = memoSlots.get(op.valueKeyId); + return v.empty() ? protozero::data_view("", 0) : protozero::data_view(v.data(), v.size()); +} + +// The signature of the attribute set of an output of layer op `layerOp`, for a segment with valueKey +// ops: the segment's id, then the value of each of its valueKey ops (its length, its bytes). The +// other attribute ops are the same each time, so equal signatures mean equal sets. 0: too long. +size_t OsmLuaProcessing::memoSegmentSig(const MemoOp &layerOp, char *sig) const { + size_t len = 0; + memcpy(sig, &layerOp.segmentId, sizeof(layerOp.segmentId)); len += sizeof(layerOp.segmentId); + const MemoOp *ops = &layerOp + 1; + for (uint32_t k = 0; k < layerOp.segmentOps; k++) { + const MemoOp &op = ops[k]; + if (!memoIsAttributeOp(op.kind) || op.valueKey.empty()) continue; + const std::string_view v = memoSlots.get(op.valueKeyId); + if (v.size() >= 256 || len + 1 + v.size() > MemoSigCache::SIG_MAX) return 0; + sig[len++] = char(v.size()); + memcpy(sig + len, v.data(), v.size()); + len += v.size(); + } + return len; +} + +// An attribute op of the replay: as Attribute*()/attributeFast(), with the key index, the pair +// (no valueKey) and the vector_layers bookkeeping kept in the op +void OsmLuaProcessing::memoAttribute(MemoOp &op, const protozero::data_view &str) { + if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + if (op.keyIndex < 0) op.keyIndex = attributeStore.keyStore.key2index(op.key); + removeAttributeIfNeeded(op.keyIndex); + AttributeSet &set = outputs.back().second; + uint type = 0; + switch (op.kind) { + case MemoOp::Attr: case MemoOp::FastStr: + type = 0; + if (!op.valueKey.empty()) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, op.key, str, op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.key, str, op.minzoom); + set.addPair(op.pairIndex); + break; + case MemoOp::AttrNumeric: case MemoOp::FastNumeric: + type = 1; + if (op.valueLength) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, static_cast(str.size()), op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.number, op.minzoom); + set.addPair(op.pairIndex); + break; + case MemoOp::AttrBoolean: case MemoOp::FastBoolean: + type = 2; + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, op.flag, op.minzoom); + set.addPair(op.pairIndex); + break; + default: // AttrInteger, FastInteger + type = 3; + if (op.valueLength) { attributeStore.addAttributeWithKeyIndex(set, op.keyIndex, static_cast(str.size()), op.minzoom); break; } + if (op.pairIndex == 0) op.pairIndex = attributeStore.pairIndexWithKeyIndex(op.keyIndex, static_cast(op.number), op.minzoom); + set.addPair(op.pairIndex); + break; + } + const uint_least8_t layer = outputs.back().first.layer; + if (op.seenLayer != layer) { + setVectorLayerMetadata(layer, op.key, type); + op.seenLayer = layer; + } +} + +void OsmLuaProcessing::memoStringAttribute(MemoOp &&op) { + if (!memoTrace.values.empty() && !op.str.empty()) { + // Which way_memo_values tags has this value? Exactly one, and this attribute one of its: a placeholder + const std::pair *source = nullptr; + int matches = 0; + for (const auto &v : memoTrace.values) if (v.second == op.str) { source = &v; matches++; } + if (matches > 0) { + const auto it = memoValueTags.find(source->first); + const bool declared = matches == 1 && std::find(it->second.begin(), it->second.end(), op.key) != it->second.end(); + if (!declared) { memoTrace.ambiguous = true; return; } + op.valueKey = source->first; + op.str.clear(); + memoOp(std::move(op)); + return; + } + } + // An empty declared attribute: a copy of a tag it is declared for, read empty (as for every + // object that replays this, so the constant is right) + if (op.str.empty()) + for (const auto &e : memoTrace.emptyValues) { + const auto it = memoValueTags.find(e); + if (it != memoValueTags.end() && std::find(it->second.begin(), it->second.end(), op.key) != it->second.end()) { memoOp(std::move(op)); return; } + } + // A declared attribute that isn't a plain copy: the profile does more with the value than declared + for (const auto &t : memoValueTags) + if (std::find(t.second.begin(), t.second.end(), op.key) != t.second.end()) { memoLuaOnly(); return; } + memoOp(std::move(op)); +} + +// { tag = { "attribute", ..., attribute = "length", ... }, ... }: the attributes the tag's value is +// copied into, and (lengths: attribute -> tag) those that are set to its length +void OsmLuaProcessing::memoNumberAttribute(MemoOp &&op) { + if (!memoValueLengths.empty()) { + const auto it = memoValueLengths.find(op.key); + if (it != memoValueLengths.end()) { + // declared as the length of a tag: a placeholder if it is that (else the profile does more than declared) + for (const auto &v : memoTrace.values) + if (v.first == it->second && static_cast(v.second.size()) == op.number) { + op.valueKey = v.first; + op.valueLength = true; + op.number = 0; + memoOp(std::move(op)); + return; + } + // the length of the tag read empty (as for every object that replays this): a constant + if (op.number == 0 && std::find(memoTrace.emptyValues.begin(), memoTrace.emptyValues.end(), it->second) != memoTrace.emptyValues.end()) { + memoOp(std::move(op)); + return; + } + memoLuaOnly(); + return; + } + } + memoOp(std::move(op)); +} + +// The comparisons of the area with the memo_area numbers: per number '0' (<), '1' (>), '2' (==), '3' (NaN) +std::string OsmLuaProcessing::memoAreaResult(double area) const { + std::string r(memoArea.size(), '3'); + for (size_t i = 0; i < memoArea.size(); i++) + r[i] = area < memoArea[i] ? '0' : area > memoArea[i] ? '1' : area == memoArea[i] ? '2' : '3'; + return r; +} + +// The length of the table at idx (the declarations are plain tables: no __len). luaL_len is Lua 5.2+; +// Lua 5.1 and LuaJIT have lua_objlen. +static lua_Integer memoTableLength(lua_State *L, int idx) { +#if LUA_VERSION_NUM >= 502 + return luaL_len(L, idx); +#else + return static_cast(lua_objlen(L, idx)); +#endif +} + +static std::unordered_map> readMemoValueTags(lua_State *L, const char *global, std::unordered_map &lengths) { + std::unordered_map> tags; + lua_getglobal(L, global); + if (lua_istable(L, -1)) { + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + if (lua_type(L, -2) == LUA_TSTRING && lua_istable(L, -1)) { + const std::string tag = lua_tostring(L, -2); + std::vector &attrs = tags[tag]; + const lua_Integer n = memoTableLength(L, -1); + for (lua_Integer i = 1; i <= n; i++) { + lua_rawgeti(L, -1, i); + if (lua_type(L, -1) == LUA_TSTRING) attrs.push_back(lua_tostring(L, -1)); + lua_pop(L, 1); + } + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + if (lua_type(L, -2) == LUA_TSTRING && lua_type(L, -1) == LUA_TSTRING && std::string(lua_tostring(L, -1)) == "length") { + lua_pushvalue(L, -2); // lua_tostring on a copy: lua_next needs the key unchanged + lengths[lua_tostring(L, -1)] = tag; + lua_pop(L, 1); + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + return tags; +} + +// { number, ... }: the numbers way_function compares MercatorArea() with. An integer that a double +// can't hold exactly would compare differently in Lua: then none (MercatorArea() runs through Lua). +static std::vector readMemoArea(lua_State *L, const char *global) { + std::vector area; + lua_getglobal(L, global); + if (lua_istable(L, -1)) { + const lua_Integer n = memoTableLength(L, -1); + for (lua_Integer i = 1; i <= n; i++) { + lua_rawgeti(L, -1, i); + bool ok = lua_type(L, -1) == LUA_TNUMBER; +#if LUA_VERSION_NUM >= 503 + if (ok && lua_isinteger(L, -1)) { + const lua_Integer v = lua_tointeger(L, -1); + ok = v >= -(lua_Integer(1) << 53) && v <= (lua_Integer(1) << 53); + } +#endif + if (ok) area.push_back(lua_tonumber(L, -1)); + else { std::cerr << global << ": entry " << i << " isn't a number a double holds exactly; ignored (MercatorArea() runs through Lua)" << std::endl; area.clear(); lua_pop(L, 1); break; } + lua_pop(L, 1); + } + } + lua_pop(L, 1); + return area; +} diff --git a/src/osm_mem_tiles.cpp b/src/osm_mem_tiles.cpp index a20bfca8..18da43cb 100644 --- a/src/osm_mem_tiles.cpp +++ b/src/osm_mem_tiles.cpp @@ -1,6 +1,7 @@ #include "osm_mem_tiles.h" #include "node_store.h" #include "way_store.h" +#include "clip_line.h" using namespace std; thread_local GeometryCache linestringCache; @@ -54,47 +55,7 @@ Geometry OsmMemTiles::buildWayGeometry( } if (geomType == LINESTRING_ && IS_WAY(objectID)) { - Linestring& ls = getOrBuildLinestring(objectID); - - MultiLinestring out; - if(ls.empty()) - return out; - - Box extBox = bbox.getExtendBox(); - const double minX = extBox.min_corner().x(), maxX = extBox.max_corner().x(); - const double minY = extBox.min_corner().y(), maxY = extBox.max_corner().y(); - auto pointInsideExtBox = [minX, maxX, minY, maxY](const Point& p) { - return p.x() >= minX && p.x() <= maxX && p.y() >= minY && p.y() <= maxY; - }; - bool needsIntersection = !pointInsideExtBox(ls[0]); - - Linestring current_ls; - current_ls.reserve(ls.size()); - geom::append(current_ls, ls[0]); - - for(size_t i = 1; i < ls.size(); ++i) { - boost::geometry::model::segment segment(ls[i-1], ls[i]); - if(!geom::intersects(segment, bbox.clippingBox)) { - if(current_ls.size() > 1) - out.push_back(std::move(current_ls)); - current_ls.clear(); - current_ls.reserve(ls.size() - i); - } - geom::append(current_ls, ls[i]); - if (!needsIntersection) - needsIntersection = !pointInsideExtBox(ls[i]); - } - - if(current_ls.size() > 1) - out.push_back(std::move(current_ls)); - - if (!needsIntersection) - return out; - - MultiLinestring result; - geom::intersection(out, extBox, result); - return result; - + return clip_linestring_for_tile(getOrBuildLinestring(objectID), bbox.clippingBox, bbox.getExtendBox()); } throw std::runtime_error("buildWayGeometry: unexpected objectID: " + std::to_string(objectID)); diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 5ec7a81c..f7a01d72 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include +#include #include "node_store.h" #include "way_store.h" @@ -16,6 +18,42 @@ static inline bool isClosed(const std::vector& way) { return !way.empty() && way.front() == way.back(); } +// geom::within(inner, outer) for many inner rings against one big outer ring. +// Boost analyses the outer ring afresh on every call (a reservoir with 2500 +// islands and a 600,000-point shore takes 47 s); here its segments are indexed +// once. An inner ring that shares no point with the outer ring lies wholly +// inside or wholly outside it, so one of its vertices decides; when the rings +// touch or cross, Boost decides as before. +class RingWithinIndex { + using Segment = bg::model::segment; + using Value = std::pair; + const Ring &outer; + bg::index::rtree> segments; + + static std::vector segmentBoxes(const Ring &ring) { + std::vector values; + for (std::size_t i = 0; i + 1 < ring.size(); i++) + values.emplace_back(bg::return_envelope(Segment(ring[i], ring[i + 1])), i); + return values; + } + +public: + RingWithinIndex(const Ring &outer) : outer(outer), segments(segmentBoxes(outer)) { } + + bool within(const Ring &inner) const { + if (inner.size() < 4) return geom::within(inner, outer); + for (std::size_t j = 0; j + 1 < inner.size(); j++) { + const Segment s(inner[j], inner[j + 1]); + for (auto it = segments.qbegin(bg::index::intersects(bg::return_envelope(s))); it != segments.qend(); ++it) { + const std::size_t i = it->second; + if (bg::intersects(s, Segment(outer[i], outer[i + 1]))) + return geom::within(inner, outer); + } + } + return geom::within(inner.front(), outer); + } +}; + UsedObjects::UsedObjects(Status status): status(status), mutex(256), ids(256 * 1024) { } @@ -66,9 +104,9 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa MultiPolygon mp; if (outerBegin == outerEnd) { return mp; } // no outers so quit - std::vector outers; - std::vector inners; - std::map done; // true=this way has already been added to outers/inners, don't reconsider + std::vector outers; + std::vector inners; + DoneWays done; // the ways already added to outers/inners, not to reconsider // merge constituent ways together mergeMultiPolygonWays(outers, done, outerBegin, outerEnd); @@ -76,17 +114,40 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa // add all inners and outers to the multipolygon std::vector filledInners; + std::vector innerBoxes; for (auto it = inners.begin(); it != inners.end(); ++it) { Ring inner; + inner.reserve(it->size()); fillPoints(inner, it->begin(), it->end()); - filledInners.emplace_back(inner); + innerBoxes.emplace_back(geom::return_envelope(inner)); + filledInners.emplace_back(std::move(inner)); } bool onlyOneOuter = outers.size()==1; for (auto ot = outers.begin(); ot != outers.end(); ot++) { Polygon poly; + poly.outer().reserve(ot->size()); fillPoints(poly.outer(), ot->begin(), ot->end()); - for (auto it = filledInners.begin(); it != filledInners.end(); ++it) { - if (onlyOneOuter || geom::within(*it, poly.outer())) { poly.inners().emplace_back(*it); } + const Box outerBox = geom::return_envelope(poly.outer()); + // Index big outer rings that many inner rings must be tested against + std::unique_ptr index; + if (!onlyOneOuter && filledInners.size() >= 16 && poly.outer().size() >= 4096) + index.reset(new RingWithinIndex(poly.outer())); + { + // The point-in-ring tests of the within tests below use ring_point_index's index of a + // big ring (the same answers), while the rings don't change + RingPointIndexScope pointIndex; + for (std::size_t i = 0; i < filledInners.size(); i++) { + const Ring &inner = filledInners[i]; + // A ring within the outer has its bounding box within the outer's, + // so the (costly) within test is only needed when it does + if (onlyOneOuter) { + // (in no other polygon) + poly.inners().emplace_back(std::move(filledInners[i])); + } else if ((inner.empty() || geom::covered_by(innerBoxes[i], outerBox)) && + (index ? index->within(inner) : geom::within(inner, poly.outer()))) { + poly.inners().emplace_back(inner); + } + } } mp.emplace_back(move(poly)); } @@ -100,13 +161,14 @@ MultiLinestring OSMStore::wayListMultiLinestring(WayVec::const_iterator outerBeg MultiLinestring mls; if (outerBegin == outerEnd) { return mls; } - std::vector linestrings; - std::map done; + std::vector linestrings; + DoneWays done; mergeMultiPolygonWays(linestrings, done, outerBegin, outerEnd); for (auto ot = linestrings.begin(); ot != linestrings.end(); ot++) { Linestring ls; + ls.reserve(ot->size()); fillPoints(ls, ot->begin(), ot->end()); mls.emplace_back(move(ls)); } @@ -119,45 +181,73 @@ MultiLinestring OSMStore::wayListMultiLinestring(WayVec::const_iterator outerBeg // - Linestrings are joined to existing linestrings with which they share a start/end // - If no matches can be found, then one linestring is added (to 'attract' others) // - The process is rerun until no ways are left -// There's quite a lot of copying going on here - could potentially be addressed -void OSMStore::mergeMultiPolygonWays(std::vector &results, std::map &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const { - - // Create maps of start/end nodes - std::unordered_map> startNodes; - std::unordered_map> endNodes; +// (The ways are read once; the joined rings and lines are grown in place, see LatpLonChain.) +void OSMStore::mergeMultiPolygonWays(std::vector &results, DoneWays &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const { + + // Create maps of start/end nodes (the way lists are short: kept in the map's nodes) + using WayList = boost::container::small_vector; + using NodeMap = std::unordered_map; + NodeMap startNodes; + NodeMap endNodes; + // The coordinates of the ways in startNodes/endNodes, read once from the way store + // (the joining below used to read each of them two or three more times), one after the other in pool + std::vector pool; + std::unordered_map> wayCoords; // offset in pool, size + struct Span { + const LatpLon *b, *e; + const LatpLon *begin() const { return b; } + const LatpLon *end() const { return e; } + std::reverse_iterator rbegin() const { return std::reverse_iterator(e); } + std::reverse_iterator rend() const { return std::reverse_iterator(b); } + const LatpLon &front() const { return *b; } + const LatpLon &back() const { return e[-1]; } + }; + std::vector way; + auto coords = [&](WayID w) -> Span { + auto it = wayCoords.find(w); + if (it == wayCoords.end()) { + ways.at(w, way); + it = wayCoords.emplace(w, std::make_pair(pool.size(), way.size())).first; + pool.insert(pool.end(), way.begin(), way.end()); + } + return Span{ pool.data() + it->second.first, pool.data() + it->second.first + it->second.second }; + }; for (auto it = itBegin; it != itEnd; ++it) { - if (done[*it]) { continue; } + if (done.has(*it)) { continue; } try { - auto const &way = ways.at(*it); + ways.at(*it, way); if (isClosed(way) || results.empty()) { // if start==end, simply add it to the set - results.emplace_back(way.begin(), way.end()); - done[*it] = true; + results.emplace_back(std::move(way)); + way = std::vector(); + done.add(*it); } else { startNodes[way.front()].emplace_back(*it); endNodes[way.back()].emplace_back(*it); + wayCoords[*it] = std::make_pair(pool.size(), way.size()); + pool.insert(pool.end(), way.begin(), way.end()); } } catch (std::out_of_range &err) { if (verbose) { cerr << "Missing way in relation: " << err.what() << endl; } - done[*it] = true; + done.add(*it); } } + if (startNodes.empty()) return; // (no way to join: all were added) - auto deleteFromWayList = [&](LatpLon n, WayID w, bool which) { - auto &nodemap = which ? startNodes : endNodes; - std::vector &waylist = nodemap.find(n)->second; + auto deleteFromWayList = [&](NodeMap::iterator found, WayID w, NodeMap &nodemap) { + WayList &waylist = found->second; waylist.erase(std::remove(waylist.begin(), waylist.end(), w), waylist.end()); - if (waylist.empty()) { nodemap.erase(nodemap.find(n)); } + if (waylist.empty()) { nodemap.erase(found); } }; auto removeWay = [&](WayID w) { - auto const &way = ways.at(w); + const Span way = coords(w); LatpLon first = way.front(); LatpLon last = way.back(); - if (startNodes.find(first) != startNodes.end()) { deleteFromWayList(first, w, true ); } - if (startNodes.find(last) != startNodes.end()) { deleteFromWayList(last, w, true ); } - if (endNodes.find(first) != endNodes.end() ) { deleteFromWayList(first, w, false); } - if (endNodes.find(last) != endNodes.end() ) { deleteFromWayList(last, w, false); } - done[w] = true; + auto f = startNodes.find(first); if (f != startNodes.end()) { deleteFromWayList(f, w, startNodes); } + f = startNodes.find(last); if (f != startNodes.end()) { deleteFromWayList(f, w, startNodes); } + f = endNodes.find(first); if (f != endNodes.end() ) { deleteFromWayList(f, w, endNodes); } + f = endNodes.find(last); if (f != endNodes.end() ) { deleteFromWayList(f, w, endNodes); } + done.add(w); }; // Loop through, repeatedly adding start/end nodes if we can @@ -170,58 +260,60 @@ void OSMStore::mergeMultiPolygonWays(std::vector &results, std::ma LatpLon rFirst = rt->front(); LatpLon rLast = rt->back(); if (rFirst==rLast) break; - if (startNodes.find(rLast)!=startNodes.end()) { + auto match = startNodes.find(rLast); + if (match!=startNodes.end()) { // append to the result - auto match = startNodes.find(rLast)->second; - auto nodes = ways.at(match.back()); - rt->insert(rt->end(), nodes.begin(), nodes.end()); - removeWay(match.back()); + const WayID w = match->second.back(); + const Span nodes = coords(w); + rt->append(nodes.begin(), nodes.end()); + removeWay(w); added++; - - } else if (endNodes.find(rLast)!=endNodes.end()) { + continue; + } + match = endNodes.find(rLast); + if (match!=endNodes.end()) { // append reversed to the original - auto match = endNodes.find(rLast)->second; - auto nodes = ways.at(match.back()); - rt->insert(rt->end(), - std::make_reverse_iterator(nodes.end()), - std::make_reverse_iterator(nodes.begin())); - removeWay(match.back()); + const WayID w = match->second.back(); + const Span nodes = coords(w); + rt->append(nodes.rbegin(), nodes.rend()); + removeWay(w); added++; - - } else if (endNodes.find(rFirst)!=endNodes.end()) { + continue; + } + match = endNodes.find(rFirst); + if (match!=endNodes.end()) { // prepend to the original - auto match = endNodes.find(rFirst)->second; - auto nodes = ways.at(match.back()); - rt->insert(rt->begin(), nodes.begin(), nodes.end()); - removeWay(match.back()); + const WayID w = match->second.back(); + const Span nodes = coords(w); + rt->prepend(nodes.begin(), nodes.end()); + removeWay(w); added++; - - } else if (startNodes.find(rFirst)!=startNodes.end()) { + continue; + } + match = startNodes.find(rFirst); + if (match!=startNodes.end()) { // prepend reversed to the original - auto match = startNodes.find(rFirst)->second; - auto nodes = ways.at(match.back()); - rt->insert(rt->begin(), - std::make_reverse_iterator(nodes.end()), - std::make_reverse_iterator(nodes.begin())); - removeWay(match.back()); + const WayID w = match->second.back(); + const Span nodes = coords(w); + rt->prepend(nodes.rbegin(), nodes.rend()); + removeWay(w); added++; - - } else { working=false; } - + continue; + } + working=false; } while (working); } // If nothing was added, then 'seed' it with a remaining unallocated way for (int i=0; i<=1; i++) { if (added>0) continue; - for (auto nt : (i==0 ? startNodes : endNodes)) { - WayID w = nt.second.back(); - auto const &way = ways.at(w); - results.emplace_back(way.begin(), way.end()); - added++; - removeWay(w); - break; - } + auto &nodemap = i==0 ? startNodes : endNodes; + if (nodemap.empty()) continue; + WayID w = nodemap.begin()->second.back(); + const Span way = coords(w); + results.emplace_back(way.begin(), way.end()); + added++; + removeWay(w); } } while (added>0); }; diff --git a/src/output_object.cpp b/src/output_object.cpp index 3d5703ca..d7efd5de 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -42,8 +42,8 @@ static void writeAttributesTo( FeatureBuilder& fbuilder, char zoom ) { - attributeStore.forEachPairUnsafe(attributes, [&](const AttributePair& pair) { - const AttributePair* it = &pair; + attributeStore.forEachPairIndexCanonicalUnsafe(attributes, [&](uint32_t pairIndex) { + const AttributePair* it = &attributeStore.pairStore.getPairUnsafe(pairIndex); if (it->minzoom > zoom) return; // TODO: consider taking a data view that is stable @@ -70,6 +70,8 @@ static const uint32_t MEMO_MAX_SETS = 1 << 13; static const uint32_t MEMO_MAX_PAIRS = 1 << 14; static const size_t MEMO_KEYS = 1 << 9; // AttributePair::keyIndex is 9 bits static const uint32_t NO_INDEX = ~0u; +static const uint32_t MEMO_MAX_CANON_SETS = 1 << 14; +static const uint32_t MEMO_MAX_CANON_PAIRS = 1 << 18; static inline uint32_t memoHash(uint32_t x) { // AttributeIndex/pair index is shard << 18 | offset: mix the high bits into the low ones @@ -179,7 +181,20 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, if (pairs.used >= MEMO_MAX_PAIRS) pairs.clear(); const uint32_t begin = entries.size(); - attributeStore.forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { + if (canonStore != &attributeStore || canon.used >= MEMO_MAX_CANON_SETS || canonPairs.size() >= MEMO_MAX_CANON_PAIRS) { + canon.clear(); + canonPairs.clear(); + canonStore = &attributeStore; + } + const Table::Slot* c = canon.find(index); + if (!c) { + const uint32_t a = canonPairs.size(); + attributeStore.forEachPairIndexCanonicalUnsafe(index, [&](uint32_t pairIndex) { canonPairs.push_back(pairIndex); }); + canon.insert(index, a, canonPairs.size()); + c = canon.find(index); + } + for (uint32_t k = c->a; k < c->b; k++) { + const uint32_t pairIndex = canonPairs[k]; uint32_t key, value; if (const Table::Slot* s = pairs.find(pairIndex)) { key = s->a; @@ -188,10 +203,10 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, resolvePair(attributeStore, pairIndex, key, value); pairs.insert(pairIndex, key, value); } - if (key == NO_INDEX) return; + if (key == NO_INDEX) continue; entries.push_back(key); entries.push_back(value); - }); + } const uint32_t end = entries.size(); sets.insert(index, begin, end); diff --git a/src/pbf_processor.cpp b/src/pbf_processor.cpp index e45f7305..81f7c18b 100644 --- a/src/pbf_processor.cpp +++ b/src/pbf_processor.cpp @@ -55,7 +55,7 @@ bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup } lastNodeId = nodeId; - LatpLon latplon = { int(lat2latp(double(node.lat)/10000000.0)*10000000.0), node.lon }; + LatpLon latplon = { lat2latpE7(node.lat), node.lon }; tags.reset(); // For tagged nodes, call Lua, then save the OutputObject @@ -124,7 +124,7 @@ bool PbfProcessor::ReadWays( for (int k=0; k usedWays; for (const PbfReader::Relation& pbfRelation : pg.relations()) { bool isMultiPolygon = relationIsType(pbfRelation, typeKey, mpKey); @@ -248,6 +250,7 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG } else { if (!wayKeys.filter(tags)) continue; + cost = std::max(cost, pbfRelation.memids.size()); } osmStore.usedRelations.set(relid); for (int n=0; n < pbfRelation.memids.size(); n++) { @@ -271,7 +274,7 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG } } else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::WAY) { if (lastID >= pow(2,42)) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID)); - osmStore.mark_way_used(static_cast(lastID)); + usedWays.push_back(static_cast(lastID)); if (isAccepted) { const auto& roleView = pb.stringTable[pbfRelation.roles_sid[n]]; std::string role(roleView.data(), roleView.size()); @@ -280,6 +283,7 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG } } } + osmStore.mark_ways_used(usedWays); return true; } @@ -290,7 +294,8 @@ bool PbfProcessor::ReadRelations( const BlockMetadata& blockMetadata, const SignificantTags& wayKeys, uint shard, - uint effectiveShards + uint effectiveShards, + int onlyPass ) { // ---- Read relations if (pg.relations().empty()) @@ -316,11 +321,18 @@ bool PbfProcessor::ReadRelations( int innerKey= findStringPosition(pb, "inner"); int outerKey= findStringPosition(pb, "outer"); if (typeKey >-1) { + // Two passes: first the relations with many member ways (the slow multipolygons), so that a + // block read early doesn't start its slowest relation only after all the others + const std::size_t heavyMembers = 1000; + for (int pass = 0; pass < 2; pass++) { + if (onlyPass >= 0 && pass != onlyPass) continue; int j = -1; for (const PbfReader::Relation& pbfRelation : pg.relations()) { j++; if (j % blockMetadata.chunks != blockMetadata.chunk) continue; + if ((pbfRelation.memids.size() >= heavyMembers) != (pass == 0)) + continue; bool isMultiPolygon = relationIsType(pbfRelation, typeKey, mpKey); bool isBoundary = relationIsType(pbfRelation, typeKey, boundaryKey); @@ -381,6 +393,7 @@ bool PbfProcessor::ReadRelations( } } } + } osmStore.relations_insert_front(relations); return true; @@ -468,6 +481,11 @@ bool PbfProcessor::ReadBlock( // Keep count of groups read during this phase. std::size_t read_groups = 0; + // Relations: first the slow ones (many member ways) of all the block's groups, then the rest + if (phase == ReadPhase::Relations) + for (auto& pg : pb.groups()) + ReadRelations(output, pg, pb, blockMetadata, wayKeys, shard, effectiveShards, 0); + int primitiveGroupSize = 0; for (auto& pg : pb.groups()) { primitiveGroupSize++; @@ -528,7 +546,13 @@ bool PbfProcessor::ReadBlock( if(phase == ReadPhase::RelationScan) { osmStore.ensureUsedWaysInited(); - bool done = ScanRelations(output, pg, pb, wayKeys); + uint64_t cost = 0; + bool done = ScanRelations(output, pg, pb, wayKeys, cost); + if (cost > 0) { + std::lock_guard lock(relationBlockCostMutex); + uint64_t &c = relationBlockCost[blockMetadata.offset]; + c = std::max(c, cost); + } if(done) { if (ioMutex.try_lock()) { size_t scanProgress = 100*blocksProcessed.load()/blocksToProcess.load(); @@ -554,7 +578,7 @@ bool PbfProcessor::ReadBlock( } if(phase == ReadPhase::Relations) { - bool done = ReadRelations(output, pg, pb, blockMetadata, wayKeys, shard, effectiveShards); + bool done = ReadRelations(output, pg, pb, blockMetadata, wayKeys, shard, effectiveShards, 1); if(done) { output_progress(); ++read_groups; @@ -786,13 +810,50 @@ int PbfProcessor::ReadPbfFile( if (phase == ReadPhase::Nodes || phase == ReadPhase::Ways) batchSize = (filteredBlocks.size() / (threadNum * 8)) + 1; + // Relations: the blocks with the largest multipolygons (member ways) first. A few huge + // multipolygons (Lake Huron: 19 s) in a late block ran on alone after all others. + std::vector> ordered(filteredBlocks.begin(), filteredBlocks.end()); + if (phase == ReadPhase::Relations && !relationBlockCost.empty()) { + auto costOf = [&](const BlockMetadata& b) { + auto c = relationBlockCost.find(b.offset); + return c == relationBlockCost.end() ? uint64_t(0) : c->second; + }; + std::stable_sort(ordered.begin(), ordered.end(), [&](const std::pair& a, const std::pair& b) { + return costOf(a.second) > costOf(b.second); + }); + } + // Relations: the blocks handed out last (two per thread) are read in 3 parts each, which + // different threads take (as for an extract with few blocks, see above). A block takes + // ~1 s on a planet, but up to 2.4 s, and how long isn't known before: the last ones ran + // on alone for ~0.7 s. Each part parses the whole block again (~0.04 s). + if (phase == ReadPhase::Relations && threadNum > 1) { + const size_t tailPerThread = 2, parts = 3; + const size_t tail = std::min(ordered.size(), (size_t)threadNum * tailPerThread); + if (tail > 0 && parts > 1) { + std::vector> split(ordered.begin(), ordered.end() - tail); + for (auto b = ordered.end() - tail; b != ordered.end(); b++) { + if (b->second.chunks != 1) { split.push_back(*b); continue; } + for (size_t c = 0; c < parts; c++) { + auto part = *b; + part.second.chunks = parts; + part.second.chunk = c; + split.push_back(part); + } + } + ordered.swap(split); + } + } size_t consumed = 0; - auto it = filteredBlocks.begin(); - while(it != filteredBlocks.end()) { + auto it = ordered.begin(); + while(it != ordered.end()) { std::vector blockRange; - blockRange.reserve(batchSize); - size_t max = consumed + batchSize; - for (; consumed < max && it != filteredBlocks.end(); consumed++) { + // Batches shrink towards the end of the phase (each at most a share of what is left), + // so that the last ones don't run on alone: 24 blocks each on a planet took seconds + const size_t left = ordered.size() - consumed; + const size_t thisBatch = std::max(1, std::min(batchSize, left / (threadNum * 2))); + blockRange.reserve(thisBatch); + size_t max = consumed + thisBatch; + for (; consumed < max && it != ordered.end(); consumed++) { IndexedBlockMetadata ibm; memcpy(&ibm, &it->second, sizeof(BlockMetadata)); ibm.index = it->first; diff --git a/src/pbf_reader.cpp b/src/pbf_reader.cpp index 75acca09..198633cd 100644 --- a/src/pbf_reader.cpp +++ b/src/pbf_reader.cpp @@ -3,6 +3,99 @@ #include #include "pbf_reader.h" #include "helpers.h" +#include +#include +#include +#include + +// Packed varint fields are decoded in one pass over the bytes. The values and the exceptions are +// those of protozero's packed iterators, which decode each element with decode_varint and then +// step over it with skip_varint (the latter never throws after a successful decode_varint and +// steps over the same bytes): an element is a varint of at most 10 bytes (of the 10th byte only +// bit 0 counts); 10 or more continuation bytes throw varint_too_long_exception, a varint cut off +// by the end of the field (shorter than 10 bytes) throws end_of_buffer_exception. +// Fast path (little endian): 8 bytes are left in the field and the element ends within them: decoded +// from one 8-byte load without branches on its length, or 8 single-byte elements at once. All else +// (9 and 10 byte varints, errors, the last bytes of the field) goes through protozero::decode_varint. +// The output vector is sized to the field's byte count (an upper bound of the elements) first, +// then shrunk to the elements decoded. +namespace { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define PBF_READER_SWAR 1 +#endif + + // Decodes the field [p, end) into o (static_cast of the elements; with Delta, of the running + // sums of the zigzag-decoded elements, kept in U, wrapping). State in locals, so that it stays + // in registers. + template + inline T* decodePacked(const char* p, const char* end, T* o, U& acc) { + U a = acc; + auto put = [&](uint64_t v) { + if (Delta) { + a = static_cast(a + static_cast(static_cast(protozero::decode_zigzag64(v)))); + *o++ = static_cast(a); + } else { + *o++ = static_cast(v); + } + }; +#ifdef PBF_READER_SWAR + constexpr uint64_t high = 0x8080808080808080ULL; + while (end - p >= 8) { + uint64_t x; + std::memcpy(&x, p, 8); + if ((x & high) == 0) { + // 8 single-byte elements + for (int i = 0; i < 8; i++) + put((x >> (8 * i)) & 0xffU); + p += 8; + continue; + } + const uint64_t term = ~x & high; + if (term == 0) { + // 9 or more bytes: protozero + put(protozero::decode_varint(&p, end)); + continue; + } + const unsigned bits = __builtin_ctzll(term) + 1; // 8 * length + x &= (~0ULL >> (64 - bits)) & 0x7f7f7f7f7f7f7f7fULL; + x = (x & 0x007f007f007f007fULL) | ((x & 0x7f007f007f007f00ULL) >> 1); + x = (x & 0x00003fff00003fffULL) | ((x & 0x3fff00003fff0000ULL) >> 2); + x = (x & 0x000000000fffffffULL) | ((x & 0x0fffffff00000000ULL) >> 4); + put(x); + p += bits >> 3; + } +#endif + while (p != end) + put(protozero::decode_varint(&p, end)); + acc = a; + return o; + } + + // Appends the elements as static_cast, like get_packed_uint32 / get_packed_int32 + push_back + template + inline void appendPacked(protozero::data_view view, std::vector& out) { + const char* p = view.data(); + const char* end = p + view.size(); + const size_t old = out.size(); + out.resize(old + (end - p)); // at most one element per byte + uint64_t unused = 0; + out.resize(decodePacked(p, end, out.data() + old, unused) - out.data()); + } + + // Appends the running sums of the zigzag-decoded elements (get_packed_sint64 + `acc += i`; + // push_back(acc)), with the sum kept in the accumulator's type (wrapping) + template + inline void appendDeltas(protozero::data_view view, std::vector& out, Acc& acc) { + const char* p = view.data(); + const char* end = p + view.size(); + const size_t old = out.size(); + using U = typename std::make_unsigned::type; + U a = static_cast(acc); + out.resize(old + (end - p)); // at most one element per byte + out.resize(decodePacked(p, end, out.data() + old, a) - out.data()); + acc = static_cast(a); + } +} // Where pbf_processor.cpp has higher-level routines that populate our structures, // pbf_reader.cpp has low-level tools that interact with the protobuf. @@ -226,33 +319,18 @@ void PbfReader::DenseNodes::readDenseNodes(protozero::data_view data) { while (message.next()) { switch (message.tag()) { case Schema::DenseNodes::repeated_sint64_id: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - id += i; - ids.push_back(id); - } + appendDeltas(message.get_view(), ids, id); break; } case Schema::DenseNodes::repeated_sint64_lat: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - lat += i; - lats.push_back(lat); - } + appendDeltas(message.get_view(), lats, lat); break; } case Schema::DenseNodes::repeated_sint64_lon: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - lon += i; - lons.push_back(lon); - } + appendDeltas(message.get_view(), lons, lon); break; } case Schema::DenseNodes::repeated_int32_keys_vals: { - auto pi = message.get_packed_int32(); - for (auto kv : pi) { - keyValues.push_back(kv); - } + appendPacked(message.get_view(), keyValues); break; } @@ -263,17 +341,30 @@ void PbfReader::DenseNodes::readDenseNodes(protozero::data_view data) { } } - for (uint32_t cur = 0, prev = 0; cur < keyValues.size(); cur++) { - if (keyValues[cur] == 0) { - tagStart.push_back(prev); - tagEnd.push_back(cur); - prev = cur + 1; + // Per 0 in keyValues (the end of a node's tags) the node's tag range, then (0, 0) up to the + // number of nodes. Sized first, then filled. + { + const size_t kvSize = keyValues.size(); + const int32_t* kv = keyValues.data(); + const size_t old = tagStart.size(); + const size_t ends = std::count(kv, kv + kvSize, 0); + tagStart.resize(old + ends); + tagEnd.resize(old + ends); + int32_t* ts = tagStart.data() + old; + int32_t* te = tagEnd.data() + old; + uint32_t prev = 0; + for (uint32_t cur = 0; cur < kvSize; cur++) { + if (kv[cur] == 0) { + *ts++ = prev; + *te++ = cur; + prev = cur + 1; + } } } - while(tagStart.size() < ids.size()) { - tagStart.push_back(0); - tagEnd.push_back(0); + if (tagStart.size() < ids.size()) { + tagStart.resize(ids.size()); + tagEnd.resize(ids.size()); } } @@ -341,26 +432,6 @@ void PbfReader::DenseNodes::clear() { keyValues.clear(); } -bool PbfReader::DenseNodes::Iterator::operator!=(Iterator& other) const { - return offset != other.offset; -} - -void PbfReader::DenseNodes::Iterator::operator++() { - offset++; - - if (offset < nodes.ids.size()) { - node.id = nodes.ids[offset]; - node.lon = nodes.lons[offset]; - node.lat = nodes.lats[offset]; - node.tagStart = nodes.tagStart[offset]; - node.tagEnd = nodes.tagEnd[offset]; - } -} - -PbfReader::DenseNodes::Node& PbfReader::DenseNodes::Iterator::operator*() { - return node; -} - bool PbfReader::DenseNodes::empty() { return ids.empty(); } @@ -420,41 +491,23 @@ void PbfReader::Ways::Iterator::readWay(protozero::data_view data) { way.id = message.get_int64(); break; case Schema::Way::repeated_uint32_keys: { - auto pi = message.get_packed_uint32(); - for (auto i : pi) { - way.keys.push_back(i); - } + appendPacked(message.get_view(), way.keys); break; } case Schema::Way::repeated_uint32_vals: { - auto pi = message.get_packed_uint32(); - for (auto i : pi) { - way.vals.push_back(i); - } + appendPacked(message.get_view(), way.vals); break; } case Schema::Way::repeated_sint64_refs: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - ref += i; - way.refs.push_back(ref); - } + appendDeltas(message.get_view(), way.refs, ref); break; } case Schema::Way::repeated_sint64_lats: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - lat += i; - way.lats.push_back(lat); - } + appendDeltas(message.get_view(), way.lats, lat); break; } case Schema::Way::repeated_sint64_lons: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - lon += i; - way.lons.push_back(lon); - } + appendDeltas(message.get_view(), way.lons, lon); break; } @@ -522,39 +575,23 @@ void PbfReader::Relations::Iterator::readRelation(protozero::data_view data) { relation.id = message.get_int64(); break; case Schema::Relation::repeated_uint32_keys: { - auto pi = message.get_packed_uint32(); - for (auto i : pi) { - relation.keys.push_back(i); - } + appendPacked(message.get_view(), relation.keys); break; } case Schema::Relation::repeated_uint32_vals: { - auto pi = message.get_packed_uint32(); - for (auto i : pi) { - relation.vals.push_back(i); - } + appendPacked(message.get_view(), relation.vals); break; } case Schema::Relation::repeated_int32_roles_sid: { - auto pi = message.get_packed_int32(); - for (auto i : pi) { - relation.roles_sid.push_back(i); - } + appendPacked(message.get_view(), relation.roles_sid); break; } case Schema::Relation::repeated_sint64_memids: { - auto pi = message.get_packed_sint64(); - for (auto i : pi) { - memid += i; - relation.memids.push_back(memid); - } + appendDeltas(message.get_view(), relation.memids, memid); break; } case Schema::Relation::repeated_MemberType_types: { - auto pi = message.get_packed_int32(); - for (auto i : pi) { - relation.types.push_back(i); - } + appendPacked(message.get_view(), relation.types); break; } diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..1b9ef44e 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,44 +6,488 @@ #include #include #include +#include +#include +#include #include "pmtiles.h" #include "helpers.h" +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#endif + +// Tile data is written in 32 MB buffers with O_DIRECT by a thread of its own: the ordered write step +// only copies the tiles into the current buffer. One thread pushing the planet's 76 GB through ext4's +// page cache managed 1.2 GB/s (all of it kernel time) and held up the end of the tile phase; direct +// writes reach 2.1 GB/s. The unaligned rest at the end goes through the stream as before, so the file +// is the same. Where O_DIRECT isn't available (e.g. tmpfs), or with TM_NO_DIRECT set, everything goes +// through the stream. +struct PMTiles::DirectWriter { + static constexpr size_t BUFFER = 32 << 20, ALIGN = 4096, BUFFERS = 64, WRITERS = 4; // 2 GB absorbs bursts + static constexpr size_t SPARE = 4; // free buffers kept resident; the others' pages are given back + int fd = -1; + std::vector buffers; + char *current = nullptr; + size_t used = 0; + uint64_t offset = 0; // file offset of the current buffer + std::mutex mutex; + std::condition_variable changed; + std::deque free; + std::deque> queue; + bool finished = false; + int error = 0; + std::vector threads; + + ~DirectWriter() { + { std::lock_guard lock(mutex); finished = true; } + changed.notify_all(); + for (auto &t : threads) if (t.joinable()) t.join(); +#ifdef __linux__ + if (fd >= 0) ::close(fd); +#endif + for (char *b : buffers) std::free(b); + } + + bool open(const std::string &filename, uint64_t start) { +#ifdef __linux__ + if (getenv("TM_NO_DIRECT")) return false; + fd = ::open(filename.c_str(), O_WRONLY | O_DIRECT | O_CLOEXEC); + if (fd < 0) return false; + for (size_t i = 0; i < BUFFERS; i++) { + void *b = nullptr; + if (posix_memalign(&b, ALIGN, BUFFER) != 0) return false; + buffers.push_back(static_cast(b)); + } + current = buffers[0]; + for (size_t i = 1; i < BUFFERS; i++) free.push_back(buffers[i]); + offset = start; + for (size_t i = 0; i < WRITERS; i++) threads.emplace_back([this]() { run(); }); + return true; +#else + (void)filename; (void)start; + return false; +#endif + } + + void run() { +#ifdef __linux__ + while (true) { + std::tuple job; + { + std::unique_lock lock(mutex); + changed.wait(lock, [&]() { return !queue.empty() || finished; }); + if (queue.empty()) return; + job = queue.front(); + queue.pop_front(); + } + char *b = std::get<0>(job); + size_t done = 0, length = std::get<1>(job); + while (done < length) { + { std::lock_guard lock(mutex); if (error != 0) break; } + const ssize_t n = pwrite(fd, b + done, length - done, std::get<2>(job) + done); + if (n < 0) { if (errno != EINTR) { std::lock_guard lock(mutex); error = errno; } } + else done += n; + } + bool spare; + { + // the buffer freed last is reused first: the others are only touched (made + // resident) when a burst needs them, rather than all of them in turn + std::lock_guard lock(mutex); + spare = free.size() < SPARE; + if (spare) free.push_front(b); + } + if (!spare) { + // A burst (the disk falling behind, e.g. busy with other work) made up to 2 GB of + // buffers resident, and they stayed so until the end: give this one's pages back + madvise(b, BUFFER, MADV_DONTNEED); + std::lock_guard lock(mutex); + free.push_back(b); + } + changed.notify_all(); + } +#endif + } + + void submit(size_t length) { + std::unique_lock lock(mutex); + queue.emplace_back(current, length, offset); + offset += length; + changed.notify_all(); + changed.wait(lock, [&]() { return !free.empty(); }); + current = free.front(); + free.pop_front(); + used = 0; + } + + void append(const char *data, size_t length) { + while (length > 0) { + const size_t n = std::min(length, BUFFER - used); + std::memcpy(current + used, data, n); + used += n; data += n; length -= n; + if (used == BUFFER) submit(BUFFER); + } + } + + // Queues the aligned part of what is left; returns the unaligned rest and its file offset, for + // the stream. The writes go on meanwhile (see wait). + std::pair finish() { + const size_t aligned = used & ~(ALIGN - 1); + std::string rest(current + aligned, used - aligned); + uint64_t restOffset = offset + aligned; + if (aligned > 0) submit(aligned); + { + std::lock_guard lock(mutex); + finished = true; + } + changed.notify_all(); + return { rest, restOffset }; + } + + // Waits for all writes + void wait() { + for (auto &t : threads) if (t.joinable()) t.join(); +#ifdef __linux__ + if (fd >= 0) { ::close(fd); fd = -1; } +#endif + if (error != 0) throw std::runtime_error(std::string("PMTiles: writing the tiles failed: ") + std::strerror(error) + " (TM_NO_DIRECT=1 writes without O_DIRECT)"); + } +}; TileOffset::TileOffset() { } -PMTiles::PMTiles() { } +// Placing tiles (offset, repeats, index) and writing their bytes are two steps, each in batch order +// and one thread at a time, but run by different threads side by side: one thread doing both was +// the bottleneck of the tile phase at 96 threads. +PMTiles::PMTiles() : + batches([this](PendingTile &tile) { writeTile(tile); }, + [](const PendingTile &tile) { return tile.data.size(); }, + [this](PendingTile &tile) { flushTile(tile); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { std::cout << "Creating pmtiles at " << filename << std::endl; + this->filename = filename; + outputBuffer.resize(4 << 20); + outputStream.rdbuf()->pubsetbuf(outputBuffer.data(), outputBuffer.size()); outputStream.open(filename, std::ios::out | std::ios::trunc | std::ios::binary); // dummy header/root directory for now - we'll write it all later char header[HEADER_ROOT] = "PMTiles"; outputStream.write(header, HEADER_ROOT); + // the tile data follows, past the page cache if possible (the stream's header must be out first) + outputStream.flush(); + direct.reset(new DirectWriter()); + if (!direct->open(filename, HEADER_ROOT)) direct.reset(); + // The tile data is copied out by a thread of its own: at the end of the tile phase, when the + // last batches are written, it goes on while close() makes the directories + batches.startFlushThread(); } // Finish writing the .pmtiles file void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; + // The tiles are all placed (the index is complete), but their data may still be on its way to + // the file: the directories are made meanwhile, and the data finished before they're written + std::unique_ptr directWrites; + auto finishTileData = [&]() { + batches.finishFlushThread(); + if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } + // The direct writes of the tile data go on while the rest is written (waited for at the end) + directWrites = std::move(direct); + if (directWrites) { + // the rest of the tile data through the stream, which then goes on after it + auto rest = directWrites->finish(); + outputStream.seekp(rest.second); + outputStream.write(rest.first.data(), rest.first.size()); + } + }; + // Big buffers that aren't needed any more are kept until the PMTiles object goes, rather than + // freed one allocation at a time on the way (~0.2 s on a planet). Exiting, tilemaker releases its + // memory on all threads anyway. + auto freeAlongside = [this](auto &&garbage) { + using T = typename std::decay::type; + laterGarbage.push_back(std::make_shared(std::move(garbage))); + }; - // add all tiles to directories, writing leaf directories as we go + // Add all tiles to directories: low zooms to the root, the rest to leaf directories of up to + // LEAF_DIRECTORY_SIZE entries. The leaves are serialized and compressed on all cores (one after + // the other that took ~9 s for the planet), then written in order, so the file is the same. std::vector rootEntries; - std::vector entries; - leafStart = outputStream.tellp(); - if (isSparse) { - for (auto it : sparseIndex) { - appendTileEntry(it.first, it.second, rootEntries, entries); + std::vector> leaves(1); + // Big buffers that aren't needed any more are freed by a thread of their own, while close() goes on + std::vector freeing; + auto freeAside = [&freeing](auto &&garbage) { + using T = typename std::decay::type; + auto owned = std::make_shared(std::move(garbage)); + freeing.emplace_back([owned]() mutable { owned.reset(); }); + }; + auto append = [&](uint64_t tileId, TileOffset offset) { + pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE + if (tileId=LEAF_DIRECTORY_SIZE) leaves.emplace_back(); } + }; + if (isSparse) { + for (auto it : sparseIndex) append(it.first, it.second); } else { - for (size_t tileId=0; tileId(denseSize, FIRST_LEAF_TILE); + for (size_t tileId=0; tileId> DENSE_BLOCK_BITS].get(); + if (block && block[tileId & (blockSize - 1)].length != 0xffffff) append(tileId, block[tileId & (blockSize - 1)]); + } + const size_t n = denseSize > rootEnd ? denseSize - rootEnd : 0; + const size_t numChunks = n == 0 ? 0 : std::min(n, std::max(1u, std::thread::hardware_concurrency()) * 4); + std::vector> chunks(numChunks); + std::atomic nextChunk(0); + auto encodeChunks = [&]() { + for (size_t c = nextChunk++; c < numChunks; c = nextChunk++) { + const size_t from = rootEnd + n * c / numChunks, to = rootEnd + n * (c + 1) / numChunks; + std::vector &entries = chunks[c]; + for (size_t tileId = from; tileId < to; tileId++) { + const TileOffset *block = denseIndex[tileId >> DENSE_BLOCK_BITS].get(); + if (!block) { // no tile in the rest of the block + tileId |= blockSize - 1; + continue; + } + const TileOffset &o = block[tileId & (blockSize - 1)]; + if (o.length == 0xffffff) continue; + if (!entries.empty() && entries.back().offset == o.offset && + entries.back().tile_id == tileId - entries.back().run_length) + entries.back().run_length++; + else + entries.emplace_back(tileId, o.offset, o.length, 1); + } + } + }; + { + std::vector threads; + const size_t numThreads = std::min(std::max(1, numChunks), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) threads.emplace_back(encodeChunks); + for (auto &t : threads) t.join(); + } + freeAside(std::move(denseIndex)); // the chunks have all its tiles + // The chunks' entries then go into the leaves as if their tiles were appended one by one + // (appendWithRLE): a run going on in the next chunk extends the entry before it, and a leaf + // ends when it is full - with one tile of the entry that filled it, the rest of that run + // starting the next leaf. The positions come from two short passes, over the chunks and over + // the leaves; then the leaves are filled on all threads (one after the other that took ~2 s). + static_assert(LEAF_DIRECTORY_SIZE >= 2, "a leaf of one entry would fill with the first entry of a run going on"); + // (1) The entries as one list: chunk c's from position firstOf[c] on; its first entry belongs + // to the one before when the run goes on (merged[c]), and those tiles are added to it (extra). + std::vector firstOf(numChunks + 1, 0); + std::vector merged(numChunks, 0); + std::vector> extra; // (position, tiles added), ascending + { + size_t g = 0; + bool haveLast = false; + uint64_t lastTile = 0, lastOffset = 0, lastRun = 0; + for (size_t c = 0; c < numChunks; c++) { + firstOf[c] = g; + const std::vector &chunk = chunks[c]; + if (chunk.empty()) continue; + size_t j = 0; + if (haveLast && chunk[0].offset == lastOffset && chunk[0].tile_id == lastTile + lastRun) { + merged[c] = 1; + if (!extra.empty() && extra.back().first == g - 1) extra.back().second += chunk[0].run_length; + else extra.emplace_back(g - 1, chunk[0].run_length); + lastRun += chunk[0].run_length; + j = 1; + } + if (j < chunk.size()) { + const pmtiles::entryv3 &e = chunk.back(); + lastTile = e.tile_id; lastOffset = e.offset; lastRun = e.run_length; haveLast = true; + g += chunk.size() - j; + } + } + firstOf[numChunks] = g; + } + const size_t total = firstOf[numChunks]; + // the entry at position g, with the tiles merged into it + auto entryAt = [&](size_t g) { + const size_t c = std::upper_bound(firstOf.begin(), firstOf.end(), g) - firstOf.begin() - 1; + pmtiles::entryv3 e = chunks[c][g - firstOf[c] + merged[c]]; + auto x = std::lower_bound(extra.begin(), extra.end(), std::make_pair(g, uint64_t(0))); + if (x != extra.end() && x->first == g) e.run_length += x->second; + return e; + }; + // (2) The leaves: each starts with the rest of the run that filled the one before (if any), + // then takes entries until it is full + struct LeafPlan { size_t first, count; bool hasRest, cutLast; pmtiles::entryv3 rest; }; + std::vector plans; + { + size_t g = 0; + bool hasRest = false; + pmtiles::entryv3 rest(0, 0, 0, 0); + while (g < total || hasRest) { + LeafPlan plan { g, 0, hasRest, false, rest }; + const size_t room = std::max(1, LEAF_DIRECTORY_SIZE - (hasRest ? 1 : 0)); // the rest doesn't fill a leaf by itself + if (total - g < room) { + plan.count = total - g; + plans.push_back(plan); + break; + } + plan.count = room; + const pmtiles::entryv3 filler = entryAt(g + room - 1); + hasRest = filler.run_length > 1; + if (hasRest) { + plan.cutLast = true; + rest = pmtiles::entryv3(filler.tile_id + 1, filler.offset, filler.length, filler.run_length - 1); + } + plans.push_back(plan); + g += room; + } } + // (3) Fill them, each in parts (a planet has a few leaves only), on all threads + if (!plans.empty()) { + leaves.assign(plans.size(), std::vector()); + const size_t partSize = 1 << 20; + std::vector> parts; // (leaf, first entry of the plan's part) + for (size_t k = 0; k < plans.size(); k++) { + leaves[k].resize(plans[k].count + (plans[k].hasRest ? 1 : 0)); + if (plans[k].hasRest) leaves[k][0] = plans[k].rest; + for (size_t from = 0; from < plans[k].count; from += partSize) parts.emplace_back(k, from); + } + std::atomic nextPart(0); + auto fillLeaves = [&]() { + for (size_t q = nextPart++; q < parts.size(); q = nextPart++) { + const LeafPlan &plan = plans[parts[q].first]; + std::vector &leaf = leaves[parts[q].first]; + const size_t first = plan.first + parts[q].second, count = std::min(partSize, plan.count - parts[q].second); + size_t pos = (plan.hasRest ? 1 : 0) + parts[q].second; + // walk the chunks from the part's first entry on + size_t c = std::upper_bound(firstOf.begin(), firstOf.end(), first) - firstOf.begin() - 1; + size_t j = first - firstOf[c] + merged[c]; + auto x = std::lower_bound(extra.begin(), extra.end(), std::make_pair(first, uint64_t(0))); + for (size_t g = first; g < first + count; g++, j++, pos++) { + while (j >= chunks[c].size()) { c++; j = merged[c]; } + leaf[pos] = chunks[c][j]; + if (x != extra.end() && x->first == g) { leaf[pos].run_length += x->second; ++x; } + } + } + }; + std::vector threads; + const size_t numThreads = std::min(std::max(1, parts.size()), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) threads.emplace_back(fillLeaves); + for (auto &t : threads) t.join(); + for (size_t k = 0; k < plans.size(); k++) + if (plans[k].cutLast) leaves[k].back().run_length = 1; + for (const auto &leaf : leaves) numTileEntries += leaf.size(); + } + freeAside(std::move(chunks)); + } + if (leaves.back().empty()) leaves.pop_back(); + auto runOnAll = [](size_t jobs, const std::function &job) { + std::atomic next(0); + std::vector threads; + const size_t numThreads = std::min(std::max(1, jobs), std::max(1u, std::thread::hardware_concurrency())); + for (size_t t = 0; t < numThreads; t++) + threads.emplace_back([&]() { for (size_t i = next++; i < jobs; i = next++) job(i); }); + for (auto &t : threads) t.join(); + }; + std::vector compressedLeaves; + if (numTileEntries >= ROOT_ONLY) { + // Serialized as pmtiles::serialize_directory does - the entry count, then the tile ids (as + // differences), run lengths, lengths and offsets (0: right after the one before), each a + // varint - but each column in parts, on all threads: one 10M-entry leaf took 0.4 s + const size_t serialPart = 1 << 20; + struct Piece { size_t leaf, column, from, to; std::string bytes; }; + std::vector pieces; + for (size_t i = 0; i < leaves.size(); i++) + for (size_t column = 0; column < 4; column++) + for (size_t from = 0; from < leaves[i].size() || from == 0; from += serialPart) + pieces.push_back(Piece { i, column, from, std::min(leaves[i].size(), from + serialPart), std::string() }); + auto putVarint = [](std::string &out, uint64_t value) { + while (value >= 0x80U) { + out.push_back(char((value & 0x7fU) | 0x80U)); + value >>= 7U; + } + out.push_back(char(value)); + }; + runOnAll(pieces.size(), [&](size_t k) { + Piece &piece = pieces[k]; + const std::vector &entries = leaves[piece.leaf]; + std::string &out = piece.bytes; + out.reserve((piece.to - piece.from) * 3 + 16); + for (size_t i = piece.from; i < piece.to; i++) { + const pmtiles::entryv3 &e = entries[i]; + switch (piece.column) { + case 0: putVarint(out, e.tile_id - (i > 0 ? entries[i - 1].tile_id : 0)); break; + case 1: putVarint(out, e.run_length); break; + case 2: putVarint(out, e.length); break; + default: + if (i > 0 && e.offset == entries[i - 1].offset + entries[i - 1].length) putVarint(out, 0); + else putVarint(out, e.offset + 1); + } + } + }); + compressedLeaves.resize(leaves.size()); + runOnAll(leaves.size(), [&](size_t i) { + std::string data; + size_t size = 10; + for (const Piece &piece : pieces) if (piece.leaf == i) size += piece.bytes.size(); + data.reserve(size); + putVarint(data, leaves[i].size()); + for (Piece &piece : pieces) + if (piece.leaf == i) { data += piece.bytes; std::string().swap(piece.bytes); } + compressedLeaves[i] = compress_string(data, Z_DEFAULT_COMPRESSION, true); + }); } + finishTileData(); + leafStart = outputStream.tellp(); if (numTileEntries < ROOT_ONLY) { - rootEntries.insert(rootEntries.end(), entries.begin(), entries.end()); - entries.clear(); + for (auto const &leaf : leaves) rootEntries.insert(rootEntries.end(), leaf.begin(), leaf.end()); } else { - flushEntries(rootEntries,entries); + // write the leaf directories, and add references to them to the root directory + std::vector locations(leaves.size()); + uint64_t location = static_cast(outputStream.tellp()); + for (size_t i = 0; i < leaves.size(); i++) { + locations[i] = location; + location += compressedLeaves[i].size(); + rootEntries.emplace_back(pmtiles::entryv3(leaves[i][0].tile_id, locations[i]-leafStart, compressedLeaves[i].size(), 0)); + } + bool written = false; +#ifdef __linux__ + // side by side, through a file descriptor of their own (they're past what the stream holds) + outputStream.flush(); + const int fd = ::open(filename.c_str(), O_WRONLY | O_CLOEXEC); + if (fd >= 0) { + std::atomic failed(false); + runOnAll(leaves.size(), [&](size_t i) { + size_t done = 0; + const std::string &bytes = compressedLeaves[i]; + while (done < bytes.size() && !failed) { + const ssize_t n = pwrite(fd, bytes.data() + done, bytes.size() - done, locations[i] + done); + if (n < 0) { if (errno != EINTR) failed = true; } + else done += n; + } + }); + if (::close(fd) != 0) failed = true; + if (failed) throw std::runtime_error(std::string("PMTiles: writing the leaf directories failed: ") + std::strerror(errno)); + outputStream.seekp(location); + written = true; + } +#endif + if (!written) { + for (size_t i = 0; i < leaves.size(); i++) + outputStream.write(compressedLeaves[i].c_str(), compressedLeaves[i].size()); + } + freeAlongside(std::move(compressedLeaves)); } uint64_t leafLength = static_cast(outputStream.tellp()) - leafStart; @@ -84,21 +529,14 @@ void PMTiles::close(std::string &metadata) { // ...and we're done! outputStream.close(); + if (directWrites) directWrites->wait(); + freeAlongside(std::move(leaves)); + freeAlongside(std::move(directWrites)); // its buffers + for (auto &t : freeing) t.join(); } // Add an entry either to the current leaf directory, or (for lowzoom) the root directory -void PMTiles::appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries) { - pmtiles::entryv3 entry = pmtiles::entryv3(tileId, offset.offset, offset.length, 1); // 1=RLE - if (tileId=LEAF_DIRECTORY_SIZE) flushEntries(rootEntries, entries); - } -} - -// Handle run-length encoding for appendTileEntry +// Handle run-length encoding for the directory entries void PMTiles::appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry) { if (entries.empty() || entries.back().offset != entry.offset || @@ -110,64 +548,66 @@ void PMTiles::appendWithRLE(std::vector &entries, pmtiles::ent entries.back().run_length++; } -// Write a leaf directory to file, and add a reference to it to the root directory -void PMTiles::flushEntries(std::vector &rootEntries, std::vector &entries) { - if (entries.size()==0) return; - uint64_t startId = entries[0].tile_id; - std::string directory = pmtiles::serialize_directory(entries); - std::string compressed = compress_string(directory, Z_DEFAULT_COMPRESSION, true); - entries.clear(); - - // write the leaf directory to disk - std::lock_guard lock(fileMutex); - uint64_t location = outputStream.tellp(); - uint64_t length = compressed.size(); - outputStream.write(compressed.c_str(), compressed.size()); - - // append reference to the root directory - pmtiles::entryv3 rootEntry = pmtiles::entryv3(startId, location-leafStart, length, 0); - rootEntries.emplace_back(rootEntry); +// Save a tile: compress it here (compress_string is expensive), but leave tiny tiles +// raw until they're written, since most of them (e.g. sea) repeat an earlier one +void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { + PendingTile tile; + tile.tileId = pmtiles::zxy_to_tileid(zoom,x,y); + tile.isTiny = data.size() lock(batches.writeMutex()); + writeTile(tile); + flushTile(tile); +} + +void PMTiles::flushTile(PendingTile &tile) { + if (!tile.data.empty()) { + if (direct) direct->append(tile.data.data(), tile.data.size()); + else outputStream.write(tile.data.c_str(), tile.data.size()); + } + tile.data = std::string(); } // Write a tile to file and store it in the index -// - if the tile is small and has already been written, reuse that instead -// - we're a bit fussy about mutexs because compress_string is expensive -void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { +// - if the tile is tiny and has already been written, reuse that instead +void PMTiles::writeTile(PendingTile &tile) { TileOffset offset; - bool isNew = false; - uint64_t tileId = pmtiles::zxy_to_tileid(zoom,x,y); - - // if it's a tiny tile (e.g. sea), see if we've written it already - std::unique_lock indexLock1(indexMutex); - if (data.size()second; + tile.data = std::string(); // already in the file } else { - indexLock1.unlock(); - std::string compressed = compress_string(data, Z_DEFAULT_COMPRESSION, true); - std::lock_guard lock(fileMutex); - // write to file - offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); - outputStream.write(compressed.c_str(), compressed.size()); + std::string compressed = tile.isTiny ? compress_string(tile.data, compressionLevel, true) : std::move(tile.data); + offset = TileOffset(tileDataEnd, compressed.size()); + tileDataEnd += compressed.size(); numTilesWritten++; - isNew = true; + // add to cache if tiny + if (tile.isTiny) { + if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); + tinyCache.insert({ tile.data, offset }); + } + tile.data = std::move(compressed); // flushTile writes it } - + // store in index - std::lock_guard indexLock2(indexMutex); + uint64_t tileId = tile.tileId; numTilesAddressed++; if (isSparse) { sparseIndex[tileId] = offset; } else { - if (tileId >= denseIndex.size()) denseIndex.resize(tileId+10000, { 0, 0xffffff }); - denseIndex[tileId] = offset; - } - - // add to cache if tiny - if (isNew && data.size()TINY_MAX_SIZE) tinyCache.clear(); - tinyCache.insert({ data, offset }); + if (tileId >= denseSize) { + denseSize = tileId + 10000; + denseIndex.resize(((denseSize - 1) >> DENSE_BLOCK_BITS) + 1); + } + std::unique_ptr &block = denseIndex[tileId >> DENSE_BLOCK_BITS]; + if (!block) { + block.reset(new TileOffset[size_t(1) << DENSE_BLOCK_BITS]); + std::fill_n(block.get(), size_t(1) << DENSE_BLOCK_BITS, TileOffset(0, 0xffffff)); + } + block[tileId & ((size_t(1) << DENSE_BLOCK_BITS) - 1)] = offset; } } diff --git a/src/shared_data.cpp b/src/shared_data.cpp index 88db71be..769213bc 100644 --- a/src/shared_data.cpp +++ b/src/shared_data.cpp @@ -237,6 +237,7 @@ Config::Config() { clippingBoxFromJSON = false; baseZoom = 0; combineBelow = 0; + compressionLevel = 6; tileFormat = TileFormat::MVT; } @@ -274,6 +275,13 @@ void Config::readConfig(rapidjson::Document &jsonConfig, bool &hasClippingBox, B compressOpt = jsonConfig["settings"]["compress"].GetString(); combineBelow = jsonConfig["settings"].HasMember("combine_below") ? jsonConfig["settings"]["combine_below"].GetUint() : 0; mvtVersion = jsonConfig["settings"].HasMember("mvt_version") ? jsonConfig["settings"]["mvt_version"].GetUint() : 2; + if (jsonConfig["settings"].HasMember("compression_level")) { + if (!jsonConfig["settings"]["compression_level"].IsInt() || jsonConfig["settings"]["compression_level"].GetInt() < 1 || jsonConfig["settings"]["compression_level"].GetInt() > 12) { + cerr << "\"compression_level\" should be a whole number from 1 (fastest) to 12 (smallest) in JSON file." << endl; + exit (EXIT_FAILURE); + } + compressionLevel = jsonConfig["settings"]["compression_level"].GetInt(); + } std::string tileFormatOpt = jsonConfig["settings"].HasMember("tile_format") ? jsonConfig["settings"]["tile_format"].GetString() : "mvt"; if (tileFormatOpt == "mvt") { tileFormat = TileFormat::MVT; } else if (tileFormatOpt == "mlt") { diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index e5c69e52..fd673e65 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -78,14 +78,58 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con uint32_t y1 = latp2tiley(latp1, spatialIndexZoom); uint32_t y2 = latp2tiley(latp2, spatialIndexZoom); - for (int x = std::min(x1, x2); x <= std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); x++) { - for (int y = std::min(y1, y2); y <= std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); y++) { - uint32_t z6x = x / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); - uint32_t z6y = y / (1u << (spatialIndexZoom - CLUSTER_ZOOM)); + const uint32_t xMin = std::min(x1, x2), xMax = std::min((1u << spatialIndexZoom) - 1u, std::max(x1, x2)); + const uint32_t yMin = std::min(y1, y2), yMax = std::min((1u << spatialIndexZoom) - 1u, std::max(y1, y2)); + const uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); + + // First pass, without any geometry checks: is a tile in the box already known to + // intersect, and how many tiles still need the exact check below? + size_t unchecked = 0; + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor, z6y = y / divisor; + auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; + uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); + if (!bitvec.empty() && bitvec[index]) { + if (bitvec[index + 1]) return true; + unchecked++; + } + } + } + if (unchecked == 0) return false; + + // Each tile's exact check intersects the tile with every candidate geometry, so its + // cost grows with the size of those geometries. A large query box that lies inside + // the bounding box of a huge geometry without touching it (say, an offshore marine + // reserve inside a country's bounding box) would check every one of thousands of + // tiles, and find nothing. Instead, check the query box once: if no geometry + // intersects it, none intersects any tile in it. Otherwise the caller's exact check + // decides. + const size_t maxTileChecks = 16; + if (unchecked > maxTileChecks) { + Box queryBox = box; + return !QueryMatchingGeometries( + layerName, + true, + queryBox, + [&](const RTree &rtree) { // indexQuery + vector results; + rtree.query(geom::index::intersects(queryBox), back_inserter(results)); + return results; + }, + [&](OutputObject const &oo) { // checkQuery + return geom::intersects(queryBox, retrieveMultiPolygon(oo.objectID)); + } + ).empty(); + } + + for (uint32_t x = xMin; x <= xMax; x++) { + for (uint32_t y = yMin; y <= yMax; y++) { + uint32_t z6x = x / divisor; + uint32_t z6y = y / divisor; auto& bitvec = sparseLayerVector[z6x * CLUSTER_ZOOM_WIDTH + z6y]; - uint32_t divisor = 1u << (spatialIndexZoom - CLUSTER_ZOOM); uint64_t index = 2u * ((x - z6x * divisor) * divisor + (y - z6y * divisor)); if (!bitvec.empty() && bitvec[index]) { @@ -137,7 +181,8 @@ void ShpMemTiles::StoreGeometry( const std::string& name, uint minzoom, int32_t score, - AttributeIndex attrIdx + AttributeIndex attrIdx, + bool knownValid ) { geom::model::box box; @@ -175,9 +220,9 @@ void ShpMemTiles::StoreGeometry( case POLYGON_: { - // Checked once here, so tiles can clip valid polygons on the fast path + // Checked once here (unless the caller has), so tiles can clip valid polygons on the fast path const MultiPolygon &mp = boost::get(geometry); - NodeID oid = storeMultiPolygon(mp, multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); + NodeID oid = storeMultiPolygon(mp, knownValid || multiPolygonDefinitelyValid(mp) || geom::is_valid(mp)); oo = std::make_shared(geomType, layerNum, oid, attrIdx, minzoom); std::vector oolist { *oo }; addGeometryToIndex(boost::get(geometry), oolist, 0); diff --git a/src/shp_processor.cpp b/src/shp_processor.cpp index f3b3c4f2..b32291b2 100644 --- a/src/shp_processor.cpp +++ b/src/shp_processor.cpp @@ -366,8 +366,8 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx // Make valid if needs be string reason; - const bool precheckValid = multiPolygonDefinitelyValid(multi); - if (!precheckValid && !geom::is_valid(multi, reason)) { + const bool valid = multiPolygonDefinitelyValid(multi) || geom::is_valid(multi, reason); + if (!valid) { if (verbose) cerr << "Shapefile entity " << shape->nShapeId << " type " << shapeType << " is invalid. Parts:" << shape->nParts << ". Reason:" << reason; make_valid(multi); if (verbose) { @@ -376,11 +376,17 @@ void ShpProcessor::processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx cerr << endl; } } - // clip to bounding box - MultiPolygon out; - geom::intersection(multi, clippingBox, out); - if (boost::size(out)>0) { - shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + // clip to bounding box (a valid polygon wholly inside it is stored as it is) + Box envelope; + geom::envelope(multi, envelope); + if (valid && geom::covered_by(envelope, clippingBox)) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, multi, layer.indexed, hasName, name, minzoom, score, attrIdx, true); + } else { + MultiPolygon out; + geom::intersection(multi, clippingBox, out); + if (boost::size(out)>0) { + shpMemTiles.StoreGeometry(layerNum, layer.name, POLYGON_, out, layer.indexed, hasName, name, minzoom, score, attrIdx); + } } } else { diff --git a/src/sorted_node_store.cpp b/src/sorted_node_store.cpp index c6d282e8..81844aad 100644 --- a/src/sorted_node_store.cpp +++ b/src/sorted_node_store.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include #include #include #include @@ -12,6 +15,10 @@ #ifdef _MSC_VER #include #endif +#if (defined(__GNUC__) || defined(__clang__)) && defined(__x86_64__) +#include +#define SORTED_NODE_STORE_SSSE3_DECODE +#endif namespace SortedNodeStoreTypes { const uint16_t GroupSize = 256; @@ -30,7 +37,15 @@ namespace SortedNodeStoreTypes { struct CachedChunk { uint64_t nodeMask[4]; uint32_t rankBase[4]; - LatpLon nodes[ChunkSize]; + // nodes[0 .. decoded - 1] are valid; the rest is decoded on demand (see extendChunk) + uint32_t decoded; + uint32_t count; + // the two streamvbyte streams: keys, and the data of the next group of 4 to decode + const uint8_t* latpKeys; + const uint8_t* lonKeys; + const uint8_t* latpNext; + const uint8_t* lonNext; + LatpLon nodes[ChunkSize + 4]; }; struct ChunkCache { @@ -156,15 +171,6 @@ namespace SortedNodeStoreTypes { return ptr->nodes[nodeOffset]; } - inline LatpLon cachedNode(const CachedChunk& cached, NodeID id) { - const size_t nodeIndex = id % ChunkSize; - const size_t w = nodeIndex / 64; - const uint64_t bit = 1ull << (nodeIndex % 64); - const uint64_t word = cached.nodeMask[w]; - if (!(word & bit)) - throw std::out_of_range("SortedNodeStore: node " + std::to_string(id) + " missing, no node"); - return cached.nodes[cached.rankBase[w] + popcount64(word & (bit - 1))]; - } // Same as svb_zigzag_decode_32 in streamvbyte_zigzag.c. inline int32_t zigzagDecode(uint32_t val) { @@ -197,6 +203,76 @@ namespace SortedNodeStoreTypes { return val; } +#ifdef SORTED_NODE_STORE_SSSE3_DECODE + // streamvbyte decoding tables: for each key byte (4 x 2-bit length codes), the pshufb + // pattern that spreads its 4..16 data bytes into 4 uint32 lanes, and the bytes it uses. + struct SvbTables { + alignas(16) uint8_t shuffle[256][16]; + uint8_t length[256]; + SvbTables() { + for (unsigned key = 0; key < 256; key++) { + unsigned offset = 0; + for (unsigned lane = 0; lane < 4; lane++) { + const unsigned bytes = ((key >> (2 * lane)) & 3) + 1; + for (unsigned b = 0; b < 4; b++) + shuffle[key][lane * 4 + b] = b < bytes ? offset + b : 0x80; + offset += bytes; + } + length[key] = offset; + } + } + }; + const SvbTables svbTables; + const bool cpuHasSsse3 = [] { __builtin_cpu_init(); return __builtin_cpu_supports("ssse3") != 0; }(); + + __attribute__((target("ssse3"))) + inline __m128i svbDecodeDeltas(const uint8_t*& data, uint8_t key, __m128i prev) { + __m128i v = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)data), _mm_load_si128((const __m128i*)svbTables.shuffle[key])); + data += svbTables.length[key]; + // zigzag decode, then the running sum: lane i = prev + d0 + .. + di (mod 2^32) + v = _mm_xor_si128(_mm_srli_epi32(v, 1), _mm_sub_epi32(_mm_setzero_si128(), _mm_and_si128(v, _mm_set1_epi32(1)))); + v = _mm_add_epi32(v, _mm_slli_si128(v, 4)); + v = _mm_add_epi32(v, _mm_slli_si128(v, 8)); + return _mm_add_epi32(v, prev); + } + +#ifndef SORTED_NODE_STORE_LAZY_QUADS +#define SORTED_NODE_STORE_LAZY_QUADS 4 +#endif + // Values are decoded on demand in steps of this many groups of 4. + const size_t LazyQuads = SORTED_NODE_STORE_LAZY_QUADS; + + // Fused streamvbyte + zigzag delta decode of both streams, 4 values at a time, writing + // interleaved LatpLons: extends c.nodes so that nodes[rank] is valid (rank < c.count). + // Writes whole groups of 4, i.e. up to 3 entries past nodes[count - 1]; the values there + // are garbage and never read. Reads up to 15 bytes past the end of the lon stream, as + // streamvbyte's own decoder does (each group is followed by STREAMVBYTE_PADDING). + __attribute__((target("ssse3"), noinline)) + void extendChunk(CachedChunk& c, size_t rank) { + const size_t keyLen = (c.count - 1 + 3) / 4; + // nodes[k] for k >= 1 is the (k - 1)-th delta of group (k - 1) / 4 + size_t q = (c.decoded - 1) / 4; + const size_t qEnd = std::min(keyLen, ((rank - 1) / 4 / LazyQuads + 1) * LazyQuads); + const uint8_t* latpPtr = c.latpNext; + const uint8_t* lonPtr = c.lonNext; + __m128i prevLatp = _mm_set1_epi32(c.nodes[c.decoded - 1].latp); + __m128i prevLon = _mm_set1_epi32(c.nodes[c.decoded - 1].lon); + LatpLon* dst = c.nodes + 1; + for (; q < qEnd; q++) { + const __m128i a = svbDecodeDeltas(latpPtr, c.latpKeys[q], prevLatp); + const __m128i b = svbDecodeDeltas(lonPtr, c.lonKeys[q], prevLon); + prevLatp = _mm_shuffle_epi32(a, 0xFF); + prevLon = _mm_shuffle_epi32(b, 0xFF); + static_assert(sizeof(LatpLon) == 8, "LatpLon is {int32 latp, int32 lon}"); + _mm_storeu_si128((__m128i*)(dst + 4 * q), _mm_unpacklo_epi32(a, b)); + _mm_storeu_si128((__m128i*)(dst + 4 * q + 2), _mm_unpackhi_epi32(a, b)); + } + c.latpNext = latpPtr; + c.lonNext = lonPtr; + c.decoded = std::min(c.count, 1 + 4 * qEnd); + } +#endif + // Decode a compressed chunk into `out`, in rank order. void decodeChunk(const CompressedChunkInfo* ptr, CachedChunk& out) { const size_t latpSize = (ptr->flags >> 10) & ((1 << 10) - 1); @@ -217,6 +293,21 @@ namespace SortedNodeStoreTypes { int32_t lon = ptr->firstLon; out.nodes[0] = { latp, lon }; LatpLon* dst = out.nodes + 1; + out.count = count; + +#ifdef SORTED_NODE_STORE_SSSE3_DECODE + if (cpuHasSsse3) { + // decoded on demand by cachedNode + const size_t keyLen = (n + 3) / 4; + out.decoded = 1; + out.latpKeys = latpData; + out.lonKeys = lonData; + out.latpNext = latpData + keyLen; + out.lonNext = lonData + keyLen; + return; + } +#endif + out.decoded = count; if (n < 64) { // Fused scalar streamvbyte + zigzag delta decode of both streams. @@ -263,6 +354,44 @@ namespace SortedNodeStoreTypes { } } + // Cache an uncompressed chunk too: copy its nodes, so that later lookups in it are hits + // rather than a walk of the group and chunk index each time. + inline void fillUncompressed(const UncompressedChunkInfo* ptr, CachedChunk& out) { + uint32_t count = 0; + for (size_t w = 0; w < 4; w++) { + const uint64_t word = loadMaskWord(ptr->nodeMask + w * 8); + out.nodeMask[w] = word; + out.rankBase[w] = count; + count += popcount64(word); + } + memcpy(out.nodes, ptr->nodes, count * sizeof(LatpLon)); + out.count = count; + out.decoded = count; + } + + // Fill a cache entry from the chunk at basePtr, compressed or not. + inline void fillEntry(const ChunkInfoBase* basePtr, CachedChunk& out) { + if (basePtr->flags & ChunkCompressed) + decodeChunk((const CompressedChunkInfo*)basePtr, out); + else + fillUncompressed((const UncompressedChunkInfo*)basePtr, out); + } + + inline LatpLon cachedNode(CachedChunk& cached, NodeID id) { + const size_t nodeIndex = id % ChunkSize; + const size_t w = nodeIndex / 64; + const uint64_t bit = 1ull << (nodeIndex % 64); + const uint64_t word = cached.nodeMask[w]; + if (!(word & bit)) + throw std::out_of_range("SortedNodeStore: node " + std::to_string(id) + " missing, no node"); + const size_t rank = cached.rankBase[w] + popcount64(word & (bit - 1)); +#ifdef SORTED_NODE_STORE_SSSE3_DECODE + if (rank >= cached.decoded) + extendChunk(cached, rank); +#endif + return cached.nodes[rank]; + } + struct ThreadStorage { ThreadStorage(): collectingOrphans(true), @@ -391,11 +520,9 @@ LatpLon SortedNodeStore::at(const NodeID id) const { return cachedNode(cache.entries[cacheIndex], id); const ChunkInfoBase* basePtr = findChunk(groups.data(), id); - if (!(basePtr->flags & ChunkCompressed)) - return uncompressedNode(basePtr, id); CachedChunk& entry = cache.entries[cacheIndex]; - decodeChunk((const CompressedChunkInfo*)basePtr, entry); + fillEntry(basePtr, entry); cache.keys[cacheIndex] = key; return cachedNode(entry, id); } @@ -404,7 +531,7 @@ void SortedNodeStore::at(const NodeID* ids, size_t n, LatpLon* out) const { ChunkCache& cache = getChunkCache(); // Consecutive ids usually share a chunk: reuse its cache entry directly. uint64_t lastChunkId = ~0ull; - const CachedChunk* entry = nullptr; + CachedChunk* entry = nullptr; for (size_t i = 0; i < n; i++) { const NodeID id = ids[i]; @@ -424,12 +551,7 @@ void SortedNodeStore::at(const NodeID* ids, size_t n, LatpLon* out) const { const size_t cacheIndex = chunkCacheIndex(key); if (cache.keys[cacheIndex] != key) { const ChunkInfoBase* basePtr = findChunk(groups.data(), id); - if (!(basePtr->flags & ChunkCompressed)) { - lastChunkId = ~0ull; - out[i] = uncompressedNode(basePtr, id); - continue; - } - decodeChunk((const CompressedChunkInfo*)basePtr, cache.entries[cacheIndex]); + fillEntry(basePtr, cache.entries[cacheIndex]); cache.keys[cacheIndex] = key; } entry = &cache.entries[cacheIndex]; @@ -530,20 +652,31 @@ void SortedNodeStore::finalize(size_t threadNum) { } workerBuffers.clear(); - // Empty the orphanage into the index. - std::vector copy; - for (const auto& entry: orphanage) { - for (const auto& orphan: entry.second) - copy.push_back(orphan); - - // Orphans may come from different workers, and thus be unsorted. - std::sort( - copy.begin(), - copy.end(), - [](auto const &a, auto const &b) { return a.first < b.first; } - ); - publishGroup(copy); - copy.clear(); + // Empty the orphanage into the index. The groups are independent and publishGroup is what the + // workers call during the pass anyway, so publish them from all threads: one after the other, + // the planet's leftover groups took ~10 s with every other thread idle. + std::vector*> orphanGroups; + for (const auto& entry: orphanage) + orphanGroups.push_back(&entry.second); + std::atomic nextGroup(0); + auto publishOrphans = [&]() { + std::vector copy; + for (size_t i = nextGroup++; i < orphanGroups.size(); i = nextGroup++) { + copy.assign(orphanGroups[i]->begin(), orphanGroups[i]->end()); + // Orphans may come from different workers, and thus be unsorted. + std::sort( + copy.begin(), + copy.end(), + [](auto const &a, auto const &b) { return a.first < b.first; } + ); + publishGroup(copy); + } + }; + { + boost::asio::thread_pool pool(std::max(1, threadNum)); + for (unsigned int t = 0; t < std::max(1, threadNum); t++) + boost::asio::post(pool, publishOrphans); + pool.join(); } orphanage.clear(); diff --git a/src/sorted_way_store.cpp b/src/sorted_way_store.cpp index 02f66147..8383d04a 100644 --- a/src/sorted_way_store.cpp +++ b/src/sorted_way_store.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -6,8 +9,41 @@ #include "external/streamvbyte.h" #include "external/streamvbyte_zigzag.h" #include "sorted_way_store.h" +#include #include "node_store.h" +namespace { +#if (defined(__GNUC__) || defined(__clang__)) && defined(__x86_64__) && !defined(__POPCNT__) && \ + defined(HAVE_ASM_POPCNT) && defined(HAVE_CPUID) + // Without -mpopcnt, __builtin_popcountll is a call into libgcc. Use the + // POPCNT instruction (libpopcnt's popcnt64) when the CPU has it. + const bool cpuHasPopcnt = (get_cpuid() & bit_POPCNT) != 0; + inline uint64_t popcountWord(uint64_t x) { return cpuHasPopcnt ? popcnt64(x) : popcount64(x); } +#elif defined(__GNUC__) || defined(__clang__) + inline uint64_t popcountWord(uint64_t x) { return __builtin_popcountll(x); } +#else + inline uint64_t popcountWord(uint64_t x) { return popcount64(x); } +#endif + + // Number of set bits in the first `size` bytes (at most 32) of a mask: the same count as + // libpopcnt's popcnt(data, size), without its call and dispatch. + inline uint64_t maskPopcnt(const void* data, uint64_t size) { + const uint8_t* p = (const uint8_t*)data; + uint64_t rv = 0; + for (; size >= 8; size -= 8, p += 8) { + uint64_t word; + memcpy(&word, p, 8); + rv += popcountWord(word); + } + if (size > 0) { + uint64_t word = 0; + memcpy(&word, p, size); + rv += popcountWord(word); + } + return rv; + } +} + namespace SortedWayStoreTypes { const uint16_t GroupSize = 256; const uint16_t ChunkSize = 256; @@ -103,10 +139,10 @@ bool SortedWayStore::contains(size_t shard, WayID id) const { size_t chunkOffset = 0; { - chunkOffset = popcnt(groupPtr->chunkMask, chunkMaskByte); + chunkOffset = maskPopcnt(groupPtr->chunkMask, chunkMaskByte); uint8_t maskByte = groupPtr->chunkMask[chunkMaskByte]; maskByte = maskByte & ((1 << chunkMaskBit) - 1); - chunkOffset += popcnt(&maskByte, 1); + chunkOffset += maskPopcnt(&maskByte, 1); if (!(groupPtr->chunkMask[chunkMaskByte] & (1 << chunkMaskBit))) return false; @@ -116,20 +152,20 @@ bool SortedWayStore::contains(size_t shard, WayID id) const { { size_t wayOffset = 0; - wayOffset = popcnt(chunkPtr->smallWayMask, wayMaskByte); + wayOffset = maskPopcnt(chunkPtr->smallWayMask, wayMaskByte); uint8_t maskByte = chunkPtr->smallWayMask[wayMaskByte]; maskByte = maskByte & ((1 << wayMaskBit) - 1); - wayOffset += popcnt(&maskByte, 1); + wayOffset += maskPopcnt(&maskByte, 1); if (chunkPtr->smallWayMask[wayMaskByte] & (1 << wayMaskBit)) return true; } size_t wayOffset = 0; - wayOffset += popcnt(chunkPtr->smallWayMask, 32); - wayOffset += popcnt(chunkPtr->bigWayMask, wayMaskByte); + wayOffset += maskPopcnt(chunkPtr->smallWayMask, 32); + wayOffset += maskPopcnt(chunkPtr->bigWayMask, wayMaskByte); uint8_t maskByte = chunkPtr->bigWayMask[wayMaskByte]; maskByte = maskByte & ((1 << wayMaskBit) - 1); - wayOffset += popcnt(&maskByte, 1); + wayOffset += maskPopcnt(&maskByte, 1); if (!(chunkPtr->bigWayMask[wayMaskByte] & (1 << wayMaskBit))) return false; @@ -159,27 +195,27 @@ void SortedWayStore::at(WayID id, std::vector& rv) const { size_t chunkOffset = 0; { - chunkOffset = popcnt(groupPtr->chunkMask, chunkMaskByte); + chunkOffset = maskPopcnt(groupPtr->chunkMask, chunkMaskByte); uint8_t maskByte = groupPtr->chunkMask[chunkMaskByte]; maskByte = maskByte & ((1 << chunkMaskBit) - 1); - chunkOffset += popcnt(&maskByte, 1); + chunkOffset += maskPopcnt(&maskByte, 1); if (!(groupPtr->chunkMask[chunkMaskByte] & (1 << chunkMaskBit))) throw std::out_of_range("SortedWayStore: way " + std::to_string(id) + " missing, no chunk"); } ChunkInfo* chunkPtr = (ChunkInfo*)((char*)groupPtr + groupPtr->chunkOffsets[chunkOffset]); - const size_t numWays = popcnt(chunkPtr->smallWayMask, 32) + popcnt(chunkPtr->bigWayMask, 32); + const size_t numWays = maskPopcnt(chunkPtr->smallWayMask, 32) + maskPopcnt(chunkPtr->bigWayMask, 32); uint8_t* const endOfWayOffsetPtr = (uint8_t*)(chunkPtr->wayOffsets + numWays); EncodedWay* wayPtr = nullptr; { size_t wayOffset = 0; - wayOffset = popcnt(chunkPtr->smallWayMask, wayMaskByte); + wayOffset = maskPopcnt(chunkPtr->smallWayMask, wayMaskByte); uint8_t maskByte = chunkPtr->smallWayMask[wayMaskByte]; maskByte = maskByte & ((1 << wayMaskBit) - 1); - wayOffset += popcnt(&maskByte, 1); + wayOffset += maskPopcnt(&maskByte, 1); if (chunkPtr->smallWayMask[wayMaskByte] & (1 << wayMaskBit)) { wayPtr = (EncodedWay*)(endOfWayOffsetPtr + chunkPtr->wayOffsets[wayOffset]); } @@ -188,11 +224,11 @@ void SortedWayStore::at(WayID id, std::vector& rv) const { // If we didn't find it in small ways, look in big ways. if (wayPtr == nullptr) { size_t wayOffset = 0; - wayOffset += popcnt(chunkPtr->smallWayMask, 32); - wayOffset += popcnt(chunkPtr->bigWayMask, wayMaskByte); + wayOffset += maskPopcnt(chunkPtr->smallWayMask, 32); + wayOffset += maskPopcnt(chunkPtr->bigWayMask, wayMaskByte); uint8_t maskByte = chunkPtr->bigWayMask[wayMaskByte]; maskByte = maskByte & ((1 << wayMaskBit) - 1); - wayOffset += popcnt(&maskByte, 1); + wayOffset += maskPopcnt(&maskByte, 1); if (!(chunkPtr->bigWayMask[wayMaskByte] & (1 << wayMaskBit))) throw std::out_of_range("SortedWayStore: way " + std::to_string(id) + " missing, no way"); @@ -241,7 +277,6 @@ void SortedWayStore::insertNodes(const std::vectorclear(); } tls.localWays->push_back(el); i++; @@ -271,27 +306,49 @@ std::size_t SortedWayStore::size() const { } void SortedWayStore::finalize(unsigned int threadNum) { - for (const auto& buffer: workerBuffers) { + for (auto& buffer: workerBuffers) { if (buffer.size() > 0) { collectOrphans(buffer); } } workerBuffers.clear(); - // Empty the orphanage into the index. - std::vector>> copy; - for (const auto& entry: orphanage) { - for (const auto& orphan: entry.second) - copy.push_back(orphan); - - // Orphans may come from different workers, and thus be unsorted. - std::sort( - copy.begin(), - copy.end(), - [](auto const &a, auto const &b) { return a.first < b.first; } - ); - publishGroup(copy); - copy.clear(); + // Empty the orphanage into the index. The groups are independent and publishGroup is what the + // workers call during the pass anyway, so publish them from all threads: one after the other, + // the planet's leftover groups took ~10 s with every other thread idle. Each group's ways are + // moved together, published and freed by the thread that took it (freeing ~70M node lists on + // one thread afterwards took seconds on the planet). + std::vector>>>*> orphanGroups; + for (auto& entry: orphanage) + orphanGroups.push_back(&entry.second); + std::atomic nextGroup(0); + auto publishOrphans = [&]() { + std::vector>> ways; + for (size_t i = nextGroup++; i < orphanGroups.size(); i = nextGroup++) { + auto& pieces = *orphanGroups[i]; + size_t total = 0; + for (const auto& piece : pieces) total += piece.size(); + ways.clear(); + ways.reserve(total); + for (auto& piece : pieces) + for (auto& way : piece) ways.emplace_back(way.first, std::move(way.second)); + std::vector>>>().swap(pieces); + // Orphans may come from different workers, and thus be unsorted. (Way IDs are unique, + // so the order doesn't depend on the order of the pieces.) + std::sort( + ways.begin(), + ways.end(), + [](auto const &a, auto const &b) { return a.first < b.first; } + ); + publishGroup(ways); + } + std::vector>>().swap(ways); + }; + { + boost::asio::thread_pool pool(std::max(1, threadNum)); + for (unsigned int t = 0; t < std::max(1, threadNum); t++) + boost::asio::post(pool, publishOrphans); + pool.join(); } orphanage.clear(); @@ -307,18 +364,21 @@ void SortedWayStore::batchStart() { return; collectOrphans(*tls.localWays); - tls.localWays->clear(); } -void SortedWayStore::collectOrphans(const std::vector>>& orphans) { - std::lock_guard lock(orphanageMutex); +void SortedWayStore::collectOrphans(std::vector>>& orphans) { size_t groupIndex = orphans[0].first / (GroupSize * ChunkSize); - - std::vector>>& vec = orphanage[groupIndex]; - const size_t i = vec.size(); - - vec.resize(i + orphans.size()); - std::copy(orphans.begin(), orphans.end(), vec.begin() + i); + // The orphanage gets the ways (moved, not copied) in a vector of their size; the thread's + // buffer keeps its capacity for the next pieces. Handing over the buffer itself kept its + // whole capacity (a group's worth of ways) for every piece until the end of the way phase. + std::vector>> piece; + piece.reserve(orphans.size()); + std::move(orphans.begin(), orphans.end(), std::back_inserter(piece)); + orphans.clear(); + { + std::lock_guard lock(orphanageMutex); + orphanage[groupIndex].push_back(std::move(piece)); + } } std::vector SortedWayStore::decodeWay(uint16_t flags, const uint8_t* input) { @@ -327,13 +387,81 @@ std::vector SortedWayStore::decodeWay(uint16_t flags, const uint8_t* inp return rv; }; +namespace { + // Read a streamvbyte value of (code + 1) bytes at p, never reading at or beyond `end`. + inline uint32_t svbValue(const uint8_t* p, const uint8_t* end, uint32_t code) { + static const uint32_t lengthMask[4] = { 0xFFu, 0xFFFFu, 0xFFFFFFu, 0xFFFFFFFFu }; + if (end - p >= 4) { + uint32_t val; + memcpy(&val, p, 4); + return val & lengthMask[code]; + } + uint32_t val = 0; + for (uint32_t b = 0; b <= code; b++) + val |= (uint32_t)p[b] << (8 * b); + return val; + } +} + void SortedWayStore::decodeWay(uint16_t flags, const uint8_t* input, std::vector& rv) { + const uint16_t length = flags & 0b0000011111111111; + if (length > 0) { + // Fast path: write the ids straight into rv, decoding the streamvbyte and zigzag delta + // in one pass (the same values as the general code below computes). + const bool closed = flags & ClosedWay; + rv.resize(length + (closed ? 1 : 0)); + NodeID* out = rv.data(); + const uint8_t* highNibbles = nullptr; + uint64_t uniformHigh = 0; + if (!(flags & UniformUpperBits)) { + highNibbles = input; + input += (length + 1) / 2; + } else { + uniformHigh = (uint64_t)*input << 31; + input++; + } + auto high = [&](size_t i) -> uint64_t { + if (!highNibbles) return uniformHigh; + return (uint64_t)((highNibbles[i / 2] >> (4 * (i & 1))) & 0b00001111) << 31; + }; + if (!(flags & CompressedWay)) { + for (size_t i = 0; i < length; i++) { + uint32_t low; + memcpy(&low, input + 4 * i, 4); + out[i] = high(i) | low; + } + } else { + uint16_t compressedSize; + memcpy(&compressedSize, input, 2); + input += 2; + uint32_t firstInt; + memcpy(&firstInt, input, 4); + input += 4; + out[0] = high(0) | firstInt; + // length - 1 zigzag deltas, streamvbyte coded: keys, then the data + const size_t n = length - 1; + const uint8_t* keys = input; + const uint8_t* data = input + (n + 3) / 4; + const uint8_t* end = input + compressedSize; + uint32_t prev = firstInt; + for (size_t k = 0; k < n; k++) { + const uint32_t code = (keys[k >> 2] >> ((k & 3) * 2)) & 3; + const uint32_t z = svbValue(data, end, code); + data += code + 1; + prev += (z >> 1) ^ (0 - (z & 1)); + out[k + 1] = high(k + 1) | prev; + } + } + if (closed) + out[length] = out[0]; + return; + } + rv.clear(); bool isCompressed = flags & CompressedWay; bool isClosed = flags & ClosedWay; - const uint16_t length = flags & 0b0000011111111111; rv.reserve(length + (isClosed ? 1 : 0)); if (!(flags & UniformUpperBits)) { diff --git a/src/tag_map.cpp b/src/tag_map.cpp index cc63958f..c4452d6d 100644 --- a/src/tag_map.cpp +++ b/src/tag_map.cpp @@ -7,6 +7,9 @@ TagMap::TagMap() { } void TagMap::reset() { + tags.clear(); + if (!built) return; + built = false; // Only clear shards that were used since the last reset. for (unsigned i = 0, mask = usedKeyShards; mask; i++, mask >>= 1) if (mask & 1) @@ -18,10 +21,6 @@ void TagMap::reset() { usedValueShards = 0; } -bool TagMap::empty() const { - return usedKeyShards == 0; -} - static inline std::size_t hashString(const char* str, size_t size) { // This is a pretty crappy hash function in terms of bit // avalanching and distribution of output values. @@ -38,20 +37,37 @@ static inline std::size_t hashString(const char* str, size_t size) { return hash; } -uint32_t TagMap::ensureKey(const protozero::data_view& key) { +// The first (up to) 8 bytes of a key as an integer: keys of the same size are equal if their +// prefixes are and, beyond 8 bytes, the rest is +static inline uint64_t keyPrefix(const char* str, size_t size) { + uint64_t prefix = 0; + if (size >= 8) + memcpy(&prefix, str, 8); + else + for (size_t i = 0; i < size; i++) + prefix |= static_cast(static_cast(str[i])) << (8 * i); + return prefix; +} + +static inline bool sameKey(const char* a, const char* b, size_t size) { + return size <= 8 || memcmp(a + 8, b + 8, size - 8) == 0; +} + +uint32_t TagMap::ensureKey(const protozero::data_view& key) const { const uint16_t shard = hashString(key.data(), key.size()) & (SHARDS - 1); std::vector& bucket = keys[shard]; const uint32_t size = key.size(); + const uint64_t prefix = keyPrefix(key.data(), size); for (size_t i = 0; i < bucket.size(); i++) - if (bucket[i].size == size && memcmp(bucket[i].data, key.data(), size) == 0) + if (bucket[i].size == size && bucket[i].prefix == prefix && sameKey(bucket[i].data, key.data(), size)) return shard << 16 | i; - bucket.push_back(KeySlot{key.data(), size, 0}); + bucket.push_back(KeySlot{key.data(), size, 0, prefix}); usedKeyShards |= 1u << shard; return shard << 16 | (bucket.size() - 1); } -uint32_t TagMap::ensureValue(const protozero::data_view& value) { +uint32_t TagMap::ensureValue(const protozero::data_view& value) const { const uint16_t shard = hashString(value.data(), value.size()) & (SHARDS - 1); std::vector& bucket = values[shard]; for (size_t i = 0; i < bucket.size(); i++) @@ -63,7 +79,13 @@ uint32_t TagMap::ensureValue(const protozero::data_view& value) { return shard << 16 | (bucket.size() - 1); } -void TagMap::addTag(const protozero::data_view& key, const protozero::data_view& value) { +void TagMap::build() const { + built = true; + for (const Tag& tag : tags) + insert(tag.key, tag.value); +} + +void TagMap::insert(const protozero::data_view& key, const protozero::data_view& value) const { uint32_t valueLoc = ensureValue(value); uint32_t keyLoc = ensureKey(key); @@ -72,13 +94,17 @@ void TagMap::addTag(const protozero::data_view& key, const protozero::data_view& int64_t TagMap::getKey(const char* key, size_t size) const { // Return -1 if key not found, else return its keyLoc. + ensureBuilt(); const uint16_t shard = hashString(key, size) & (SHARDS - 1); const std::vector& bucket = keys[shard]; + if (bucket.empty()) + return -1; + const uint64_t prefix = keyPrefix(key, size); for (size_t i = 0; i < bucket.size(); i++) { - if (bucket[i].size != size) + if (bucket[i].size != size || bucket[i].prefix != prefix) continue; - if (memcmp(bucket[i].data, key, size) == 0) + if (sameKey(bucket[i].data, key, size)) return shard << 16 | i; } @@ -87,6 +113,7 @@ int64_t TagMap::getKey(const char* key, size_t size) const { int64_t TagMap::getValue(const char* value, size_t size) const { // Return -1 if value not found, else return its valueLoc. + ensureBuilt(); const uint16_t shard = hashString(value, size) & (SHARDS - 1); const std::vector& bucket = values[shard]; for (size_t i = 0; i < bucket.size(); i++) { @@ -101,16 +128,19 @@ int64_t TagMap::getValue(const char* value, size_t size) const { } const protozero::data_view* TagMap::getValueFromKey(uint32_t keyLoc) const { + ensureBuilt(); const uint32_t valueLoc = keys[keyLoc >> 16][keyLoc & 0xFFFF].valueLoc; return &values[valueLoc >> 16][valueLoc & 0xFFFF]; } const protozero::data_view* TagMap::getValue(uint32_t valueLoc) const { + ensureBuilt(); return &values[valueLoc >> 16][valueLoc & 0xFFFF]; } boost::container::flat_map TagMap::exportToBoostMap() const { boost::container::flat_map rv; + ensureBuilt(); for (size_t i = 0; i < keys.size(); i++) { for (size_t j = 0; j < keys[i].size(); j++) { @@ -124,6 +154,7 @@ boost::container::flat_map TagMap::exportToBoostMap() } TagMap::Iterator TagMap::begin() const { + ensureBuilt(); size_t shard = 0; while(keys.size() > shard && keys[shard].size() == 0) shard++; diff --git a/src/tile_coordinates_set.cpp b/src/tile_coordinates_set.cpp index e6d6e78e..a23fbc99 100644 --- a/src/tile_coordinates_set.cpp +++ b/src/tile_coordinates_set.cpp @@ -40,6 +40,12 @@ void TileCoordinatesSet::forEachSet(const std::function> 12] |= (uint64_t)1 << ((w >> 6) & 63); } +// Fills each row of the box word by word, rather than tile by tile. +void PreciseTileCoordinatesSet::setBox(TileCoordinate minX, TileCoordinate maxX, TileCoordinate minY, TileCoordinate maxY) { + const uint64_t dim = 1ull << zoom_; + if (minX > maxX || minY > maxY || minX >= dim || minY >= dim) + return; + maxX = std::min(maxX, dim - 1); + maxY = std::min(maxY, dim - 1); + for (uint64_t x = minX; x <= maxX; x++) { + const uint64_t first = x * dim + minY, last = x * dim + maxY; + const uint64_t w0 = first >> 6, w1 = last >> 6; + for (uint64_t w = w0; w <= w1; w++) { + uint64_t mask = ~0ull; + if (w == w0) mask &= ~0ull << (first & 63); + if (w == w1) mask &= ~0ull >> (63 - (last & 63)); + tiles[w] |= mask; + } + for (uint64_t block = w0 >> 6; block <= (w1 >> 6); block++) + summary[block >> 6] |= (uint64_t)1 << (block & 63); + } +} + +void PreciseTileCoordinatesSet::merge(const PreciseTileCoordinatesSet& other) { + if (other.zoom_ != zoom_) + throw std::invalid_argument("PreciseTileCoordinatesSet::merge: zooms differ"); + forEachMarkedWord(other.summary, numWords, [&](size_t w) { tiles[w] |= other.tiles[w]; }); + for (size_t s = 0; s < summary.size(); s++) + summary[s] |= other.summary[s]; +} + +void PreciseTileCoordinatesSet::mergeRange(const PreciseTileCoordinatesSet& other, size_t sBegin, size_t sEnd) { + if (other.zoom_ != zoom_) + throw std::invalid_argument("PreciseTileCoordinatesSet::mergeRange: zooms differ"); + sEnd = std::min(sEnd, summary.size()); + for (size_t s = sBegin; s < sEnd; s++) { + uint64_t bits = other.summary[s]; + summary[s] |= bits; + while (bits) { + const size_t block = s * 64 + countTrailingZeros(bits); + bits &= bits - 1; + const size_t wEnd = std::min((block + 1) * 64, numWords); + for (size_t w = block * 64; w < wEnd; w++) + tiles[w] |= other.tiles[w]; + } + } +} + void PreciseTileCoordinatesSet::forEachSet(const std::function& f) const { // loc = x * 2^zoom + y, so ascending loc order is x-major, y-minor. // Only blocks marked in the summary can hold set bits; visiting them in @@ -134,3 +186,24 @@ void LossyTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { throw std::logic_error("LossyTileCoordinatesSet::set() is not implemented; LossyTileCoordinatesSet is read-only"); } +void LossyTileCoordinatesSet::forEachSet(const std::function& f) const { + // Expand the underlying set instead of testing all 4^zoom tiles (2^34 at z17). + // Its tiles arrive x-major, y-minor; one column of them becomes `scale` + // columns here, which keeps this set's order x-major, y-minor too. + std::vector column; + TileCoordinate columnX = 0; + auto flush = [&]() { + for (TileCoordinate dx = 0; dx < scale; dx++) + for (TileCoordinate y : column) + for (TileCoordinate dy = 0; dy < scale; dy++) + f(columnX * scale + dx, y * scale + dy); + column.clear(); + }; + tiles.forEachSet([&](TileCoordinate x, TileCoordinate y) { + if (!column.empty() && x != columnX) flush(); + columnX = x; + column.push_back(y); + }); + if (!column.empty()) flush(); +} + diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 89b367e0..b2c5a749 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -1,16 +1,37 @@ +#include +#include #include +#include #include #include +#include #include #include +#include #include "tile_data.h" +#include #include "coordinates_geom.h" #include "leased_store.h" +#include +#include "stable_hash.h" +#include "clip_line.h" + +thread_local unsigned objectsUpToZoom = 0; +std::vector z12ObjectCounts; #include using namespace std; extern bool verbose; +namespace { +// Each thread's staging for the source it last added to; epochs are unique across sources +std::atomic nextStagingEpoch(1); +thread_local uint64_t tlsStagingEpoch = 0; +thread_local void* tlsStaging = nullptr; // TileDataSource::IndexStagings +// Entries staged per thread before they are moved to the shards +const size_t INDEX_STAGING_SIZE = 1024; +} + // Human-readable name for a Boost.Geometry validity failure, used for diagnostics. static const char* validityFailureName(geom::validity_failure_type failure) { switch (failure) { @@ -46,11 +67,40 @@ thread_local LeasedStore linestringStore; thread_local LeasedStore multilinestringStore; thread_local LeasedStore multipolygonStore; +// Never destroyed: the thread-locals that use them may be destroyed after the statics (macOS) +static std::unordered_map &leaseSources() { + static std::unordered_map &sources = *new std::unordered_map(); + return sources; +} +std::mutex &leaseSourcesMutex() { + static std::mutex &m = *new std::mutex; + return m; +} +uint64_t registerLeaseSource(const TileDataSource *source) { + static uint64_t last = 0; + std::lock_guard lock(leaseSourcesMutex()); + const uint64_t number = ++last; + leaseSources()[source] = number; + return number; +} +void unregisterLeaseSource(const TileDataSource *source) { + std::lock_guard lock(leaseSourcesMutex()); + leaseSources().erase(source); +} +bool leaseSourceAlive(const TileDataSource *source, uint64_t number) { + const auto it = leaseSources().find(source); + return it != leaseSources().end() && it->second == number; +} +TileDataSource::~TileDataSource() { + unregisterLeaseSource(this); +} + TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID) : includeID(includeID), z6OffsetDivisor(indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1), objectsMutex(threadNum * 4), + stagingEpoch(0), objects(CLUSTER_ZOOM_AREA), lowZoomObjects(CLUSTER_ZOOM_AREA), objectsWithIds(CLUSTER_ZOOM_AREA), @@ -70,6 +120,8 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool in throw std::out_of_range("TileDataSource: indexZoom cannot be higher than 14, but was " + std::to_string(indexZoom)); + stagingEpoch = nextStagingEpoch++; + shardBits = 0; numShards = 1; while(numShards < threadNum) { @@ -85,26 +137,147 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool in } } -thread_local std::vector>* tlsPendingSmallIndexObjects = nullptr; thread_local uint64_t clipCacheBatchId = 0; thread_local bool clipCacheNoStore = false; +thread_local bool clipCacheBypass = false; +thread_local bool clipNeededRepair = false; // set when a POLYGON_ clip went down the repair path +thread_local bool clipResultValid = false; // whether the last POLYGON_ clip is known to be valid + +// Shared pieces. Every tile batch that needs a large polygon (often 100,000+ points) used to copy +// it from the store and clip it from scratch, at every zoom. Instead, polygons with at least +// SHARED_PIECE_MIN_POINTS points are clipped once per SHARED_PIECE_ZOOM tile before the tiles are +// written, and tiles below clip from that piece. A piece only depends on the polygon and the tile, +// never on the thread or the order, so the output stays the same for any thread count. +// +// The pieces are known up front and computed by all threads, largest polygon first, before they +// start on tiles: computed on demand, a slow piece (up to ~15 s on the planet) held up every +// thread whose tiles needed it, a third of the tile phase on a 96-thread planet run. A thread +// that runs out of pieces to start goes on to tiles; only a tile that needs a piece still being +// computed waits for it. The map itself doesn't change, so the tiles look pieces up without a lock. +#define SHARED_PIECE_MIN_POINTS 10000 + +void TileDataSource::precomputeSharedPieces(const std::function &present, unsigned threadNum) { + const unsigned zoom = SHARED_PIECE_ZOOM; + if (zoom > indexZoom) return; + const unsigned shift = indexZoom - zoom; + // Candidates: polygons in the large-object index (those spanning 16+ index tiles), with the + // index tiles they cover + std::map boxes; + auto add = [&](const Box &box, const OutputObject &oo) { + if (oo.geomType != POLYGON_ || !storesMultiPolygon(oo.objectID)) return; + auto it = boxes.find(oo.objectID); + if (it == boxes.end()) boxes.emplace(oo.objectID, box); + else boost::geometry::expand(it->second, box); + }; + for (auto const &r : boxRtree) add(r.first, r.second); + for (auto const &r : boxRtreeWithIds) add(r.first, r.second.oo); + if (boxes.empty()) return; + const std::vector> candidates(boxes.begin(), boxes.end()); + + // The pieces each candidate needs: tiles below which a tile is written, that its envelope reaches + using Job = PieceJob; + std::vector jobs; + std::mutex jobsMutex; + std::atomic next(0); + auto findJobs = [&]() { + std::vector local; + for (size_t i = next++; i < candidates.size(); i = next++) { + const NodeID objectID = candidates[i].first; + uint64_t points = 0; + Box envelope; + boost::geometry::assign_inverse(envelope); + for (auto const &p : retrieveMultiPolygon(objectID)) { + points += p.outer().size(); + for (auto const &inner : p.inners()) points += inner.size(); + for (auto const &pt : p.outer()) boost::geometry::expand(envelope, pt); + } + if (points < SHARED_PIECE_MIN_POINTS) continue; + const Box &box = candidates[i].second; + const TileCoordinate x0 = (TileCoordinate)box.min_corner().x() >> shift, x1 = (TileCoordinate)box.max_corner().x() >> shift; + const TileCoordinate y0 = (TileCoordinate)box.min_corner().y() >> shift, y1 = (TileCoordinate)box.max_corner().y() >> shift; + for (TileCoordinate x = x0; x <= x1; x++) + for (TileCoordinate y = y0; y <= y1; y++) + if (present(x, y) && boost::geometry::intersects(envelope, TileBbox(TileCoordinates(x, y), zoom, false, false).clippingBox)) + local.push_back(Job { std::make_tuple(objectID, x, y), points, nullptr }); + } + std::lock_guard lock(jobsMutex); + jobs.insert(jobs.end(), local.begin(), local.end()); + }; + { + boost::asio::thread_pool pool(threadNum); + for (unsigned t = 0; t < threadNum; t++) boost::asio::post(pool, findJobs); + pool.join(); + } + std::sort(jobs.begin(), jobs.end(), [](const Job &a, const Job &b) { return a.points != b.points ? a.points > b.points : a.key < b.key; }); + for (Job &job : jobs) { + pieceObjects.insert(std::get<0>(job.key)); + job.promise = std::make_shared>>(); + sharedPieces.emplace(job.key, job.promise->get_future().share()); + } + pieceJobs = std::move(jobs); + nextPieceJob = 0; +} -void TileDataSource::finalize(size_t threadNum) { - uint64_t finalized = 0; - for (const auto& vec : pendingSmallIndexObjects) { - for (const auto& tuple : vec) { - finalized++; - addObjectToSmallIndexUnsafe(std::get<0>(tuple), std::get<1>(tuple), std::get<2>(tuple)); +void TileDataSource::computeSharedPieces() { + const unsigned zoom = SHARED_PIECE_ZOOM; + for (size_t i = nextPieceJob++; i < pieceJobs.size(); i = nextPieceJob++) { + { + const auto &key = pieceJobs[i].key; + auto piece = std::make_shared(); + try { + clipCacheBypass = true; + clipNeededRepair = false; + Geometry g = buildWayGeometry(POLYGON_, std::get<0>(key), TileBbox(TileCoordinates(std::get<1>(key), std::get<2>(key)), zoom, false, false)); + // A piece that needed the repair path can be wrong (once a lake came out empty), and + // every tile below it would inherit that: leave those tiles to clip the whole polygon. + if (!clipNeededRepair) { + piece->geom = std::make_shared(std::move(boost::get(g))); + // buildWayGeometry has just checked it (for a huge piece, is_valid takes seconds) + piece->valid = clipResultValid; + } + } catch (std::exception &) { + // The tiles clip the whole polygon, and report the problem there + } + clipCacheBypass = false; + pieceJobs[i].promise->set_value(piece); + pieceJobs[i].promise.reset(); // the map's future keeps the piece, until its z6 tile is released } } +} - // Nothing adds to the small index after finalize (shapefiles, GeoJSON, - // Lua and declutter all run before), so release the pending storage. - // TLS pointers of other (finished) threads now dangle; they are never used again. - std::deque>>().swap(pendingSmallIndexObjects); - tlsPendingSmallIndexObjects = nullptr; +uint64_t TileDataSource::releaseSharedPieces(TileCoordinate z6x, TileCoordinate z6y) { + const unsigned shift = SHARED_PIECE_ZOOM - CLUSTER_ZOOM; + uint64_t bytes = 0; + for (auto &e : sharedPieces) { + if ((std::get<1>(e.first) >> shift) != z6x || (std::get<2>(e.first) >> shift) != z6y || !e.second.valid()) continue; + // (a piece no tile needed may still be being computed) + if (e.second.wait_for(std::chrono::seconds(0)) == std::future_status::ready && e.second.get() && e.second.get()->geom) { + const std::shared_ptr &piece = e.second.get(); + for (const auto &polygon : *piece->geom) { + bytes += polygon.outer().size() * sizeof(Point); + for (const auto &inner : polygon.inners()) bytes += inner.size() * sizeof(Point); + } + } + e.second = std::shared_future>(); + } + return bytes; +} - std::cout << "indexed " << finalized << " contended objects" << std::endl; +void TileDataSource::finalize(size_t threadNum) { + // The objects the threads have staged last (at most INDEX_STAGING_SIZE + one object's tiles per + // thread). The order within a shard doesn't matter: shards are sorted below. + // Nothing adds to the small index after finalize (shapefiles, GeoJSON, Lua and declutter all + // run before). Renewing the epoch makes threads' cached staging pointers stale. + uint64_t finalized = 0; + for (IndexStagings& staging : stagings) { + finalized += staging.objects.entries.size() + staging.objectsWithIds.entries.size(); + flushStaging(staging.objects, objects); + flushStaging(staging.objectsWithIds, objectsWithIds); + } + std::deque().swap(stagings); + stagingEpoch = nextStagingEpoch++; + + std::cout << "indexed " << finalized << " staged objects" << std::endl; finalizeObjects(name(), threadNum, indexZoom, objects.begin(), objects.end(), lowZoomObjects); finalizeObjects(name(), threadNum, indexZoom, objectsWithIds.begin(), objectsWithIds.end(), lowZoomObjectsWithIds); @@ -121,22 +294,112 @@ void TileDataSource::addObjectToSmallIndex(const TileCoordinates& index, const O } const size_t z6index = z6x * CLUSTER_ZOOM_WIDTH + z6y; - auto& mutex = objectsMutex[z6index % objectsMutex.size()]; + IndexStagings& staging = indexStaging(); + stageObject(staging, z6index, index, oo, id); + flushStagingIfFull(staging); +} + +TileDataSource::IndexStagings& TileDataSource::indexStaging() { + if (tlsStagingEpoch != stagingEpoch) { + std::lock_guard lock(stagingMutex); + stagings.emplace_back(); + if (!includeID) stagings.back().objects.entries.reserve(INDEX_STAGING_SIZE + 64); + tlsStaging = &stagings.back(); + tlsStagingEpoch = stagingEpoch; + } + return *static_cast(tlsStaging); +} +void TileDataSource::stageObject(IndexStagings& staging, size_t z6index, const TileCoordinates& index, const OutputObject& oo, uint64_t id) { + const size_t z6x = z6index / CLUSTER_ZOOM_WIDTH, z6y = z6index % CLUSTER_ZOOM_WIDTH; + const Z6Offset x = index.x - (z6x * z6OffsetDivisor), y = index.y - (z6y * z6OffsetDivisor); + if (id == 0 || !includeID) { + staging.objects.entries.push_back({ oo, x, y }); + staging.objects.shards.push_back(z6index); + } else { + staging.objectsWithIds.entries.push_back({ oo, x, y, id }); + staging.objectsWithIds.shards.push_back(z6index); + } +} + +void TileDataSource::flushStagingIfFull(IndexStagings& staging) { + if (staging.objects.entries.size() >= INDEX_STAGING_SIZE) flushStaging(staging.objects, objects); + if (staging.objectsWithIds.entries.size() >= INDEX_STAGING_SIZE) flushStaging(staging.objectsWithIds, objectsWithIds); +} - if (mutex.try_lock()) { - addObjectToSmallIndexUnsafe(index, oo, id); - mutex.unlock(); +// Moves the staged entries to their shards: grouped by shard (a counting sort), then one lock and +// one bulk append per shard, instead of a lock per object (and z6 run). Shards whose lock is busy +// are done after the others, then waiting for the lock: its holder only appends. +template +void TileDataSource::flushStaging(IndexStaging& staging, std::vector>& shards) { + const size_t n = staging.entries.size(); + if (n == 0) return; + const uint16_t* z6 = staging.shards.data(); + size_t same = 1; + while (same < n && z6[same] == z6[0]) same++; + if (same == n) { + std::lock_guard lock(objectsMutex[z6[0] % objectsMutex.size()]); + shards[z6[0]].append(staging.entries.data(), n); } else { - // add to tlsPendingSmallIndexObjects - if (tlsPendingSmallIndexObjects == nullptr) { - std::lock_guard lock(objectsMutex[0]); - pendingSmallIndexObjects.push_back(std::vector>()); - tlsPendingSmallIndexObjects = &pendingSmallIndexObjects.back(); + static_assert(std::is_trivially_copyable::value && sizeof(OO) % sizeof(uint64_t) == 0, "staged as raw words"); + thread_local std::vector counts(CLUSTER_ZOOM_AREA, 0); + thread_local std::vector touched; + thread_local std::vector begins; + thread_local std::vector busy; + thread_local std::vector sortedWords; // OO isn't default constructible + touched.clear(); busy.clear(); + for (size_t k = 0; k < n; k++) + if (counts[z6[k]]++ == 0) touched.push_back(z6[k]); + begins.resize(touched.size() + 1); + uint32_t offset = 0; + for (size_t t = 0; t < touched.size(); t++) { + begins[t] = offset; + const uint32_t c = counts[touched[t]]; + counts[touched[t]] = offset; + offset += c; } + begins[touched.size()] = offset; + sortedWords.resize(n * sizeof(OO) / sizeof(uint64_t)); + OO* sorted = reinterpret_cast(sortedWords.data()); + for (size_t k = 0; k < n; k++) + std::memcpy(static_cast(&sorted[counts[z6[k]]++]), &staging.entries[k], sizeof(OO)); + for (size_t t = 0; t < touched.size(); t++) { + counts[touched[t]] = 0; + std::unique_lock lock(objectsMutex[touched[t] % objectsMutex.size()], std::try_to_lock); + if (lock.owns_lock()) shards[touched[t]].append(sorted + begins[t], begins[t + 1] - begins[t]); + else busy.push_back(t); + } + for (size_t t : busy) { + std::lock_guard lock(objectsMutex[touched[t] % objectsMutex.size()]); + shards[touched[t]].append(sorted + begins[t], begins[t + 1] - begins[t]); + } + if (sortedWords.capacity() > 16 * INDEX_STAGING_SIZE * sizeof(OO) / sizeof(uint64_t)) std::vector().swap(sortedWords); + } + staging.entries.clear(); + staging.shards.clear(); + // one huge object (a long line) can leave a big buffer + if (staging.entries.capacity() > 16 * INDEX_STAGING_SIZE) { + std::vector().swap(staging.entries); + std::vector().swap(staging.shards); + staging.entries.reserve(INDEX_STAGING_SIZE + 64); + } +} - tlsPendingSmallIndexObjects->push_back(std::make_tuple(index, oo, id)); +template +void TileDataSource::addToSmallIndex(const std::vector& tiles, uint64_t id, Each each) { + IndexStagings& staging = indexStaging(); + for (const TileCoordinates& tile : tiles) { + const size_t z6x = tile.x / z6OffsetDivisor; + const size_t z6y = tile.y / z6OffsetDivisor; + if (z6x >= 64 || z6y >= 64) { + // addObjectToSmallIndex reports and ignores it + each(tile, [&](const OutputObject& oo) { addObjectToSmallIndex(tile, oo, id); }); + continue; + } + const size_t z6index = z6x * CLUSTER_ZOOM_WIDTH + z6y; + each(tile, [&](const OutputObject& oo) { stageObject(staging, z6index, tile, oo, id); }); } + flushStagingIfFull(staging); } void TileDataSource::addObjectToSmallIndexUnsafe(const TileCoordinates& index, const OutputObject& oo, uint64_t id) { @@ -166,6 +429,13 @@ void TileDataSource::collectTilesWithObjectsAtZoom(std::vector(indexZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); } +void TileDataSource::collectTilesWithObjectsInShard(size_t shard, std::vector>& zooms) { + if (shard < objects.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objects[shard], shard, zooms); + if (shard < objectsWithIds.size()) + collectTilesWithObjectsInShardTemplate(indexZoom, objectsWithIds[shard], shard, zooms); +} + void addCoveredTilesToOutput(const uint indexZoom, std::vector>& zooms, const Box& box) { size_t maxZoom = zooms.size() - 1; // std::cout << "addCoveredTilesToOutput maxZoom=" << maxZoom << ", indexZoom - maxZoom = " << (indexZoom - maxZoom) << std::endl; @@ -174,16 +444,11 @@ void addCoveredTilesToOutput(const uint indexZoom, std::vector= 0; zoom--) { - zooms[zoom]->set(zx, zy); - zx /= 2; - zy /= 2; - } - } + // The parents of the tiles in the box are the tiles in the box scaled down + for (int zoom = maxZoom; zoom >= 0; zoom--) { + zooms[zoom]->setBox(minx, maxx, miny, maxy); + minx /= 2; maxx /= 2; + miny /= 2; maxy /= 2; } } @@ -200,7 +465,8 @@ void TileDataSource::collectTilesWithLargeObjectsAtZoom(std::vector& output + std::vector& output, + bool* nonPolygon ) { if (zoom < CLUSTER_ZOOM) { collectLowZoomObjectsForTile(indexZoom, lowZoomObjects, zoom, dstIndex, output); @@ -223,8 +489,50 @@ void TileDataSource::collectObjectsForTile( iEnd = iStart + 1; } - collectObjectsForTileTemplate(indexZoom, objects.begin(), iStart, iEnd, zoom, dstIndex, output); - collectObjectsForTileTemplate(indexZoom, objectsWithIds.begin(), iStart, iEnd, zoom, dstIndex, output); + collectObjectsForTileTemplate(indexZoom, objects.begin(), iStart, iEnd, zoom, dstIndex, output, nonPolygon); + if (nonPolygon && *nonPolygon) return; + collectObjectsForTileTemplate(indexZoom, objectsWithIds.begin(), iStart, iEnd, zoom, dstIndex, output, nonPolygon); +} + +int TileDataSource::objectKindsForTile(uint zoom, TileCoordinates dstIndex) { + // The objects collected are those getObjectsForTile sorts, deduplicates and orders: the same + // kinds, and some object if any + thread_local std::vector collected; + collected.clear(); + bool nonPolygon = false; + collectObjectsForTile(zoom, dstIndex, collected, &nonPolygon); + if (nonPolygon) return 2; + collectLargeObjectsForTile(zoom, dstIndex, collected); + for (const auto& o : collected) + if (o.oo.geomType != POLYGON_) return 2; + const bool any = !collected.empty(); + if (collected.capacity() > (1u << 16)) std::vector().swap(collected); + return any ? 1 : 0; +} + +namespace { +// The large objects of the tiles the thread is working in, from LARGE_OBJECT_CACHE_ZOOM down: the +// tiles of a LARGE_OBJECT_CACHE_ZOOM tile come one after another, depth first. Per zoom, the objects +// that intersect the current tile at that zoom (whatever their minZoom), in the order the rtree query +// for its box returns them. +const unsigned LARGE_OBJECT_CACHE_ZOOM = 12; +struct LargeObjectLevel { + bool valid = false; + TileCoordinates index{0, 0}; + std::vector> objects; + std::vector> objectsWithIds; +}; +struct LargeObjectCache { + const TileDataSource* source = nullptr; + std::vector levels; // by zoom +}; +thread_local std::vector tlsLargeObjects; // one per source + +Box indexBox(TileCoordinates tile, unsigned zoom, unsigned indexZoom) { + const int scale = 1 << (indexZoom - zoom); + return Box(geom::make(tile.x * scale, tile.y * scale), + geom::make((tile.x + 1) * scale - 1, (tile.y + 1) * scale - 1)); +} } // Copy objects from the large index into output @@ -244,13 +552,78 @@ void TileDataSource::collectLargeObjectsForTile( TileCoordinates srcIndex2((dstIndex.x+1)*scale-1, (dstIndex.y+1)*scale-1); Box box = Box(geom::make(srcIndex1.x, srcIndex1.y), geom::make(srcIndex2.x, srcIndex2.y)); + const unsigned upToZoom = objectsUpToZoom ? objectsUpToZoom : zoom; + if (clampedZoom >= LARGE_OBJECT_CACHE_ZOOM && indexZoom >= LARGE_OBJECT_CACHE_ZOOM) { + // The rtree query visits the nodes depth first in a fixed order and returns the values that + // intersect the box in that order. A value intersecting a box lies in nodes that intersect it, + // and those intersect any larger box too. So the query for a tile returns the values of the + // query for an enclosing box that intersect the tile, in the same order: each tile's objects + // are filtered from those of its parent (from one query per LARGE_OBJECT_CACHE_ZOOM tile). + std::vector& caches = tlsLargeObjects; + size_t c = 0; + while (c < caches.size() && caches[c].source != this) c++; + if (c == caches.size()) { caches.emplace_back(); caches.back().source = this; } + std::vector& levels = caches[c].levels; + if (levels.size() < clampedZoom + 1) levels.resize(clampedZoom + 1); + // Bring the levels above this tile up to date + for (unsigned z = LARGE_OBJECT_CACHE_ZOOM; z < clampedZoom; z++) { + const TileCoordinates tile(dstIndex.x >> (clampedZoom - z), dstIndex.y >> (clampedZoom - z)); + LargeObjectLevel& level = levels[z]; + if (level.valid && level.index.x == tile.x && level.index.y == tile.y) continue; + level.valid = true; + level.index = tile; + level.objects.clear(); + level.objectsWithIds.clear(); + const Box levelBox = indexBox(tile, z, indexZoom); + if (z == LARGE_OBJECT_CACHE_ZOOM) { + for (auto const& result: boxRtree | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(levelBox))) + level.objects.push_back(result); + for (auto const& result: boxRtreeWithIds | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(levelBox))) + level.objectsWithIds.push_back(result); + } else { + for (auto const& result: levels[z - 1].objects) + if (boost::geometry::intersects(result.first, levelBox)) level.objects.push_back(result); + for (auto const& result: levels[z - 1].objectsWithIds) + if (boost::geometry::intersects(result.first, levelBox)) level.objectsWithIds.push_back(result); + } + for (unsigned d = z + 1; d < levels.size(); d++) levels[d].valid = false; + } + if (clampedZoom == LARGE_OBJECT_CACHE_ZOOM) { + // The top of the cache: this tile's own objects (its descendants filter them) + LargeObjectLevel& level = levels[clampedZoom]; + if (!level.valid || level.index.x != dstIndex.x || level.index.y != dstIndex.y) { + level.valid = true; + level.index = dstIndex; + level.objects.clear(); + level.objectsWithIds.clear(); + for (auto const& result: boxRtree | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(box))) + level.objects.push_back(result); + for (auto const& result: boxRtreeWithIds | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(box))) + level.objectsWithIds.push_back(result); + for (unsigned d = clampedZoom + 1; d < levels.size(); d++) levels[d].valid = false; + } + for (auto const& result: level.objects) + if (result.second.minZoom <= upToZoom) output.push_back({result.second, 0}); + for (auto const& result: level.objectsWithIds) + if (result.second.oo.minZoom <= upToZoom) output.push_back({result.second.oo, result.second.id}); + return; + } + const LargeObjectLevel& parent = levels[clampedZoom - 1]; + for (auto const& result: parent.objects) + if (result.second.minZoom <= upToZoom && boost::geometry::intersects(result.first, box)) + output.push_back({result.second, 0}); + for (auto const& result: parent.objectsWithIds) + if (result.second.oo.minZoom <= upToZoom && boost::geometry::intersects(result.first, box)) + output.push_back({result.second.oo, result.second.id}); + return; + } for(auto const& result: boxRtree | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(box))) { - if (result.second.minZoom <= zoom) + if (result.second.minZoom <= (objectsUpToZoom ? objectsUpToZoom : zoom)) output.push_back({result.second, 0}); } for(auto const& result: boxRtreeWithIds | boost::geometry::index::adaptors::queried(boost::geometry::index::intersects(box))) { - if (result.second.oo.minZoom <= zoom) + if (result.second.oo.minZoom <= (objectsUpToZoom ? objectsUpToZoom : zoom)) output.push_back({result.second.oo, result.second.id}); } } @@ -264,44 +637,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } case LINESTRING_: { - auto const &ls = retrieveLinestring(objectID); - - MultiLinestring out; - if(ls.empty()) - return out; - - Box extBox = bbox.getExtendBox(); - const double minX = extBox.min_corner().x(), maxX = extBox.max_corner().x(); - const double minY = extBox.min_corner().y(), maxY = extBox.max_corner().y(); - auto pointInsideExtBox = [minX, maxX, minY, maxY](const Point& p) { - return p.x() >= minX && p.x() <= maxX && p.y() >= minY && p.y() <= maxY; - }; - bool needsIntersection = !pointInsideExtBox(ls[0]); - - Linestring current_ls; - geom::append(current_ls, ls[0]); - - for(size_t i = 1; i < ls.size(); ++i) { - boost::geometry::model::segment segment(ls[i-1], ls[i]); - if(!geom::intersects(segment, bbox.clippingBox)) { - if(current_ls.size() > 1) - out.push_back(std::move(current_ls)); - current_ls.clear(); - } - geom::append(current_ls, ls[i]); - if (!needsIntersection) - needsIntersection = !pointInsideExtBox(ls[i]); - } - - if(current_ls.size() > 1) - out.push_back(std::move(current_ls)); - - if (!needsIntersection) - return out; - - MultiLinestring result; - geom::intersection(out, extBox, result); - return result; + return clip_linestring_for_tile(retrieveLinestring(objectID), bbox.clippingBox, bbox.getExtendBox()); } case MULTILINESTRING_: { @@ -329,6 +665,18 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, auto cachedEntry = multiPolygonClipCache.getEntry(bbox.zoom, bbox.index.x, bbox.index.y, objectID); std::shared_ptr cachedClip = cachedEntry.geom; bool inputKnownValid = cachedClip == nullptr ? polygonKnownValid(objectID) : cachedEntry.valid; + if (cachedClip == nullptr && bbox.zoom > SHARED_PIECE_ZOOM && pieceObjects.count(objectID)) { + // A large polygon: clip from its shared piece + const unsigned shift = bbox.zoom - SHARED_PIECE_ZOOM; + auto it = sharedPieces.find(std::make_tuple(objectID, bbox.index.x >> shift, bbox.index.y >> shift)); + if (it != sharedPieces.end() && it->second.valid()) { + const std::shared_ptr &piece = it->second.get(); // waits if still being computed + if (piece && piece->geom) { + cachedClip = piece->geom; + inputKnownValid = piece->valid; + } + } + } MultiPolygon uncached; @@ -339,83 +687,53 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, const auto &input = cachedClip == nullptr ? uncached : *cachedClip; - Box box = bbox.clippingBox; - - if (bbox.endZoom) { - for(auto const &p: input) { - for(auto const &inner: p.inners()) { - for(std::size_t i = 0; i < inner.size() - 1; ++i) - { - Point p1 = inner[i]; - Point p2 = inner[i + 1]; - - if(geom::within(p1, bbox.clippingBox) != geom::within(p2, bbox.clippingBox)) { - box.min_corner() = Point( - std::min(box.min_corner().x(), std::min(p1.x(), p2.x())), - std::min(box.min_corner().y(), std::min(p1.y(), p2.y()))); - box.max_corner() = Point( - std::max(box.max_corner().x(), std::max(p1.x(), p2.x())), - std::max(box.max_corner().y(), std::max(p1.y(), p2.y()))); - } - } - } - - for(std::size_t i = 0; i < p.outer().size() - 1; ++i) { - Point p1 = p.outer()[i]; - Point p2 = p.outer()[i + 1]; - - if(geom::within(p1, bbox.clippingBox) != geom::within(p2, bbox.clippingBox)) { - box.min_corner() = Point( - std::min(box.min_corner().x(), std::min(p1.x(), p2.x())), - std::min(box.min_corner().y(), std::min(p1.y(), p2.y()))); - box.max_corner() = Point( - std::max(box.max_corner().x(), std::max(p1.x(), p2.x())), - std::max(box.max_corner().y(), std::max(p1.y(), p2.y()))); - } - } - } - - Box extBox = bbox.getExtendBox(); - box.min_corner() = Point( - std::max(box.min_corner().x(), extBox.min_corner().x()), - std::max(box.min_corner().y(), extBox.min_corner().y())); - box.max_corner() = Point( - std::min(box.max_corner().x(), extBox.max_corner().x()), - std::min(box.max_corner().y(), extBox.max_corner().y())); - } + // At the end zoom, the box takes in both ends of every segment crossing its border + Box box = bbox.endZoom ? tile_clip_box_end_zoom(input, bbox.clippingBox, bbox.getExtendBox()) : bbox.clippingBox; MultiPolygon mp; - if (cachedClip == nullptr) + // rect_clip below needs the input as it was before fast_clip. For a large valid + // polygon, keep it rather than load it from the store again for every tile. + std::size_t inputPoints = 0; + if (cachedClip == nullptr && inputKnownValid) + for (auto const &p : uncached) inputPoints += geom::num_points(p); + const bool keepUncached = inputPoints > 1000; + bool modified; + if (cachedClip == nullptr && !keepUncached) { mp = std::move(uncached); - else - geom::assign(mp, input); - bool modified = fast_clip(mp, box); + modified = fast_clip(mp, box); + } else + modified = fast_clip_copy(input, box, mp); if (inputKnownValid && !modified) { // fast_clip was the identity on a geometry that already passed // is_valid, so correct() is a no-op and it is still valid. multiPolygonClipCache.add(bbox, objectID, mp, true); + clipResultValid = true; return mp; } geom::correct(mp); geom::validity_failure_type failure = geom::validity_failure_type::no_failure; - bool valid = multiPolygonDefinitelyValid(mp); + bool valid = modified ? multiPolygonDefinitelyValidClipped(mp, box) : multiPolygonDefinitelyValid(mp); if (!valid && inputKnownValid) { // fast_clip leaves zero-width bridges along the box edge where a concave // polygon leaves and re-enters the box. Clip the valid input exactly // instead, which is much cheaper than the is_valid/intersection/repair // path below; rect_clip bails out on anything it can't handle. MultiPolygon original; - if (cachedClip == nullptr) populateMultiPolygon(original, objectID); + if (cachedClip == nullptr && !keepUncached) populateMultiPolygon(original, objectID); MultiPolygon clipped; - if (rect_clip(cachedClip == nullptr ? original : input, box, clipped) && - (multiPolygonDefinitelyValid(clipped) || geom::is_valid(clipped))) { + if (rect_clip(cachedClip == nullptr && !keepUncached ? original : input, box, clipped) && + (multiPolygonDefinitelyValidAfterClip(clipped) || [&] { RingPointIndexScope pointIndex; return geom::is_valid(clipped); }())) { multiPolygonClipCache.add(bbox, objectID, clipped, true); + clipResultValid = true; return clipped; } } - if (!valid) + if (!valid) { + RingPointIndexScope pointIndex; valid = geom::is_valid(mp,failure); + } if (!valid) { + clipNeededRepair = true; if (verbose) { // Build the whole line first and emit it with a single stream write: // tilemaker runs multi-threaded and chained operator<< calls are not @@ -451,6 +769,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, repairMultiPolygon(output, objectID, &outputValid); // Children of this tile can then take the fast paths for valid input multiPolygonClipCache.add(bbox, objectID, output, outputValid); + clipResultValid = outputValid; return output; } else if (!valid) { // occasionally also wrong_topological_dimension, disconnected_interior: @@ -461,6 +780,7 @@ Geometry TileDataSource::buildWayGeometry(OutputGeometryType const geomType, } multiPolygonClipCache.add(bbox, objectID, mp, valid); + clipResultValid = valid; return mp; } @@ -498,35 +818,209 @@ void TileDataSource::reportSize() const { void populateTilesAtZoom( const std::vector& sources, - std::vector>& zooms + std::vector>& zooms, + size_t threadNum ) { if (zooms.size() > 15) throw std::out_of_range("populateTilesAtZoom: expected at most z14 zooms (15), but found " + std::to_string(zooms.size()) + " vectors"); - for(size_t i=0; icollectTilesWithObjectsAtZoom(zooms); - sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); + std::vector precise; + for (const auto& zoom : zooms) + precise.push_back(dynamic_cast(zoom.get())); + const bool allPrecise = std::find(precise.begin(), precise.end(), nullptr) == precise.end(); + + if (threadNum <= 1 || !allPrecise) { + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithObjectsAtZoom(zooms); + } else { + // Objects are kept per z6 tile. Each worker takes z6 tiles, the ones with the most + // objects first (by columns, one column held a tenth of the planet's objects and ran on + // alone), sets bits in bitmaps of its own, and these are merged at the end, all threads + // merging their part of the bitmaps. A z6 tile's z12 tiles (z12ObjectCounts) are only + // counted by the thread that takes it. The sets don't depend on who set which bit. + std::vector> order; // (objects, z6 tile) + for (size_t shard = 0; shard < CLUSTER_ZOOM_AREA; shard++) { + size_t n = 0; + for (const auto source : sources) n += source->shardObjectCount(shard); + if (n > 0) order.emplace_back(n, shard); + } + std::sort(order.begin(), order.end(), [](const std::pair& a, const std::pair& b) { + return a.first != b.first ? a.first > b.first : a.second < b.second; + }); + std::atomic nextShard(0); + std::vector>> local(threadNum); + { + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) { + boost::asio::post(pool, [&, t]() { + auto& mine = local[t]; + for (size_t zoom = 0; zoom < zooms.size(); zoom++) + mine.push_back(std::make_shared(zoom)); + for (size_t k; (k = nextShard++) < order.size(); ) + for (const auto source : sources) + source->collectTilesWithObjectsInShard(order[k].second, mine); + }); + } + pool.join(); + } + // Merge: (zoom, range of summary words) parts, each merging all threads' bitmaps + std::vector> parts; + for (size_t zoom = 0; zoom < zooms.size(); zoom++) { + const size_t words = precise[zoom]->summaryWords(); + const size_t step = std::max(1, words / (threadNum * 4)); + for (size_t from = 0; from < words; from += step) parts.emplace_back(zoom, from, std::min(words, from + step)); + } + std::atomic nextPart(0); + { + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) { + boost::asio::post(pool, [&]() { + for (size_t k; (k = nextPart++) < parts.size(); ) { + const size_t zoom = std::get<0>(parts[k]); + for (const auto& mine : local) + precise[zoom]->mergeRange(static_cast(*mine[zoom]), std::get<1>(parts[k]), std::get<2>(parts[k])); + } + }); + } + pool.join(); + } + // The threads' bitmaps (32 MB each at z14) are freed on all threads too + { + boost::asio::thread_pool pool(threadNum); + for (size_t t = 0; t < threadNum; t++) + boost::asio::post(pool, [&, t]() { local[t].clear(); }); + pool.join(); + } } + + for (size_t i = 0; i < sources.size(); i++) + sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); } void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ); std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ) { std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - sortOutputObjectIDs(sortOrders, data); + sortOutputObjectIDs(sortOrders, attributeStore, data); data.erase(unique(data.begin(), data.end()), data.end()); + orderStoredGeometryTies(data); return data; } +namespace { +struct GeometryTieKey { + double x, y; // first point + uint64_t points; // number of points + uint64_t hash; // of all points, only computed when the above are equal + OutputObjectID o; +}; + +// Adds a ring/linestring to the key: first point and point count, or (withHash) the hash of all points +template +void addToTieKey(GeometryTieKey& key, const R& ring, bool withHash) { + if (withHash) { + key.hash = stableCombine(key.hash, static_cast(ring.size())); + for (const auto& p : ring) { + key.hash = stableCombine(key.hash, static_cast(p.x())); + key.hash = stableCombine(key.hash, static_cast(p.y())); + } + return; + } + if (key.points == 0 && !ring.empty()) { + key.x = ring.front().x(); + key.y = ring.front().y(); + } + key.points += ring.size(); +} + +void geometryTieKey(const TileDataSource& source, GeometryTieKey& key, bool withHash) { + const NodeID objectID = key.o.oo.objectID; + switch (key.o.oo.geomType) { + case POINT_: { + const Point& p = source.retrievePoint(objectID); + addToTieKey(key, std::array{p}, withHash); + break; + } + case LINESTRING_: + addToTieKey(key, source.retrieveLinestring(objectID), withHash); + break; + case MULTILINESTRING_: + for (const auto& ls : source.retrieveMultiLinestring(objectID)) + addToTieKey(key, ls, withHash); + break; + case POLYGON_: + for (const auto& poly : source.retrieveMultiPolygon(objectID)) { + addToTieKey(key, poly.outer(), withHash); + for (const auto& inner : poly.inners()) + addToTieKey(key, inner, withHash); + } + break; + } +} +} + +// Geometries stored at runtime (relations, shapefiles, ...) have an objectID that +// depends on which thread stored them, so the sort ties objects that share layer, +// z_order, type, attributes and id in an arbitrary order. Order such runs by +// geometry instead. OSM ways and nodes carry their OSM id in the objectID, and +// sort after stored geometries, so they never need this. +void TileDataSource::orderStoredGeometryTies(std::vector& data) const { + const NodeID storedLimit = 1ull << TILE_DATA_ID_SIZE; + auto tied = [](const OutputObjectID& a, const OutputObjectID& b) { + return a.oo.layer == b.oo.layer && a.oo.z_order == b.oo.z_order && a.oo.geomType == b.oo.geomType && + a.oo.attributes == b.oo.attributes && a.id == b.id; + }; + + std::vector keys; + size_t i = 0; + while (i < data.size()) { + size_t j = i + 1; + if (data[i].oo.objectID < storedLimit) + while (j < data.size() && data[j].oo.objectID < storedLimit && tied(data[i], data[j])) j++; + if (j - i < 2) { i = j; continue; } + + keys.clear(); + for (size_t k = i; k < j; k++) { + keys.push_back(GeometryTieKey{0, 0, 0, 0, data[k]}); + geometryTieKey(*this, keys.back(), false); + } + auto byPoint = [](const GeometryTieKey& a, const GeometryTieKey& b) { + if (a.x != b.x) return a.x < b.x; + if (a.y != b.y) return a.y < b.y; + return a.points < b.points; + }; + std::sort(keys.begin(), keys.end(), byPoint); + + // Still tied: hash all points (identical geometries may stay in any order) + for (size_t a = 0; a < keys.size(); ) { + size_t b = a + 1; + while (b < keys.size() && !byPoint(keys[a], keys[b])) b++; + if (b - a > 1) { + for (size_t k = a; k < b; k++) geometryTieKey(*this, keys[k], true); + std::sort(keys.begin() + a, keys.begin() + b, [](const GeometryTieKey& p, const GeometryTieKey& q) { + if (p.hash != q.hash) return p.hash < q.hash; + return p.o.oo.objectID < q.o.oo.objectID; + }); + } + a = b; + } + + for (size_t k = i; k < j; k++) data[k] = keys[k - i].o; + i = j; + } +} + // ------------------------------------ // Add geometries to tile/large indices @@ -549,21 +1043,24 @@ void TileDataSource::addGeometryToIndex( try { insertIntermediateTiles(geom, indexZoom, tiles); - bool polygonExists = false; + bool polygonExists = false, otherExists = false; + for (const auto& output : outputs) { + if (output.geomType == POLYGON_) polygonExists = true; + else otherExists = true; + } + if (tiles.empty()) polygonExists = false; TileCoordinate minTileX = std::numeric_limits::max(), maxTileX = 0, minTileY = std::numeric_limits::max(), maxTileY = 0; for (const TileCoordinates& index : tiles) { minTileX = std::min(index.x, minTileX); minTileY = std::min(index.y, minTileY); maxTileX = std::max(index.x, maxTileX); maxTileY = std::max(index.y, maxTileY); - for (const auto& output : outputs) { - if (output.geomType == POLYGON_) { - polygonExists = true; - continue; - } - addObjectToSmallIndex(index, output, id); // not a polygon - } } + if (otherExists) + addToSmallIndex(tiles, id, [&](const TileCoordinates&, auto&& add) { + for (const auto& output : outputs) + if (output.geomType != POLYGON_) add(output); // not a polygon + }); // for polygon, fill inner tiles if (polygonExists) { @@ -581,9 +1078,7 @@ void TileDataSource::addGeometryToIndex( if (!tilesetFilled) { fillCoveredTiles(tiles); tilesetFilled = true; } - for (const TileCoordinates& index : tiles) { - addObjectToSmallIndex(index, output, id); - } + addToSmallIndex(tiles, id, [&](const TileCoordinates&, auto&& add) { add(output); }); } } } @@ -602,11 +1097,9 @@ void TileDataSource::addGeometryToIndex( for (const auto& ls : geom) { tiles.clear(); insertIntermediateTiles(ls, indexZoom, tiles); - for (const TileCoordinates& index : tiles) { - for (const auto& output : outputs) { - addObjectToSmallIndex(index, output, id); - } - } + addToSmallIndex(tiles, id, [&](const TileCoordinates&, auto&& add) { + for (const auto& output : outputs) add(output); + }); } releaseIfHuge(tiles); } @@ -638,20 +1131,20 @@ void TileDataSource::addGeometryToIndex( } const size_t tileSetSize = tiles.size(); - for (auto& output : outputs) { - if (tileSetSize >= 16) { + if (tileSetSize >= 16) { + for (auto& output : outputs) { // Larger objects - add to rtree // note that the bbox is currently the envelope of the entire multipolygon, // which is suboptimal in shapes like (_) ...... (_) where the outers are significantly disjoint Box box = Box(geom::make(minTileX, minTileY), geom::make(maxTileX, maxTileY)); addObjectToLargeIndex(box, output, id); - } else { - // Smaller objects - add to each individual tile index - for (const TileCoordinates& index : tiles) { - addObjectToSmallIndex(index, output, id); - } } + } else { + // Smaller objects - add to each individual tile index + addToSmallIndex(tiles, id, [&](const TileCoordinates&, auto&& add) { + for (const auto& output : outputs) add(output); + }); } releaseIfHuge(tiles); } @@ -660,7 +1153,7 @@ NodeID TileDataSource::storePoint(const Point& input) { const auto& store = pointStore.get(this); NodeID offset = store.second->size(); - store.second->emplace_back(input); + store.second->push_back(input); NodeID rv = (store.first << (TILE_DATA_ID_SIZE - shardBits)) + offset; return rv; } @@ -681,14 +1174,12 @@ NodeID TileDataSource::storeMultiPolygon(const MultiPolygon& src, bool knownVali multi_polygon_t dst; dst.resize(src.size()); for(std::size_t i = 0; i < src.size(); ++i) { - dst[i].outer().resize(src[i].outer().size()); - boost::geometry::assign(dst[i].outer(), src[i].outer()); + // (the same points: a range copy, one allocation per ring) + dst[i].outer().assign(src[i].outer().begin(), src[i].outer().end()); dst[i].inners().resize(src[i].inners().size()); - for(std::size_t j = 0; j < src[i].inners().size(); ++j) { - dst[i].inners()[j].resize(src[i].inners()[j].size()); - boost::geometry::assign(dst[i].inners()[j], src[i].inners()[j]); - } + for(std::size_t j = 0; j < src[i].inners().size(); ++j) + dst[i].inners()[j].assign(src[i].inners()[j].begin(), src[i].inners()[j].end()); } NodeID offset = store.second->size(); @@ -699,6 +1190,22 @@ NodeID TileDataSource::storeMultiPolygon(const MultiPolygon& src, bool knownVali return rv; } +void TileDataSource::checkValidityLater(NodeID objectID, MultiPolygon mp) { + auto valid = std::async(std::launch::async, [mp = std::move(mp)]() { RingPointIndexScope pointIndex; return geom::is_valid(mp); }); + std::lock_guard lock(deferredValidityMutex); + deferredValidity.emplace_back(objectID, std::move(valid)); +} + +void TileDataSource::finishDeferredValidity() { + for (auto& entry : deferredValidity) { + const bool valid = entry.second.get(); + auto& flags = multipolygonValid[getShard(entry.first)]; + const auto offset = getId(entry.first); + if (offset < flags.size()) flags[offset] = valid; + } + deferredValidity.clear(); +} + NodeID TileDataSource::storeMultiLinestring(const MultiLinestring& src) { const auto& store = multilinestringStore.get(this); diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index 64588163..da78d6db 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -88,6 +90,41 @@ static void checkInterleaveKey() { } #endif +// The counting sort's buffer, kept by each thread for its next sort (see releaseSortBuffers) +static thread_local std::unique_ptr sortBuffer; +static thread_local size_t sortBufferWords = 0; +static thread_local std::vector sortOffsets; + +void releaseSortBuffers() { + sortBuffer.reset(); + sortBufferWords = 0; + std::vector().swap(sortOffsets); +} + +namespace { +// Threads 0..n-1 wait in arrive() until all n have arrived +class SortBarrier { +public: + explicit SortBarrier(size_t n) : n(n) { } + void arrive() { + std::unique_lock lock(m); + const size_t gen = generation; + if (++waiting == n) { + waiting = 0; + generation++; + cv.notify_all(); + } else { + cv.wait(lock, [&]() { return generation != gen; }); + } + } +private: + const size_t n; + size_t waiting = 0, generation = 0; + std::mutex m; + std::condition_variable cv; +}; +} + // The interleaved key has 2 * Z6OffsetBits bits; it is sorted by a single // counting-sort pass over all possible key values. static const size_t keyValues = size_t(1) << (2 * Z6OffsetBits); @@ -98,9 +135,15 @@ template void sortOutputObjects( const unsigned int indexZoom, const size_t threadNum, typename AppendVectorNS::AppendVector::Iterator begin, - typename AppendVectorNS::AppendVector::Iterator end + typename AppendVectorNS::AppendVector::Iterator end, + std::vector* lowZoom ) { + auto collectLowZoom = [&]() { + if (lowZoom) + for (auto it = begin; it != end; ++it) + if (it->oo.minZoom < CLUSTER_ZOOM) lowZoom->push_back(*it); + }; // If the user is doing a a small extract, there are few populated // entries in `object`. // @@ -124,8 +167,10 @@ template void sortOutputObjects( // contiguous range of entries for any tile at zoom 6 or higher. // i.e. compare x then y at each zoom from CLUSTER_ZOOM to indexZoom, // which is the same as comparing the bit-interleaved (x, y) key. - if (indexZoom < CLUSTER_ZOOM) + if (indexZoom < CLUSTER_ZOOM) { + collectLowZoom(); return; // the zoom-loop comparator treats all objects as equal + } // Levels with a shift of Z6OffsetBits or more compare 0 with 0. const unsigned B = std::min(indexZoom - CLUSTER_ZOOM, Z6OffsetBits); @@ -136,6 +181,7 @@ template void sortOutputObjects( const size_t n = end - begin; if (n < countingSortMinSize) { + collectLowZoom(); boost::sort::pdqsort(begin, end, [&key](const OO& a, const OO& b) { return key(a) < key(b); }); return; } @@ -149,26 +195,47 @@ template void sortOutputObjects( // OO has no default constructor, so use raw storage for the copies. static_assert(std::is_trivially_copyable::value && std::is_trivially_destructible::value, "the counting sort buffer copies objects as raw storage"); - std::unique_ptr::type[]> storage( - new typename std::aligned_storage::type[n]); - OO* buffer = reinterpret_cast(storage.get()); - std::vector offsets(chunks * keyValues, 0); + static_assert(sizeof(OO) % sizeof(uint64_t) == 0 && alignof(OO) <= alignof(uint64_t), "the buffer is kept as words"); + const size_t words = n * (sizeof(OO) / sizeof(uint64_t)); + if (sortBufferWords < words) { + sortBuffer.reset(); + sortBuffer.reset(new uint64_t[words]); + sortBufferWords = words; + } + OO* buffer = reinterpret_cast(sortBuffer.get()); + std::vector& offsets = sortOffsets; + offsets.assign(chunks * keyValues, 0); - auto runChunks = [&](const std::function& fn) { - boost::asio::thread_pool pool(chunks); - for (size_t c = 0; c < chunks; c++) { - const size_t from = c * chunkSize, to = std::min(n, from + chunkSize); - boost::asio::post(pool, [&fn, c, from, to]() { fn(c, from, to); }); - } - pool.join(); - }; + // The three steps below, each on all chunks; the chunks' threads are started once + std::vector> steps; + auto runChunks = [&](const std::function& fn) { steps.push_back(fn); }; + std::vector> lowZoomChunks(lowZoom ? chunks : 0); runChunks([&](size_t c, size_t from, size_t to) { size_t* counts = &offsets[c * keyValues]; auto it = begin + from; - for (size_t i = from; i < to; i++, ++it) - counts[key(*it)]++; + if (lowZoom) { + std::vector& low = lowZoomChunks[c]; + for (size_t i = from; i < to; i++, ++it) { + counts[key(*it)]++; + if (it->oo.minZoom < CLUSTER_ZOOM) low.push_back(*it); + } + } else { + for (size_t i = from; i < to; i++, ++it) + counts[key(*it)]++; + } }); + // Between the first two steps, on one thread + auto between = [&]() { + if (lowZoom) { + size_t lowTotal = lowZoom->size(); + for (const auto& low : lowZoomChunks) lowTotal += low.size(); + lowZoom->reserve(lowTotal); + for (auto& low : lowZoomChunks) { + lowZoom->insert(lowZoom->end(), low.begin(), low.end()); + std::vector().swap(low); + } + } // Turn counts into starting positions: by key, then by chunk. size_t pos = 0; @@ -179,6 +246,7 @@ template void sortOutputObjects( slot = pos; pos += count; } + }; runChunks([&](size_t c, size_t from, size_t to) { size_t* next = &offsets[c * keyValues]; @@ -192,51 +260,276 @@ template void sortOutputObjects( for (size_t i = from; i < to; i++, ++it) *it = buffer[i]; }); + + // Run the steps: chunk c on thread c (this thread is chunk 0), all chunks of a step done + // before the next step starts + auto runChunk = [&](size_t c, SortBarrier* barrier) { + const size_t from = std::min(n, c * chunkSize), to = std::min(n, from + chunkSize); + for (size_t step = 0; step < steps.size(); step++) { + steps[step](c, from, to); + if (barrier) barrier->arrive(); + if (step == 0) { + if (c == 0) between(); + if (barrier) barrier->arrive(); + } + } + }; + if (chunks == 1) { + runChunk(0, nullptr); + } else { + SortBarrier barrier(chunks); + std::vector threads; + threads.reserve(chunks - 1); + for (size_t c = 1; c < chunks; c++) threads.emplace_back(runChunk, c, &barrier); + runChunk(0, &barrier); + for (auto& t : threads) t.join(); + } } template void sortOutputObjects( const unsigned int indexZoom, const size_t threadNum, typename AppendVectorNS::AppendVector::Iterator begin, - typename AppendVectorNS::AppendVector::Iterator end + typename AppendVectorNS::AppendVector::Iterator end, + std::vector* lowZoom ); template void sortOutputObjects( const unsigned int indexZoom, const size_t threadNum, typename AppendVectorNS::AppendVector::Iterator begin, - typename AppendVectorNS::AppendVector::Iterator end + typename AppendVectorNS::AppendVector::Iterator end, + std::vector* lowZoom ); -void sortOutputObjectIDs( - const std::vector& sortOrders, +namespace { +struct AttributeKeyCacheEntry { + AttributeIndex index; + uint64_t key; +}; +// Direct-mapped, per thread: neighbouring tiles share most attribute sets +// 64K entries (1 MB): the sets of neighbouring tiles stay cached +const size_t ATTRIBUTE_KEY_CACHE_SIZE = size_t(1) << 16; +thread_local std::vector tlsAttributeKeys; +thread_local const AttributeStore* tlsAttributeKeysStore = nullptr; + +// What the comparator below looks at first, as one 128-bit number: layer (8 bits), z_order +// in the layer's sort direction (16), geomType (2), attribute content hash (64); and the +// position of the object in the input. +struct SortEntry { + uint64_t hi, lo; + uint32_t index; +}; +thread_local std::vector tlsSortEntries; +thread_local std::vector tlsSortedObjects; + +uint64_t attributeKey(const AttributeStore& attributeStore, AttributeIndex index) { + if (tlsAttributeKeysStore != &attributeStore) { + tlsAttributeKeys.assign(ATTRIBUTE_KEY_CACHE_SIZE, AttributeKeyCacheEntry{~0u, 0}); + tlsAttributeKeysStore = &attributeStore; + } + uint32_t slot = index; + slot ^= slot >> 16; slot *= 0x7feb352dU; slot ^= slot >> 15; + AttributeKeyCacheEntry& entry = tlsAttributeKeys[slot & (ATTRIBUTE_KEY_CACHE_SIZE - 1)]; + if (entry.index != index) + entry = AttributeKeyCacheEntry{index, attributeStore.contentHashUnsafe(index)}; + return entry.key; +} +} + +static void sortOutputObjectIDsByHash( + const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ) { - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Lexicographic comparison, with the order of: layer, geomType, attributes, id and objectID. // Note that attributes is preferred to objectID. // It is to arrange objects with the identical attributes continuously. // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; + // + // Attribute sets are compared by content, not by AttributeIndex: the index is + // handed out in insertion order, which depends on thread scheduling. The same + // goes for the objectID of stored geometries, so the OSM id goes first; + // TileDataSource::orderStoredGeometryTies handles the remaining ties. + // + // The comparison is done on a 128-bit key holding layer, z_order (flipped for a descending + // layer), geomType and the attribute content hash, in this order of significance; when the + // keys are equal, the rest of the comparison (content, index, id, objectID) as before. + if (data.size() < 2) return; + std::vector& entries = tlsSortEntries; + entries.resize(data.size()); + for (size_t i = 0; i < data.size(); i++) { + const OutputObject& o = data[i].oo; + // ascending: z_order with the sign bit flipped, so it orders as unsigned; descending: inverted + uint64_t z = uint16_t(o.z_order) ^ 0x8000u; + if (!sortOrders[o.layer]) z = 0xffffu - z; + const uint64_t key = attributeKey(attributeStore, o.attributes); + entries[i].hi = uint64_t(o.layer) << 56 | z << 40 | uint64_t(o.geomType) << 38 | key >> 26; + entries[i].lo = key << 38; + entries[i].index = i; + } + + boost::sort::pdqsort(entries.begin(), entries.end(), [&data, &attributeStore](const SortEntry& a, const SortEntry& b) -> bool { + if (a.hi != b.hi) return a.hi < b.hi; + if (a.lo != b.lo) return a.lo < b.lo; + // same layer, z_order, geomType and attribute hash + const OutputObjectID& p = data[a.index]; + const OutputObjectID& q = data[b.index]; + const OutputObject& x = p.oo; + const OutputObject& y = q.oo; + if (x.attributes != y.attributes) { + const int c = attributeStore.compareContentUnsafe(x.attributes, y.attributes); + if (c != 0) return c < 0; + return x.attributes < y.attributes; + } + if (p.id != q.id) return p.id < q.id; + return x.objectID < y.objectID; }); + + std::vector& sorted = tlsSortedObjects; + sorted.clear(); + sorted.reserve(data.size()); + for (const SortEntry& e : entries) + sorted.push_back(data[e.index]); + std::copy(sorted.begin(), sorted.end(), data.begin()); + if (sorted.capacity() > (1u << 20)) { std::vector().swap(sorted); std::vector().swap(entries); } +} + + +namespace { +// Per call: the attribute sets of the objects being sorted, in the comparator's order (content +// hash, content, index), so each object compares by the position of its set (rank) +struct RankSlot { uint32_t gen, index, rank; uint64_t hash; }; +thread_local std::vector tlsRankSlots; +thread_local uint32_t tlsRankGen = 0; +thread_local std::vector tlsRankSets; // slots of the distinct sets +thread_local std::vector tlsObjectSlot; // per object, its set's slot +// layer, z_order, geomType and the rank of the attribute set (hi), id (lo), objectID << 28 | position (oid) +struct RankEntry { + uint64_t hi, lo, oid; +}; +thread_local std::vector tlsRankEntries; +const unsigned RANK_POS_BITS = 28; +#ifndef RANK_MIN_SIZE +#define RANK_MIN_SIZE 64 +#endif + +inline uint32_t rankSlotHash(uint32_t x) { + x ^= x >> 16; x *= 0x7feb352dU; x ^= x >> 15; x *= 0x846ca68bU; x ^= x >> 16; + return x; +} +} + +void sortOutputObjectIDs( + const std::vector& sortOrders, + const AttributeStore& attributeStore, + std::vector& data +) { + // The order of sortOutputObjectIDsByHash (see there), computed differently: the attribute sets of + // the objects are ranked first, by content hash, then content, then AttributeIndex - the order + // in which that comparator puts objects that agree in layer, z_order and geomType. The objects + // are then sorted on (layer, z_order, geomType, rank), id, objectID: the comparator gives the + // same answer for every pair as the original one (equal rank means the same AttributeIndex), so + // the sort, which only depends on the comparator's answers, puts every object where it did, + // without touching the objects or the attribute store while sorting. + const size_t n = data.size(); + if (n < 2) return; + // Small inputs (most tiles at z13 and z14): comparing hashes directly is cheaper than ranking the + // sets first (same order) + if (n < RANK_MIN_SIZE || n >= (size_t(1) << RANK_POS_BITS)) { sortOutputObjectIDsByHash(sortOrders, attributeStore, data); return; } + + // Distinct sets + std::vector& slots = tlsRankSlots; + if (slots.empty()) slots.assign(64, RankSlot{0, 0, 0, 0}); + if (++tlsRankGen == 0) { for (RankSlot& s : slots) s.gen = 0; tlsRankGen = 1; } + uint32_t gen = tlsRankGen; + std::vector& sets = tlsRankSets; + sets.clear(); + std::vector& objectSlot = tlsObjectSlot; + objectSlot.resize(n); + size_t mask = slots.size() - 1; + for (size_t i = 0; i < n; i++) { + const uint32_t index = data[i].oo.attributes; + size_t s = rankSlotHash(index) & mask; + while (slots[s].gen == gen && slots[s].index != index) s = (s + 1) & mask; + if (slots[s].gen != gen) { + if ((sets.size() + 1) * 2 > slots.size()) { + // grow: rehash the distinct sets found so far, and the objects' slots + std::vector old; + old.swap(slots); + slots.assign(old.size() * 2, RankSlot{0, 0, 0, 0}); + if (++tlsRankGen == 0) tlsRankGen = 1; + gen = tlsRankGen; + mask = slots.size() - 1; + std::vector moved(old.size()); + for (uint32_t& k : sets) { + size_t t = rankSlotHash(old[k].index) & mask; + while (slots[t].gen == gen) t = (t + 1) & mask; + slots[t] = old[k]; + slots[t].gen = gen; + moved[k] = t; + k = t; + } + for (size_t j = 0; j < i; j++) objectSlot[j] = moved[objectSlot[j]]; + s = rankSlotHash(index) & mask; + while (slots[s].gen == gen) s = (s + 1) & mask; + } + slots[s] = RankSlot{gen, index, 0, attributeKey(attributeStore, index)}; + sets.push_back(s); + } + objectSlot[i] = s; + } + + // Rank them: hash, then content, then index (as the comparator below) + if (sets.size() > 1) + std::sort(sets.begin(), sets.end(), [&slots, &attributeStore](uint32_t a, uint32_t b) { + const RankSlot& x = slots[a]; + const RankSlot& y = slots[b]; + if (x.hash != y.hash) return x.hash < y.hash; + const int c = attributeStore.compareContentUnsafe(x.index, y.index); + if (c != 0) return c < 0; + return x.index < y.index; + }); + for (uint32_t r = 0; r < sets.size(); r++) slots[sets[r]].rank = r; + + std::vector& entries = tlsRankEntries; + entries.resize(n); + for (size_t i = 0; i < n; i++) { + const OutputObject& o = data[i].oo; + // ascending: z_order with the sign bit flipped, so it orders as unsigned; descending: inverted + uint64_t z = uint16_t(o.z_order) ^ 0x8000u; + if (!sortOrders[o.layer]) z = 0xffffu - z; + entries[i].hi = uint64_t(o.layer) << 56 | z << 40 | uint64_t(o.geomType) << 38 | slots[objectSlot[i]].rank; + entries[i].lo = data[i].id; + entries[i].oid = uint64_t(o.objectID) << RANK_POS_BITS | i; + } + + boost::sort::pdqsort(entries.begin(), entries.end(), [](const RankEntry& a, const RankEntry& b) -> bool { + if (a.hi != b.hi) return a.hi < b.hi; + if (a.lo != b.lo) return a.lo < b.lo; + return (a.oid >> RANK_POS_BITS) < (b.oid >> RANK_POS_BITS); + }); + + std::vector& sorted = tlsSortedObjects; + sorted.clear(); + sorted.reserve(n); + const uint64_t posMask = (uint64_t(1) << RANK_POS_BITS) - 1; + for (const RankEntry& e : entries) + sorted.push_back(data[e.oid & posMask]); + std::copy(sorted.begin(), sorted.end(), data.begin()); + if (sorted.capacity() > (1u << 20)) { + std::vector().swap(sorted); std::vector().swap(entries); std::vector().swap(objectSlot); + } + if (slots.size() > (1u << 16)) { std::vector().swap(slots); std::vector().swap(sets); } } void sortTileCoordinates( const size_t baseZoom, const size_t threadNum, - std::deque>& tileCoordinates + std::vector>& tileCoordinates ) { - boost::sort::block_indirect_sort( + boost::sort::sample_sort( tileCoordinates.begin(), tileCoordinates.end(), [baseZoom](auto const &a, auto const &b) { const auto aZoom = a.first; @@ -286,5 +579,13 @@ void sortTileCoordinates( return false; }, - threadNum); + // sample_sort scales with the threads (block_indirect_sort slowed down badly beyond 16: 200-300 s + // at 96 threads for the planet's 346M tiles, ~7 s at 16). The keys are unique, so the order + // doesn't depend on the algorithm. + std::max(1, threadNum)); +} + +// The tile list's 64-bit order keys (see tileOrderKey in tilemaker.cpp), on all threads +void sortTileKeys(uint64_t *keys, size_t count, size_t threadNum) { + boost::sort::sample_sort(keys, keys + count, std::less(), std::max(1, threadNum)); } \ No newline at end of file diff --git a/src/tile_worker.cpp b/src/tile_worker.cpp index 054f7b72..6cfe1107 100644 --- a/src/tile_worker.cpp +++ b/src/tile_worker.cpp @@ -7,21 +7,14 @@ #include "helpers.h" #include "visvalingam.h" #include "simplify_buildings.h" +#include "tile_encode.h" #ifdef TILEMAKER_WITH_MLT #include "mlt_writer.h" #endif using namespace std; extern bool verbose; -// The tile is written through one of these backends. Both expose the same builder API, -// so the geometry and attribute code below is shared between the output formats. -struct MvtBackend { - using LayerBuilder = vtzero::layer_builder; - using PointFeature = vtzero::point_feature_builder; - using LinestringFeature = vtzero::linestring_feature_builder; - using PolygonFeature = vtzero::polygon_feature_builder; - static void addRing(PolygonFeature& fbuilder, uint32_t points, bool /*outer*/) { fbuilder.add_ring(points); } -}; +// MvtBackend and the feature encoding: see tile_encode.h #ifdef TILEMAKER_WITH_MLT struct MltBackend { @@ -131,13 +124,6 @@ void writeMultiLinestring( const MultiLinestring& mls, MvtAttributeMemo* attrMemo ) { - typename Backend::LinestringFeature fbuilder{vtLayer}; - - if (sharedData.config.includeID && oo.id) - fbuilder.set_id(oo.id); - - bool hadLine = false; - MultiLinestring tmp; const MultiLinestring* toWrite = nullptr; @@ -155,90 +141,9 @@ void writeMultiLinestring( toWrite = &mls; } - for (const Linestring& ls : *toWrite) { - if (ls.size() <= 1) - continue; - - pair lastXy = std::make_pair(0, 0); - - // vtzero dislikes linesegments that have zero-length segments, - // e.g. where p(x) == p(x + 1). So filter those out. - int points = 0; - for (const Point& p : ls) { - pair xy = bbox.scaleLatpLon(p.get<1>(), p.get<0>()); - if (points == 0 || xy != lastXy) { - points++; - lastXy = xy; - } - } - - // A line has at least 2 points. - if (points <= 1) - continue; - - hadLine = true; - fbuilder.add_linestring(points); - bool firstPoint = true; - for (const Point& p : ls) { - pair xy = bbox.scaleLatpLon(p.get<1>(), p.get<0>()); - if (firstPoint || xy != lastXy) { - // vtzero doesn't like linesegments with zero-length segments, - // so filter those out - fbuilder.set_point(xy.first, xy.second); - firstPoint = false; - lastXy = xy; - } - } - } - - if (hadLine) { - // add the properties - oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); - // call commit() when you are done - fbuilder.commit(); - } + writeLinestringFeature(vtLayer, sharedData.config.includeID, oo, bbox, *toWrite, attributeStore, zoom, attrMemo); } -template -bool writeRing( - typename Backend::PolygonFeature& fbuilder, - const Ring& ring, - bool outer -) { - // vtzero doesn't like zero-length segments in rings - pair lastXy = std::make_pair(0, 0); - int points = 0; - for (const Point& point : ring) { - pair xy = std::make_pair(point.get<0>(), point.get<1>()); - if (points == 0 || xy != lastXy) { - points++; - lastXy = xy; - } - } - - // A ring has at least 4 points. - if (points <= 3) - return false; - - bool firstPoint = true; - Backend::addRing(fbuilder, points, outer); - for (auto it = ring.rbegin(); it != ring.rend(); ++it) { - const Point& point = *it; - - pair xy = - std::make_pair(point.get<0>(), point.get<1>()); - - if (firstPoint || xy != lastXy) { - firstPoint = false; - lastXy = xy; - fbuilder.set_point(xy.first, xy.second); - } - } - - return true; -} - - template void writeMultiPolygon( const AttributeStore& attributeStore, @@ -263,7 +168,7 @@ void writeMultiPolygon( if (simplifyLevel>0) { if (simplifyAlgo == LayerDef::VISVALINGAM) { bool allValid = false; - current = simplifyVis(current, simplifyLevel/bbox.xscale, allValid); + simplifyVisInPlace(current, simplifyLevel/bbox.xscale, allValid); const std::size_t numPoints = boost::geometry::num_points(current); geom::remove_spikes(current); knownPolygonsValid = allValid && !verbose && boost::geometry::num_points(current) == numPoints; @@ -296,7 +201,9 @@ void writeMultiPolygon( const bool mayRepair = (simplifyLevel > 0 || repairPolygons); geom::validity_failure_type failure; - if (!knownPolygonsValid && (mayRepair || verbose) && !geom::is_valid(current, failure)) { + // The precheck (sound: "definitely valid" means Boost finds it valid too) answers most + // polygons without Boost; verbose wants Boost's reason for an invalid one + if (!knownPolygonsValid && (mayRepair || verbose) && (verbose || !multiPolygonDefinitelyValid(current)) && !geom::is_valid(current, failure)) { if (verbose) { cout << "output multipolygon has " << boost_validity_error(failure) << endl; @@ -321,33 +228,7 @@ void writeMultiPolygon( } } - typename Backend::PolygonFeature fbuilder{vtLayer}; - - if (sharedData.config.includeID && oo.id) - fbuilder.set_id(oo.id); - - bool hadPoly = false; - - for (const auto& p : current) { - const Ring& ring = geom::exterior_ring(p); - - // If we failed to write the outer, no need to write the inners. - if (!writeRing(fbuilder, ring, true)) - continue; - - hadPoly = true; - - const InteriorRing& interiors = geom::interior_rings(p); - for (const Ring& ring : interiors) - writeRing(fbuilder, ring, false); - } - - if (hadPoly) { - // add the properties - oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); - // call commit() when you are done - fbuilder.commit(); - } + writePolygonFeature(vtLayer, sharedData.config.includeID, oo, current, attributeStore, zoom, attrMemo); } template @@ -406,19 +287,10 @@ void ProcessObjects( continue; } - typename Backend::PointFeature fbuilder{vtLayer}; - if (sharedData.config.includeID && oo.id) fbuilder.set_id(oo.id); - - fbuilder.add_points(multipoint.size()); - if (verbose && multipoint.size() > 1) std::cout << "Merging " << multipoint.size() << " points into a multipoint" << std::endl; - for (const auto &point : multipoint) - fbuilder.set_point(point.first, point.second); - - oo.oo.writeAttributes(attributeStore, fbuilder, zoom, attrMemo); - fbuilder.commit(); + writePointFeature(vtLayer, sharedData.config.includeID, oo, multipoint, attributeStore, zoom, attrMemo); oo = *jt; } else { @@ -574,7 +446,8 @@ void outputProc( const AttributeStore& attributeStore, const std::vector>& data, TileCoordinates coordinates, - uint zoom + uint zoom, + std::string* rawTile ) { TileBbox bbox(coordinates, zoom, sharedData.config.highResolution && zoom==sharedData.config.endZoom, zoom==sharedData.config.endZoom); if (sharedData.config.clippingBoxFromJSON && ( @@ -611,8 +484,6 @@ void outputProc( } else #endif { - vtzero::tile_builder tile; - // Read existing tile if merging std::string rawExistingTile; if (sharedData.mergeSqlite) { @@ -621,21 +492,19 @@ void outputProc( vtzero::vector_tile existingTile{rawExistingTile}; thread_local MvtAttributeMemo attrMemo; - for (auto lt = sharedData.layers.layerOrder.begin(); lt != sharedData.layers.layerOrder.end(); ++lt) { - if (signalStop) break; - const std::string& layerName = sharedData.layers.layers[lt->at(0)].name; - vtzero::layer_builder vtLayer{tile, layerName, sharedData.config.mvtVersion, extent}; - attrMemo.reset(vtLayer, zoom); - CopyExistingLayer(existingTile, layerName, vtLayer); - ProcessLayer(sources, attributeStore, coordinates, zoom, data, vtLayer, bbox, *lt, sharedData, &attrMemo); - } - tile.serialize(outputdata); + encodeMvtTile(sharedData.layers, zoom, extent, sharedData.config.mvtVersion, data, sharedData.mergeSqlite, signalStop, attrMemo, outputdata, + [&](std::size_t group, vtzero::layer_builder& vtLayer) { + const std::vector& lt = sharedData.layers.layerOrder[group]; + CopyExistingLayer(existingTile, sharedData.layers.layers[lt.at(0)].name, vtLayer); + ProcessLayer(sources, attributeStore, coordinates, zoom, data, vtLayer, bbox, lt, sharedData, &attrMemo); + }); } + if (rawTile) *rawTile = outputdata; // Write to file or sqlite if (sharedData.outputMode == OptionsParser::OutputMode::MBTiles) { // Write to sqlite - if (sharedData.config.compress) { compressed = compress_string(outputdata, Z_DEFAULT_COMPRESSION, sharedData.config.gzip); } + if (sharedData.config.compress) { compressed = compress_string(outputdata, sharedData.config.compressionLevel, sharedData.config.gzip); } sharedData.mbtiles.saveTile(zoom, bbox.index.x, bbox.index.y, sharedData.config.compress ? &compressed : &outputdata, sharedData.mergeSqlite); } else if (sharedData.outputMode == OptionsParser::OutputMode::PMTiles) { @@ -651,7 +520,7 @@ void outputProc( boost::filesystem::create_directories(dirname.str()); fstream outfile(filename.str(), ios::out | ios::trunc | ios::binary); if (sharedData.config.compress) { - outfile << compress_string(outputdata, Z_DEFAULT_COMPRESSION, sharedData.config.gzip); + outfile << compress_string(outputdata, sharedData.config.compressionLevel, sharedData.config.gzip); } else { outfile << outputdata; } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..2362fc20 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -1,3 +1,4 @@ +#include /*! \file */ // C++ includes @@ -29,6 +30,12 @@ #include "rapidjson/filereadstream.h" #include "rapidjson/filewritestream.h" #include "rapidjson/error/en.h" +#if defined(__unix__) || defined(__APPLE__) +#include // sysconf +#endif +#ifdef __linux__ +#include // madvise +#endif #ifndef _MSC_VER #include @@ -83,8 +90,48 @@ bool verbose = false; void sortTileCoordinates( const size_t baseZoom, const size_t threadNum, - std::deque>& tileCoordinates + std::vector>& tileCoordinates ); +void sortTileKeys(uint64_t *keys, size_t count, size_t threadNum); + + +// The order of sortTileCoordinates as a 64-bit key, so that the tile list can be sorted as plain +// numbers. z0..5 first, breadth-first: (zoom, x, y). Then depth-first from z6: the comparator +// compares the z6 x and y, then at each further zoom the x and y of the tiles' ancestors - with +// equal ancestors those differ only in the new bit, so each zoom adds the digit (x bit, y bit). A +// tile whose ancestors all equal the other's comes first (smaller zoom): its missing digits are +// 0, and the zoom is the last field. Needs 2 * (maxZoom - 6) + 12 + 5 + 1 bits. +static inline uint64_t tileOrderKey(unsigned zoom, TileCoordinate x, TileCoordinate y, unsigned maxZoom) { + if (zoom < CLUSTER_ZOOM) + return (uint64_t(zoom) << 40) | (uint64_t(x) << 20) | uint64_t(y); + const unsigned levels = maxZoom - CLUSTER_ZOOM; // digits after z6 + uint64_t key = (uint64_t(x >> (zoom - CLUSTER_ZOOM)) << 6) | uint64_t(y >> (zoom - CLUSTER_ZOOM)); + for (unsigned z = CLUSTER_ZOOM + 1; z <= maxZoom; z++) { + unsigned digit = 0; + if (z <= zoom) digit = (((x >> (zoom - z)) & 1) << 1) | ((y >> (zoom - z)) & 1); + key = (key << 2) | digit; + } + (void)levels; + return (uint64_t(1) << 63) | (key << 5) | zoom; +} + +static inline std::pair tileFromOrderKey(uint64_t key, unsigned maxZoom) { + if (!(key >> 63)) { + const unsigned zoom = (key >> 40) & 0xff; + return { zoom, TileCoordinates((key >> 20) & 0xfffff, key & 0xfffff) }; + } + const unsigned zoom = key & 31; + uint64_t digits = (key >> 5) & ~(uint64_t(1) << 58); + TileCoordinate x = 0, y = 0; + for (unsigned z = maxZoom; z > CLUSTER_ZOOM; z--) { + const unsigned digit = digits & 3; + digits >>= 2; + if (z <= zoom) { x |= TileCoordinate(digit >> 1) << (zoom - z); y |= TileCoordinate(digit & 1) << (zoom - z); } + } + x |= TileCoordinate((digits >> 6) & 63) << (zoom - CLUSTER_ZOOM); + y |= TileCoordinate(digits & 63) << (zoom - CLUSTER_ZOOM); + return { zoom, TileCoordinates(x, y) }; +} /** *\brief The Main function is responsible for command line processing, loading data and starting worker threads. @@ -93,15 +140,40 @@ void sortTileCoordinates( * * Worker threads write the output tiles, and start in the outputProc function. */ +// Memory this process may use: physical RAM, or the cgroup (v2/v1) limit if lower; 0 if unknown +static uint64_t availableMemoryBytes() { + uint64_t bytes = 0; +#if defined(__unix__) || defined(__APPLE__) + const long pages = sysconf(_SC_PHYS_PAGES), pageSize = sysconf(_SC_PAGESIZE); + if (pages > 0 && pageSize > 0) bytes = (uint64_t)pages * (uint64_t)pageSize; +#endif +#ifdef __linux__ + for (const char *path : { "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes" }) { + std::ifstream f(path); + unsigned long long limit = 0; + if (f >> limit && limit > 0 && (bytes == 0 || limit < bytes)) bytes = limit; + } +#endif + return bytes; +} + int main(const int argc, const char* argv[]) { #ifdef __GLIBC__ // glibc raises its mmap threshold dynamically (up to 32/64 MB) once a large // mmapped chunk is freed; multi-MB buffers then land in per-thread arenas - // that are never trimmed, inflating peak RSS. Setting the trim threshold - // explicitly also disables that dynamic adjustment (the mmap threshold - // stays at its 128 KB default), unless the user tunes malloc via the environment. - if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("GLIBC_TUNABLES")) { - mallopt(M_TRIM_THRESHOLD, 4 << 20); + // that are never trimmed, inflating peak RSS. Setting malloc parameters + // explicitly turns that dynamic adjustment off, so both are set here: + // - a 16 MB top pad makes the heap grow and shrink in large steps; with many + // threads allocating, the default 128 KB steps cost many brk() calls and + // page faults + // - a fixed 8 MB mmap threshold: otherwise it would stay at 128 KB, and every + // larger buffer would be a fresh mmap with zeroed pages. Boost's validity + // check on a huge polygon grows such buffers over and over (one relation + // with 1.9M points took 3x as long). + // Users who tune malloc via the environment keep their settings. + if (!getenv("MALLOC_TRIM_THRESHOLD_") && !getenv("MALLOC_MMAP_THRESHOLD_") && !getenv("MALLOC_TOP_PAD_") && !getenv("GLIBC_TUNABLES")) { + mallopt(M_TOP_PAD, 16 << 20); + mallopt(M_MMAP_THRESHOLD, 8 << 20); } #endif @@ -288,9 +360,23 @@ int main(const int argc, const char* argv[]) { OsmLuaProcessing osmLuaProcessing(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, declutter, options.osm.materializeGeometries, true); + // ---- Read significant node/way tags (before the external sources: they use this Lua state) + const SignificantTags significantNodeTags = osmLuaProcessing.GetSignificantNodeKeys(); + const SignificantTags significantWayTags = osmLuaProcessing.GetSignificantWayKeys(); + // ---- Load external sources (shp/geojson) - { + bool anyIndexed = false, anyExternal = false; + for (auto const &layer : layers.layers) { + if (layer.indexed) anyIndexed = true; + if (layer.source.size() > 0) anyExternal = true; + } + if (anyExternal && !hasClippingBox) { + cerr << "Can't read shapefiles unless a bounding box is provided." << endl; + exit(EXIT_FAILURE); + } + int externalResult = 0; + auto readExternal = [&]() { ShpProcessor shpProcessor(clippingBox, options.threadNum, shpMemTiles, osmLuaProcessing); GeoJSONProcessor geoJSONProcessor(clippingBox, options.threadNum, shpMemTiles, osmLuaProcessing); for (size_t layerNum=0; layerNum0) { - if (!hasClippingBox) { - cerr << "Can't read shapefiles unless a bounding box is provided." << endl; - exit(EXIT_FAILURE); - } else { - try { - if (ends_with(layer.source, "json") || ends_with(layer.source, "jsonl") || ends_with(layer.source, "JSON") || ends_with(layer.source, "JSONL") || ends_with(layer.source, "jsonseq") || ends_with(layer.source, "JSONSEQ")) { - cout << "Reading GeoJSON " << layer.name << endl; - geoJSONProcessor.read(layers.layers[layerNum], layerNum); - } else { - cout << "Reading shapefile " << layer.name << endl; - shpProcessor.read(layers.layers[layerNum], layerNum); - } - } catch (const std::exception& e) { - cerr << "Error reading external source " << layer.source << ": " << e.what() << endl; - return -1; - } catch (...) { - cerr << "Unknown error reading external source " << layer.source << endl; - return -1; + try { + if (ends_with(layer.source, "json") || ends_with(layer.source, "jsonl") || ends_with(layer.source, "JSON") || ends_with(layer.source, "JSONL") || ends_with(layer.source, "jsonseq") || ends_with(layer.source, "JSONSEQ")) { + cout << "Reading GeoJSON " << layer.name << endl; + geoJSONProcessor.read(layers.layers[layerNum], layerNum); + } else { + cout << "Reading shapefile " << layer.name << endl; + shpProcessor.read(layers.layers[layerNum], layerNum); } + } catch (const std::exception& e) { + cerr << "Error reading external source " << layer.source << ": " << e.what() << endl; + externalResult = -1; + return; + } catch (...) { + cerr << "Unknown error reading external source " << layer.source << endl; + externalResult = -1; + return; } } } + }; + // Unless a layer is indexed (the .pbf's Lua code may look things up in it), nothing read from + // the .pbf depends on the external sources, so read them alongside the .pbf: the ocean shapefile + // alone took ~11 s, mostly on a few threads, at the start of a planet run. + std::thread externalThread; + struct JoinOnExit { std::thread &t; ~JoinOnExit() { if (t.joinable()) t.join(); } } joinExternal { externalThread }; + if (anyIndexed || !anyExternal) { + readExternal(); + if (externalResult != 0) return externalResult; + shpMemTiles.reportSize(); + } else { + externalThread = std::thread(readExternal); } - shpMemTiles.reportSize(); - - // ---- Read significant node/way tags - const SignificantTags significantNodeTags = osmLuaProcessing.GetSignificantNodeKeys(); - const SignificantTags significantWayTags = osmLuaProcessing.GetSignificantWayKeys(); // ---- Read all PBFs @@ -363,6 +453,11 @@ int main(const int argc, const char* argv[]) { ); if (ret != 0) return ret; } + if (externalThread.joinable()) { + externalThread.join(); + if (externalResult != 0) return externalResult; + shpMemTiles.reportSize(); + } attributeStore.finalize(); osmMemTiles.reportSize(); attributeStore.reportSize(); @@ -387,6 +482,7 @@ int main(const int argc, const char* argv[]) { sharedData.writeMBTilesProjectData(); } else if (sharedData.outputMode == OptionsParser::OutputMode::PMTiles) { sharedData.pmtiles.open(sharedData.outputFile); + sharedData.pmtiles.compressionLevel = config.compressionLevel; if (config.tileFormat == TileFormat::MLT) sharedData.pmtiles.tileType = TILETYPE_MLT; } @@ -431,7 +527,10 @@ int main(const int argc, const char* argv[]) { sharedData.pmtiles.isSparse = false; } - std::deque> tileCoordinates; + std::vector> tileCoordinates; + // z(SHARED_PIECE_ZOOM) tiles below which a tile is written (for the shared pieces) + const unsigned pieceZoom = TileDataSource::SHARED_PIECE_ZOOM; + std::vector piecePresent((size_t)1 << (2 * pieceZoom), 0); std::vector> zoomResults; zoomResults.reserve(sharedData.config.endZoom + 1); @@ -446,7 +545,8 @@ int main(const int argc, const char* argv[]) { clock_gettime(CLOCK_MONOTONIC, &start); #endif std::cout << "collecting tiles" << std::flush; - populateTilesAtZoom(sources, zoomResults); + if (options.threadNum > 1 && sharedData.config.endZoom >= 12) z12ObjectCounts.assign((size_t)1 << 24, 0); + populateTilesAtZoom(sources, zoomResults, options.threadNum); #ifdef CLOCK_MONOTONIC clock_gettime(CLOCK_MONOTONIC, &end); uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; @@ -460,6 +560,9 @@ int main(const int argc, const char* argv[]) { } std::cout << ", filtering tiles:" << std::flush; + // Parts of the tile list, in order (zoom by zoom, stripe by stripe): copied into one allocation + // at the end, on all threads + std::vector>> listParts; for (uint zoom=sharedData.config.startZoom; zoom <= sharedData.config.endZoom; zoom++) { std::cout << " z" << std::to_string(zoom) << std::flush; #ifdef CLOCK_MONOTONIC @@ -468,8 +571,7 @@ int main(const int argc, const char* argv[]) { #endif const auto& zoomResult = zoomResults[zoom]; - int numTiles = 0; - zoomResult->forEachSet([&](TileCoordinate x, TileCoordinate y) { + auto keep = [&](TileCoordinate x, TileCoordinate y) { if (hasClippingBox) { bool isInAWhollyCoveredZ6Tile = false; if (zoom >= 6) { @@ -479,12 +581,56 @@ int main(const int argc, const char* argv[]) { } if(!isInAWhollyCoveredZ6Tile && !boost::geometry::intersects(TileBbox(TileCoordinates(x, y), zoom, false, false).getTileBox(), clippingBox)) - return; + return false; } - - tileCoordinates.push_back(std::make_pair(zoom, TileCoordinates(x, y))); - numTiles++; - }); + return true; + }; + auto markPresent = [&](std::vector &present, TileCoordinate x, TileCoordinate y) { + if (zoom > pieceZoom) + present[((size_t)(x >> (zoom - pieceZoom)) << pieceZoom) + (y >> (zoom - pieceZoom))] = 1; + }; + size_t numTiles = 0; + const auto *precise = dynamic_cast(zoomResult.get()); + const TileCoordinate dim = 1u << std::min(zoom, 30u); + if (precise && zoom >= 6 && options.threadNum > 1) { + // Column stripes in parallel (z14 alone has ~260M tiles on the planet); the order + // doesn't matter, the list is sorted below. + const size_t stripes = std::min(dim, options.threadNum * 8); + std::vector>> parts(stripes); + std::vector> presents(options.threadNum); + std::atomic nextStripe(0); + boost::asio::thread_pool pool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&, t]() { + std::vector &present = presents[t]; + present.assign(piecePresent.size(), 0); + for (size_t s = nextStripe++; s < stripes; s = nextStripe++) { + const TileCoordinate x0 = (uint64_t)dim * s / stripes, x1 = (uint64_t)dim * (s + 1) / stripes; + precise->forEachSetInColumns(x0, x1, [&](TileCoordinate x, TileCoordinate y) { + if (!keep(x, y)) return; + parts[s].emplace_back(zoom, TileCoordinates(x, y)); + markPresent(present, x, y); + }); + } + }); + } + pool.join(); + for (auto &part : parts) { + numTiles += part.size(); + listParts.emplace_back(std::move(part)); + } + for (auto const &present : presents) + for (size_t i = 0; i < present.size(); i++) piecePresent[i] |= present[i]; + } else { + listParts.emplace_back(); + auto &part = listParts.back(); + zoomResult->forEachSet([&](TileCoordinate x, TileCoordinate y) { + if (!keep(x, y)) return; + part.push_back(std::make_pair(zoom, TileCoordinates(x, y))); + markPresent(piecePresent, x, y); + numTiles++; + }); + } std::cout << " (" << numTiles; #ifdef CLOCK_MONOTONIC @@ -496,103 +642,676 @@ int main(const int argc, const char* argv[]) { std::cout << ")" << std::flush; } zoomResults.clear(); + const unsigned listMaxZoom = std::max(CLUSTER_ZOOM, sharedData.config.endZoom); + const bool keyedSort = tileCoordinates.empty() && 2 * (listMaxZoom - CLUSTER_ZOOM) + 12 + 5 + 1 <= 64; + if (keyedSort) { + std::vector offsets(listParts.size() + 1, 0); + for (size_t i = 0; i < listParts.size(); i++) offsets[i + 1] = offsets[i] + listParts[i].size(); + const size_t total = offsets.back(); + std::unique_ptr keys(new uint64_t[std::max(1, total)]); // not zeroed: every slot is written + { + std::atomic nextPart(0); + boost::asio::thread_pool keyPool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(keyPool, [&]() { + for (size_t i = nextPart++; i < listParts.size(); i = nextPart++) { + uint64_t *out = keys.get() + offsets[i]; + for (const auto &tile : listParts[i]) *out++ = tileOrderKey(tile.first, tile.second.x, tile.second.y, listMaxZoom); + std::vector>().swap(listParts[i]); + } + }); + } + keyPool.join(); + } + listParts.clear(); + sortTileKeys(keys.get(), total, options.threadNum); + // The sorted list is made only now: alongside the parts, the keys and the sort's own buffer of + // as many keys, it raised the peak by its size (4 GB for a planet's 346M tiles). resize() writes + // every element from one thread, which on fresh memory is mostly page faults (1.8 s for 346M + // tiles): touch it on all threads first. + tileCoordinates.reserve(total); + { + char *storage = reinterpret_cast(tileCoordinates.data()); + const size_t bytes = total * sizeof(tileCoordinates[0]), chunk = 64 << 20; + std::atomic nextChunk(0); + boost::asio::thread_pool touchPool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(touchPool, [&]() { + for (size_t c = nextChunk++; c * chunk < bytes; c = nextChunk++) + std::memset(storage + c * chunk, 0, std::min(chunk, bytes - c * chunk)); + }); + } + touchPool.join(); + } + tileCoordinates.resize(total); + { + const size_t chunk = 1 << 20; + std::atomic nextChunk(0); + boost::asio::thread_pool decodePool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(decodePool, [&]() { + for (size_t c = nextChunk++; c * chunk < total; c = nextChunk++) + for (size_t i = c * chunk; i < std::min(total, (c + 1) * chunk); i++) + tileCoordinates[i] = tileFromOrderKey(keys[i], listMaxZoom); + }); + } + decodePool.join(); + } + } else { + std::vector offsets(listParts.size() + 1, tileCoordinates.size()); + for (size_t i = 0; i < listParts.size(); i++) offsets[i + 1] = offsets[i] + listParts[i].size(); + tileCoordinates.resize(offsets.back()); + std::atomic nextPart(0); + boost::asio::thread_pool copyPool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(copyPool, [&]() { + for (size_t i = nextPart++; i < listParts.size(); i = nextPart++) { + std::copy(listParts[i].begin(), listParts[i].end(), tileCoordinates.begin() + offsets[i]); + std::vector>().swap(listParts[i]); + } + }); + } + copyPool.join(); + listParts.clear(); + } std::cout << std::endl; - // Cluster tiles: breadth-first for z0..z5, depth-first for z6 - sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + // Cluster tiles: breadth-first for z0..z5, depth-first for z6 (already done by the keyed sort) + if (!keyedSort) sortTileCoordinates(config.baseZoom, options.threadNum, tileCoordinates); + + OsmLuaProcessing::printMemoStats(); + + // Validity of big relations, checked while the rest was read (see checkValidityLater) + for (auto source : sources) source->finishDeferredValidity(); + + // Shared pieces of large polygons, for the SHARED_PIECE_ZOOM tiles below which a tile is written + { + auto present = [&](TileCoordinate x, TileCoordinate y) { + return x < (1u << pieceZoom) && y < (1u << pieceZoom) && piecePresent[((size_t)x << pieceZoom) + y]; + }; + for (auto source : sources) source->precomputeSharedPieces(present, options.threadNum); + } std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { - // Compute how many tiles should be assigned to this batch -- - // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. - batchSize = 0; - size_t weight = 0; - // Batches are a fixed partition: each z<=11 tile is its own batch, and - // z12 subtrees are grouped (~2 per batch), so z13/z14 tiles usually share - // a batch with their z12 ancestor (batch-local ClipCache can reuse its clips). - // The partition doesn't depend on thread count, so neither does the output. - while (startIndex + batchSize < tileCoordinates.size()) { - const auto& zoom = tileCoordinates[startIndex + batchSize].first; - if (batchSize > 0 && (weight >= 1000 || (weight >= 50 && zoom <= cutZoom))) - break; - if (zoom > 12) - weight++; - else if (zoom > 11) - weight += 10; - else if (zoom > 10) - weight += 100; - else - weight += 1000; - - batchSize++; + // Batches are handed out as threads become free rather than all posted up + // front: a planet has ~14M of them, and their queued handlers held GBs. + std::mutex batchMutex; + uint64_t nextBatchSeq = 0; + // Shared pieces (TileDataSource::precomputeSharedPieces) lie inside one z6 tile. Tiles from z6 on + // are written z6 tile by z6 tile, so once the hand-out has moved past a z6 tile and none of its + // batches is still running, no tile needs its pieces any more. + std::vector z6Running(CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH, 0); + int64_t z6HandedOut = -1; + auto z6Of = [&](std::size_t i) -> int64_t { + const unsigned z = tileCoordinates[i].first; + if (z < CLUSTER_ZOOM) return -1; + return (int64_t)(tileCoordinates[i].second.x >> (z - CLUSTER_ZOOM)) * CLUSTER_ZOOM_WIDTH + (tileCoordinates[i].second.y >> (z - CLUSTER_ZOOM)); + }; + // The pieces are freed by the threads that computed them, mostly into the middle of their malloc + // arenas, where glibc keeps the pages (the planet's pieces took several GB at the start of the + // tile phase): once enough is freed, give it back, outside batchMutex + std::atomic releasedPieceBytes(0); + auto releaseZ6 = [&](int64_t b) { + for (auto source : sources) releasedPieceBytes += source->releaseSharedPieces(b / CLUSTER_ZOOM_WIDTH, b % CLUSTER_ZOOM_WIDTH); + }; + auto trimReleased = [&]() { +#ifdef __GLIBC__ + if (releasedPieceBytes.load(std::memory_order_relaxed) >= (256u << 20) && releasedPieceBytes.exchange(0) >= (256u << 20)) + malloc_trim(0); +#endif + }; + // Batches are a fixed partition: each z<=11 tile is its own batch, and + // z12 subtrees are grouped (~2 per batch), so z13/z14 tiles usually share + // a batch with their z12 ancestor (batch-local ClipCache can reuse its clips). + // The partition doesn't depend on thread count, so neither does the output. + std::vector batchStarts; // batch i: tiles batchStarts[i] .. batchStarts[i+1] + // Appends the starts of the batches from `start` (a batch start) on, up to `stop`. Returns + // whether `stop` is a batch start. + auto scanBatches = [&](std::size_t start, std::size_t stop, std::vector& starts) { + while (start < stop) { + starts.push_back(start); + // Higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive + std::size_t size = 0, weight = 0; + while (start + size < tileCoordinates.size()) { + const auto& zoom = tileCoordinates[start + size].first; + if (size > 0 && (weight >= 1000 || (weight >= 50 && zoom <= cutZoom))) + break; + if (zoom > 12) + weight++; + else if (zoom > 11) + weight += 10; + else if (zoom > 10) + weight += 100; + else + weight += 1000; + size++; + } + start += size; } + return start == stop; + }; + { + // One pass over the planet's 346M tiles took 0.5 s on one thread. The list is cut into parts + // at tiles expected to start a batch: after a tile of z11 or below (weight >= 100), the next + // tile of z12 or below is one (unless a run of 1000 z13/z14 tiles came between). Each part is + // scanned from its start on its own; if a part doesn't end exactly at the next one's start, + // the expectation was wrong and the whole list is scanned again, from the start. + const std::size_t n = tileCoordinates.size(); + std::vector cuts { 0 }; + const std::size_t numParts = options.threadNum > 1 ? (std::size_t)options.threadNum * 4 : 1; + for (std::size_t k = 1; k < numParts; k++) { + std::size_t i = std::max(cuts.back() + 1, n * k / numParts); + while (i < n && tileCoordinates[i].first > 11) i++; + i++; + while (i < n && tileCoordinates[i].first > cutZoom) i++; + if (i < n) cuts.push_back(i); + } + cuts.push_back(n); + std::vector> partStarts(cuts.size() - 1); + std::atomic allExact(true); + std::atomic nextPart(0); + { + boost::asio::thread_pool partPool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(partPool, [&]() { + for (std::size_t k; (k = nextPart++) < partStarts.size(); ) + if (!scanBatches(cuts[k], cuts[k + 1], partStarts[k])) allExact = false; + }); + } + partPool.join(); + } + if (allExact) { + std::size_t total = 0; + for (const auto& part : partStarts) total += part.size(); + batchStarts.reserve(total + 1); + for (auto& part : partStarts) { + batchStarts.insert(batchStarts.end(), part.begin(), part.end()); + std::vector().swap(part); + } + } else { + scanBatches(0, n, batchStarts); + } + } + batchStarts.push_back(tileCoordinates.size()); + // Handed out first: the tiles up to this zoom. One of them can take half a minute (a fjord + // coast at z6/z7); in its place in the order, the batches after it piled up until all + // threads waited for it. They are written in their place all the same, and are small. + const unsigned prefetchZoom = 8; + // Consecutive batches are handed out and written in groups: on a planet there are ~14M + // batches, and taking and handing over each one (two shared locks) limited the tile phase. + // Each batch of a group is still worked on as its own batch, so the output is the same. + // Prefetched batches stay in groups of their own. groupStarts[g]: first batch of group g. + // With few batches per thread (an extract) they're handed out one by one, as grouping + // would only make the load less even; a planet has groups of 8 (4..8 measured best). + const std::size_t batchesPerThread = (batchStarts.size() - 1) / std::max(1u, options.threadNum); + std::size_t groupSize = std::min(8, std::max(1, batchesPerThread / 16384)); + std::vector groupStarts; + for (std::size_t b = 0; b + 1 < batchStarts.size(); ) { + groupStarts.push_back(b); + const bool prefetch = tileCoordinates[batchStarts[b]].first <= prefetchZoom; + std::size_t n = 1; + while (!prefetch && n < groupSize && b + n + 1 < batchStarts.size() && tileCoordinates[batchStarts[b + n]].first > prefetchZoom) n++; + b += n; + } + groupStarts.push_back(batchStarts.size() - 1); + std::cout << "Tile batches: " << (batchStarts.size() - 1) << " in " << (groupStarts.size() - 1) << " groups of up to " << groupSize << std::endl; + std::vector handOutOrder; // group numbers + handOutOrder.reserve(groupStarts.size() - 1); + for (uint64_t g = 0; g + 1 < groupStarts.size(); g++) + if (tileCoordinates[batchStarts[groupStarts[g]]].first <= prefetchZoom) handOutOrder.push_back(g); + const std::size_t prefetched = handOutOrder.size(); + for (uint64_t g = 0; g + 1 < groupStarts.size(); g++) + if (tileCoordinates[batchStarts[groupStarts[g]]].first > prefetchZoom) handOutOrder.push_back(g); + std::size_t nextHandOut = 0; + + // The groups expected to be slowest (a big city's z12 tiles took up to 7 s, while all the + // batches behind it waited to be written; at the end of the planet the tiles only came out 4 s + // after the rest was done) are handed out ahead of their place, when the hand-out comes within + // slowWindow groups of them. Expected cost: the objects in the group's z12 tiles (a z13 tile a + // quarter of its z12 tile's, a z11 tile its four z12 tiles', ...). + std::vector slowPos; // positions in handOutOrder, ascending + std::vector handedEarly(handOutOrder.size(), 0); + std::size_t nextSlow = 0, slowWindow = 0; + if (!z12ObjectCounts.empty() && handOutOrder.size() > prefetched) { + const double slowFraction = 0.001, windowFraction = 0.05; + const std::size_t n = handOutOrder.size() - prefetched; + std::vector cost(n, 0); + { + std::atomic nextChunk(0); + const std::size_t chunk = 4096; + boost::asio::thread_pool costPool(options.threadNum); + for (unsigned t = 0; t < options.threadNum; t++) { + boost::asio::post(costPool, [&]() { + for (std::size_t c = nextChunk++; c * chunk < n; c = nextChunk++) { + for (std::size_t k = c * chunk; k < std::min(n, (c + 1) * chunk); k++) { + const uint64_t g = handOutOrder[prefetched + k]; + double sum = 0; + for (std::size_t i = batchStarts[groupStarts[g]]; i < batchStarts[groupStarts[g + 1]]; i++) { + const unsigned z = tileCoordinates[i].first; + const TileCoordinates &tc = tileCoordinates[i].second; + if (z >= 12) { + sum += z12ObjectCounts[((size_t)(tc.x >> (z - 12)) << 12) | (tc.y >> (z - 12))] / double(1ull << (2 * (z - 12))); + } else { + const unsigned d = 12 - z; + for (TileCoordinate x = tc.x << d; x < (tc.x + 1) << d; x++) + for (TileCoordinate y = tc.y << d; y < (tc.y + 1) << d; y++) + sum += z12ObjectCounts[((size_t)x << 12) | y]; + } + } + cost[k] = sum; + } + } + }); + } + costPool.join(); + } + const std::size_t slowCount = std::min(n, (std::size_t)(n * slowFraction)); + if (slowCount > 0) { + std::vector sorted(cost); + std::nth_element(sorted.begin(), sorted.begin() + (n - slowCount), sorted.end()); + const double threshold = sorted[n - slowCount]; + for (std::size_t k = 0; k < n && slowPos.size() < slowCount; k++) + if (cost[k] >= threshold && cost[k] > 0) slowPos.push_back(prefetched + k); + slowWindow = std::max(1, (std::size_t)(n * windowFraction)); + std::cout << "Groups handed out early: " << slowPos.size() << " (expected cost >= " << threshold << "), up to " << slowWindow << " groups ahead" << std::endl; + } + std::vector().swap(z12ObjectCounts); + } - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() { - clipCacheBatchId = ++batchCounter; - // A single-tile batch has no descendants to reuse its clips. - clipCacheNoStore = batchSize == 1; - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; + // Work stealing at the end. A group's tiles are worked on in units: a batch is cut before each tile + // of z12 or below (cutZoom), so a unit is a tile below z12, or a z12 tile with its descendants. + // The thread the group was handed out to takes its units from the front; once all groups are + // handed out, threads that ran out of work take units from the back of the groups still being + // worked on (at the end of a planet run, one group of 172 tiles ran on alone for 2 s). + // - A unit's tiles reuse clips only of their ancestors in the same unit (ClipCache), and the + // covered-z12 reuse starts at its z12 tile, so a unit comes out the same on any thread and + // after any other unit; a thread starts a new clip-cache batch at a batch start, and wherever + // it doesn't go on with the unit right after its last one. + // - The writers number the units (batch << 16 | unit within the batch) and write them in that + // order, a run of units worked on one after another as one, so the file stays the same. + struct GroupWork { + std::mutex m; + bool active = false; + uint64_t group = 0; + std::vector unitStart; // tile index; units are followed by the next one's start + std::vector unitKey; // writer number; unitKey.back() is the number after the group + std::vector unitNoStore; // in a batch of a single tile + std::size_t front = 0, back = 0; // units not taken yet: front .. back-1 + }; + std::vector> groupWork; + for (unsigned t = 0; t < options.threadNum; t++) groupWork.emplace_back(new GroupWork()); + std::atomic nextWorker(0); + std::vector groupCountsZ6(groupStarts.size() - 1, 0); + static thread_local std::vector batchZ6; // z6 tiles of the batch this thread is working on + // Groups handed out ahead of their place (prefetched, slow) are held until their turn. When that + // is more than the writer may hold, the threads must not all wait (waitForRoom) while the group + // to be written next isn't handed out: the writer is told the first group in file order not + // handed out yet, and lets one thread go on to take it. + const std::size_t numGroups = groupStarts.size() - 1; + std::vector groupTaken(numGroups, 0); + std::vector handOutPos(numGroups); // position in handOutOrder + for (std::size_t k = 0; k < handOutOrder.size(); k++) handOutPos[handOutOrder[k]] = k; + std::size_t firstUntaken = 0; + // its first unit's writer number (see GroupWork) + std::atomic firstUntakenKey(numGroups > 0 ? (uint64_t)groupStarts[0] << 16 : UINT64_MAX); + // Hands out the next group: returns 0 if none is left, 1 if one is. With `needed` (a writer + // number from waitForRoom), only the group starting with it, the first in file order not handed + // out yet: 2 if it was handed out meanwhile. + auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq, uint64_t needed) -> int { + const std::lock_guard lock(batchMutex); + batchZ6.clear(); + while (nextHandOut < handOutOrder.size() && handedEarly[nextHandOut]) nextHandOut++; + while (nextSlow < slowPos.size() && (slowPos[nextSlow] < nextHandOut || handedEarly[slowPos[nextSlow]])) nextSlow++; + if (nextHandOut >= handOutOrder.size()) return 0; + // A slow group coming up soon goes first; it doesn't move the hand-out on (z6HandedOut) + bool early = false; + std::size_t k; + if (needed != UINT64_MAX) { + if (firstUntaken >= numGroups || ((uint64_t)groupStarts[firstUntaken] << 16) != needed) return 2; + k = handOutPos[firstUntaken]; + if (k == nextHandOut) { + nextHandOut++; + } else { + handedEarly[k] = 1; + early = true; + } + } else if (nextSlow < slowPos.size() && slowPos[nextSlow] > nextHandOut && slowPos[nextSlow] <= nextHandOut + slowWindow) { + k = slowPos[nextSlow++]; + handedEarly[k] = 1; + early = true; + } else { + k = nextHandOut++; + } + seq = handOutOrder[k]; + startIndex = batchStarts[groupStarts[seq]]; + batchSize = batchStarts[groupStarts[seq + 1]] - startIndex; + groupTaken[seq] = 1; + while (firstUntaken < numGroups && groupTaken[firstUntaken]) firstUntaken++; + firstUntakenKey = firstUntaken < numGroups ? (uint64_t)groupStarts[firstUntaken] << 16 : UINT64_MAX; + // Prefetched tiles don't use the shared pieces, and are out of order: only the others count + // for releasing the pieces of z6 tiles the hand-out has moved past + if (k < prefetched) return 1; + groupCountsZ6[seq] = 1; + for (std::size_t i = startIndex; i < startIndex + batchSize; i++) { + const int64_t b = z6Of(i); + if (b >= 0 && (batchZ6.empty() || batchZ6.back() != b)) batchZ6.push_back(b); + } + for (int64_t b : batchZ6) z6Running[b]++; + if (!early && !batchZ6.empty() && batchZ6.back() != z6HandedOut) { + if (z6HandedOut >= 0 && z6Running[z6HandedOut] == 0) releaseZ6(z6HandedOut); + z6HandedOut = batchZ6.back(); + } + return 1; + }; -#ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); + // .pmtiles and .mbtiles are written in batch order, so the file doesn't depend on thread scheduling + const bool toPMTiles = options.outputMode == OptionsParser::OutputMode::PMTiles; + const bool toMBTiles = options.outputMode == OptionsParser::OutputMode::MBTiles; + // While an earlier batch is still being worked on, threads may run ahead by 1/16 of the memory + // (physical RAM, or the cgroup limit if lower), at least 64 MB of finished tiles per thread and at + // most 4 GB. With less, threads often wait for a slow batch (a dense z12 area, or a big + // multipolygon read from --store files), above all when two threads share a core and each runs at + // half speed: a 16-thread planet with --store took 54 min with 1 GB, 25 min with 4 GB. + const uint64_t maxHeldBytes = std::max((64ull << 20) * options.threadNum, + std::min(4ull << 30, availableMemoryBytes() / 16)); + if (toPMTiles) sharedData.pmtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toMBTiles) sharedData.mbtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toPMTiles) sharedData.pmtiles.batches.setFirstUnhanded(&firstUntakenKey); + if (toMBTiles) sharedData.mbtiles.batches.setFirstUnhanded(&firstUntakenKey); + + // Below a z12 tile whose objects, down to the last zoom, are all polygons covering the whole + // tile (open sea, large forests or ice), every tile of a zoom comes out the same: one per zoom + // is computed and its bytes are saved for the others. A z12 subtree is in the same batch as its + // z12 tile (see nextBatch). + const unsigned coverZoom = 12; + const bool reuseCovered = toPMTiles && !sharedData.mergeSqlite && !sharedData.config.clippingBoxFromJSON && config.endZoom > coverZoom; + auto coveredByPolygons = [&](TileCoordinates coords) { + objectsUpToZoom = config.endZoom; + { + // Most tiles have some object that isn't a polygon, or none: find out without sorting them + bool any = false, other = false; + for (auto source : sources) { + const int kinds = source->objectKindsForTile(coverZoom, coords); + if (kinds == 2) { other = true; break; } + if (kinds == 1) any = true; + } + if (other || !any) { objectsUpToZoom = 0; return false; } + } + std::vector> objects; + bool any = false, polygons = true; + for (auto source : sources) { + objects.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, coverZoom, coords)); + for (const auto &o : objects.back()) { + any = true; + if (o.oo.geomType != POLYGON_) polygons = false; + } + } + objectsUpToZoom = 0; + if (!any || !polygons) return false; + // Clipped to the tile, each must be exactly the clipping box. These clips aren't kept in + // the clip cache, so the tiles below are built from the same clips as without this check. + const TileBbox bbox(coords, coverZoom, false, false); + const Box &clip = bbox.clippingBox; + const bool noStore = clipCacheNoStore; + clipCacheNoStore = true; + bool covered = true; + for (std::size_t k = 0; k < sources.size() && covered; k++) { + for (const auto &o : objects[k]) { + const Geometry g = sources[k]->buildWayGeometry(o.oo.geomType, o.oo.objectID, bbox); + const MultiPolygon *mp = boost::get(&g); + if (!mp || mp->size() != 1 || !(*mp)[0].inners().empty() || (*mp)[0].outer().size() != 5) { covered = false; break; } + Box env; + geom::envelope(*mp, env); + if (env.min_corner().x() != clip.min_corner().x() || env.min_corner().y() != clip.min_corner().y() || + env.max_corner().x() != clip.max_corner().x() || env.max_corner().y() != clip.max_corner().y()) { covered = false; break; } + } + } + clipCacheNoStore = noStore; + return covered; + }; + + // The tile list (12 bytes a tile: 4 GB for a planet) is only read up to the tiles not written yet. + // The pages of the written part are given back as the tiles go, rather than all at the end. + std::mutex releaseMutex; + std::size_t tilesReleased = 0; // tiles before this one are given back + auto releaseWrittenTiles = [&]() { +#ifdef __linux__ + if (!toPMTiles && !toMBTiles) return; + const uint64_t nextKey = toPMTiles ? sharedData.pmtiles.batches.nextToWrite() : sharedData.mbtiles.batches.nextToWrite(); + const std::size_t written = batchStarts[std::min(nextKey >> 16, batchStarts.size() - 1)]; + const std::size_t step = (64u << 20) / sizeof(tileCoordinates[0]); + if (!releaseMutex.try_lock()) return; + if (written < tilesReleased + step) { releaseMutex.unlock(); return; } + const std::size_t page = sysconf(_SC_PAGESIZE); + const uintptr_t base = reinterpret_cast(tileCoordinates.data()); + const uintptr_t from = (base + tilesReleased * sizeof(tileCoordinates[0]) + page - 1) / page * page; + const uintptr_t to = (base + written * sizeof(tileCoordinates[0])) / page * page; + if (to > from) madvise(reinterpret_cast(from), to - from, MADV_DONTNEED); + tilesReleased = written; + releaseMutex.unlock(); #endif + }; - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); + for (unsigned int t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&]() { + std::size_t startIndex, batchSize; + uint64_t seq; + GroupWork &mine = *groupWork[nextWorker++]; + // Shared pieces first (see TileDataSource::precomputeSharedPieces) + for (auto source : sources) source->computeSharedPieces(); + std::vector tileTimings; + std::vector coverTiles(config.endZoom + 1); + std::vector coverDone(config.endZoom + 1); + auto beginWriting = [&](uint64_t key) { + if (toPMTiles) sharedData.pmtiles.batches.begin(key); + if (toMBTiles) sharedData.mbtiles.batches.begin(key); + }; + auto endWriting = [&](uint64_t nextKey) { + if (toPMTiles) { sharedData.pmtiles.batches.setNext(nextKey); sharedData.pmtiles.batches.end(); } + if (toMBTiles) { sharedData.mbtiles.batches.setNext(nextKey); sharedData.mbtiles.batches.end(); } + }; + // (the tile is read before it's written: see releaseWrittenTiles) + auto showProgress = [&](const std::pair &tile) { + if (io_mutex.try_lock()) { + uint64_t written = tilesWritten.load(); + + if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { + lastTilesWritten = written; + // Show progress grouped by z6 (or lower) + size_t z = tile.first; + size_t x = tile.second.x; + size_t y = tile.second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; + } + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + } + io_mutex.unlock(); } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); - -#ifdef CLOCK_MONOTONIC + }; + auto showTimings = [&]() { if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; } + tileTimings.clear(); + }; + // The tiles batchBegin .. endIndex-1 (a unit), in a new clip-cache batch or going on with + // the current one + auto workOnTiles = [&](std::size_t batchBegin, std::size_t endIndex, bool newCacheBatch, bool noStore) { + if (newCacheBatch) clipCacheBatchId = ++batchCounter; + // A single-tile batch has no descendants to reuse its clips. + clipCacheNoStore = noStore; + // The covered z12 tile of this unit, and the tiles saved for its descendants per zoom + bool coverActive = false; + TileCoordinates coverRoot(0, 0); + for(std::size_t i = batchBegin; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; + +#ifdef CLOCK_MONOTONIC + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif - } - if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; + const bool inCovered = coverActive && zoom > coverZoom && + (coords.x >> (zoom - coverZoom)) == coverRoot.x && (coords.y >> (zoom - coverZoom)) == coverRoot.y; + if (inCovered && coverDone[zoom]) { + sharedData.pmtiles.saveTile(zoom, coords.x, coords.y, coverTiles[zoom]); + } else { + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, zoom, coords)); + } + outputProc(sharedData, sources, attributeStore, data, coords, zoom, inCovered ? &coverTiles[zoom] : nullptr); + if (inCovered) { + coverDone[zoom] = true; + } else if (zoom <= coverZoom) { + coverActive = reuseCovered && zoom == coverZoom && coveredByPolygons(coords); + if (coverActive) { + coverRoot = coords; + std::fill(coverDone.begin(), coverDone.end(), false); + } + } + } + +#ifdef CLOCK_MONOTONIC + if (options.logTileTimings) { + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); + } +#endif + } + }; + + // The groups handed out to this thread + while (true) { + uint64_t needed = UINT64_MAX; + if (toPMTiles) needed = sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) needed = sharedData.mbtiles.batches.waitForRoom(); + const int got = nextBatch(startIndex, batchSize, seq, needed); + trimReleased(); + if (got == 0) break; + if (got == 2) continue; + { + // The group's units, open to other threads from now on + std::lock_guard lock(mine.m); + mine.unitStart.clear(); mine.unitKey.clear(); mine.unitNoStore.clear(); + for (std::size_t b = groupStarts[seq]; b < groupStarts[seq + 1]; b++) { + uint64_t unit = 0; + for (std::size_t i = batchStarts[b]; i < batchStarts[b + 1]; i++) { + if (i > batchStarts[b] && tileCoordinates[i].first > cutZoom) continue; + mine.unitStart.push_back(i); + mine.unitKey.push_back(((uint64_t)b << 16) | unit++); + mine.unitNoStore.push_back(batchStarts[b + 1] - batchStarts[b] == 1); + } + } + mine.unitStart.push_back(startIndex + batchSize); + mine.unitKey.push_back((uint64_t)groupStarts[seq + 1] << 16); + mine.group = seq; + mine.front = 0; + mine.back = mine.unitNoStore.size(); + mine.active = true; + } + const std::pair firstTile = tileCoordinates[startIndex]; + // Consecutive units are written as one run + std::size_t tilesDone = 0; + bool writing = false; + std::size_t last = 0; + while (true) { + std::size_t u; + { + std::lock_guard lock(mine.m); + if (mine.front >= mine.back) { mine.active = false; break; } + u = mine.front++; + } + const bool goesOn = writing && u == last + 1; + if (writing && !goesOn) { endWriting(mine.unitKey[last + 1]); writing = false; } + if (!writing) { beginWriting(mine.unitKey[u]); writing = true; } + // a new clip-cache batch at a batch start, and after a gap + workOnTiles(mine.unitStart[u], mine.unitStart[u + 1], !goesOn || (mine.unitKey[u] & 0xffff) == 0, mine.unitNoStore[u]); + tilesDone += mine.unitStart[u + 1] - mine.unitStart[u]; + last = u; + } + if (writing) endWriting(mine.unitKey[last + 1]); + tilesWritten += tilesDone; + showTimings(); + + { + const std::lock_guard lock(batchMutex); + // z6 tiles the hand-out has moved past (an early group's may still be ahead of it) + for (int64_t b : batchZ6) + if (--z6Running[b] == 0 && b < z6HandedOut) releaseZ6(b); + } + trimReleased(); + releaseWrittenTiles(); + showProgress(firstTile); } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - uint64_t written = tilesWritten.load(); - - if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { - lastTilesWritten = written; - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; + // All groups are handed out: take the last units not started yet of the groups still + // being worked on + while (true) { + if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); + std::size_t from = 0, to = 0; + uint64_t key = 0, nextKey = 0; + bool noStore = false, found = false; + { + // Under batchMutex, so that the owner can't release the unit's z6 tiles meanwhile + const std::lock_guard lock(batchMutex); + batchZ6.clear(); + uint64_t group = 0; + for (auto &work : groupWork) { + std::lock_guard workLock(work->m); + if (!work->active || work->front >= work->back) continue; + const std::size_t u = --work->back; + from = work->unitStart[u]; to = work->unitStart[u + 1]; + key = work->unitKey[u]; nextKey = work->unitKey[u + 1]; + noStore = work->unitNoStore[u]; + group = work->group; + found = true; + break; } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + if (found && groupCountsZ6[group]) { + // Its z6 tiles' shared pieces are kept until it is done + for (std::size_t i = from; i < to; i++) { + const int64_t z6 = z6Of(i); + if (z6 >= 0 && (batchZ6.empty() || batchZ6.back() != z6)) batchZ6.push_back(z6); + } + for (int64_t z6 : batchZ6) z6Running[z6]++; + } + } + if (!found) break; + const std::pair firstTile = tileCoordinates[from]; + beginWriting(key); + workOnTiles(from, to, true, noStore); + tilesWritten += to - from; + endWriting(nextKey); + showTimings(); + { + const std::lock_guard lock(batchMutex); + for (int64_t z6 : batchZ6) + if (--z6Running[z6] == 0 && z6 < z6HandedOut) releaseZ6(z6); } - io_mutex.unlock(); + trimReleased(); + releaseWrittenTiles(); + showProgress(firstTile); } }); } // Wait for all tasks in the pool to complete. pool.join(); + if (toPMTiles) sharedData.pmtiles.batches.setFirstUnhanded(nullptr); + if (toMBTiles) sharedData.mbtiles.batches.setFirstUnhanded(nullptr); // ---- Close tileset diff --git a/src/visvalingam.cpp b/src/visvalingam.cpp index c278f5b2..de7d1df9 100644 --- a/src/visvalingam.cpp +++ b/src/visvalingam.cpp @@ -33,6 +33,8 @@ #include #include #include +#include +#include // Stuff to create the priority queue, or min heap. @@ -184,8 +186,10 @@ static double doubleTriangleArea(GeometryType const &ls, int start, int i1, int } +// The original implementation; visvalingam() below gives the same results +// faster and uses this one for the degenerate cases. template -GeometryType visvalingam(const GeometryType &ls, double threshold, size_t retain) { +static GeometryType visvalingamReference(const GeometryType &ls, double threshold, size_t retain) { size_t start = 0; size_t end = ls.size(); // - 1; int removed = 0; @@ -275,6 +279,529 @@ GeometryType visvalingam(const GeometryType &ls, double threshold, size_t retain return output; } +// Faster implementation of the same algorithm. It makes exactly the same +// removals in the same order as visvalingamReference, so its output is the +// same for every input (that is checked against recorded planet inputs): +// - The heap is the same binary heap. Every operation leaves every slot +// holding the same (area, item) entry as the reference's operation does, +// so equal areas are popped in the same order: +// - sifts move a hole instead of swapping (same final slots); +// - down(): "the smaller child, the left one on a tie, if it is smaller +// than the entry" equals the reference's "left if smaller than the +// entry, then right if smaller than that" (needs NaN-free areas, see +// the coordinate check below); +// - Pop() moves the hole along that smaller-child path to a leaf and then +// lets the last entry rise while its parent is >= it. The areas along +// the path never decrease, so the entry ends below exactly the path +// entries smaller than it: the slot the reference's down() gives it. +// Two levels per step: the grandchild pairs are compared together with +// the children, then the pair below the chosen child is used; +// - Update() to an unchanged area moves nothing in the reference either +// (no child is smaller), so it is skipped; +// - while the heap is built only Push() runs, so pos[] is filled once +// after the pushes. +// - Items are numbered like points (the reference's separate list-head item +// n becomes item 0, which the reference never uses); only the item numbers +// in the heap entries change, never a comparison. +// - The stop tests are made on the heap top before popping it. +// - Inputs where nothing can be removed (n <= retain, or every initial +// triangle area above the threshold) return ls without building the heap. +// - The output is the linked list in order, as in the reference. + +namespace { + +struct visFastEntry { + double area; + int32_t item; +}; + +struct visFastLink { + int32_t prev, next; // linked list, -1 = none +}; + +struct visFastScratch { + std::vector h; + std::vector pos; // item -> slot in the heap + std::vector link; + std::vector area; // initial triangle areas + + void prepare(size_t n) { + if (pos.size() < n) { + h.resize(n + 4); + pos.resize(n); + link.resize(n); + area.resize(n); + } + } + + // Heap storage starts at an address = 16 (mod 64): the children 2i+1, 2i+2 + // of a slot then share 32 bytes and its four grandchildren a cache line. + visFastEntry *heapBase() { + visFastEntry *p = h.data(); + while ((reinterpret_cast(p) & 63) != 16) p++; + return p; + } + + // Don't keep huge buffers alive after an unusually large ring. + void trim() { + if (pos.size() > (1 << 16)) { + std::vector().swap(h); + std::vector().swap(pos); + std::vector().swap(link); + std::vector().swap(area); + } + } +}; + +thread_local visFastScratch visFastTls; + +struct visFastHeap { + visFastEntry *h; + int32_t *pos; + size_t size = 0; + + void up(size_t i, visFastEntry object) { + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + const visFastEntry parent = h[up]; + if (parent.area <= object.area) break; + h[i] = parent; + pos[parent.item] = i; + i = up; + } + h[i] = object; + pos[object.item] = i; + } + + void down(size_t i, visFastEntry object) { + while (1) { + const size_t left = 2 * i + 1; + if (left >= size) break; + size_t c = left; + if (left + 1 < size && h[left + 1].area < h[left].area) c = left + 1; + const visFastEntry child = h[c]; + if (!(child.area < object.area)) break; + h[i] = child; + pos[child.item] = i; + i = c; + } + h[i] = object; + pos[object.item] = i; + } + + // Push without maintaining pos (fixPos() before any other operation). + void PushBuild(int32_t item, double area) { + size_t i = size++; + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + if (h[up].area <= area) break; + h[i] = h[up]; + i = up; + } + h[i] = visFastEntry{area, item}; + } + + void fixPos() { + for (size_t i = 0; i < size; i++) pos[h[i].item] = i; + } + + void Pop() { + const visFastEntry object = h[--size]; + if (size == 0) return; + size_t i = 0; + while (4 * i + 6 < size) { + const size_t left = 2 * i + 1; + const size_t g = 4 * i + 3; + const size_t c = left + (h[left + 1].area < h[left].area); + const size_t gl = g + (h[g + 1].area < h[g].area); + const size_t gr = g + 2 + (h[g + 3].area < h[g + 2].area); + const size_t gc = c == left ? gl : gr; + const visFastEntry child = h[c]; + const visFastEntry grandchild = h[gc]; + h[i] = child; + pos[child.item] = i; + h[c] = grandchild; + pos[grandchild.item] = c; + i = gc; + } + while (1) { + const size_t left = 2 * i + 1; + if (left >= size) break; + size_t c = left; + if (left + 1 < size && h[left + 1].area < h[left].area) c = left + 1; + const visFastEntry child = h[c]; + h[i] = child; + pos[child.item] = i; + i = c; + } + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + const visFastEntry parent = h[up]; + if (parent.area < object.area) break; + h[i] = parent; + pos[parent.item] = i; + i = up; + } + h[i] = object; + pos[object.item] = i; + } + + void Update(int32_t item, double area) { + const size_t i = pos[item]; + if (h[i].area > area) up(i, visFastEntry{area, item}); + else if (h[i].area < area) down(i, visFastEntry{area, item}); + } +}; + +} + +static inline double doubleTriangleArea(const Point &a, const Point &b, const Point &c) { + return std::abs((b.x() - a.x()) * (c.y() - a.y()) - (b.y() - a.y()) * (c.x() - a.x())); +} + +// Packed variant of visFastHeap for inputs with integral coordinates of +// bounded size (rings in tile coordinates): an entry is one uint64_t, the bit +// pattern of its area with the item number in the low K bits. +// - For x, y >= 0 (not NaN), x < y as doubles exactly when their bit +// patterns are in that order as unsigned integers; +infinity included. +// - With |coordinate| < 2^((50 - K) / 2) every doubled triangle area is an +// integer below 2^(53 - K) (|area| <= 8 max|c|^2), computed exactly as the +// same double, and such a double has its K lowest mantissa bits zero, so +// or-ing the item into them loses nothing (and infinity's are zero too). +// - "area of a < area of b" is a < (b & ~M) for M = 2^K - 1: the masked +// patterns differ by at least 2^K when the areas differ, so the item bits +// of a never change the outcome; for equal areas it is false, as for the +// doubles. +// So every comparison has the outcome visFastHeap's has, every operation +// moves the same entries to the same slots, and the removals are the same. +namespace { + +struct visPackHeap { + uint64_t *h; + int32_t *pos; + uint64_t hi; // ~M + uint64_t M; + size_t size = 0; + + bool lt(uint64_t a, uint64_t b) const { return a < (b & hi); } // area(a) < area(b) + + void up(size_t i, uint64_t object) { + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + const uint64_t parent = h[up]; + if (!lt(object, parent)) break; // parent <= object + h[i] = parent; + pos[parent & M] = i; + i = up; + } + h[i] = object; + pos[object & M] = i; + } + + void down(size_t i, uint64_t object) { + while (1) { + const size_t left = 2 * i + 1; + if (left >= size) break; + size_t c = left; + if (left + 1 < size && lt(h[left + 1], h[left])) c = left + 1; + const uint64_t child = h[c]; + if (!lt(child, object)) break; + h[i] = child; + pos[child & M] = i; + i = c; + } + h[i] = object; + pos[object & M] = i; + } + + void PushBuild(uint64_t key) { + size_t i = size++; + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + if (!lt(key, h[up])) break; + h[i] = h[up]; + i = up; + } + h[i] = key; + } + + void fixPos() { + for (size_t i = 0; i < size; i++) pos[h[i] & M] = i; + } + + void Pop() { + const uint64_t object = h[--size]; + if (size == 0) return; + size_t i = 0; + while (4 * i + 6 < size) { + const size_t left = 2 * i + 1; + const size_t g = 4 * i + 3; + const size_t c = left + lt(h[left + 1], h[left]); + const size_t gl = g + lt(h[g + 1], h[g]); + const size_t gr = g + 2 + lt(h[g + 3], h[g + 2]); + const size_t gc = c == left ? gl : gr; + const uint64_t child = h[c]; + const uint64_t grandchild = h[gc]; + h[i] = child; + pos[child & M] = i; + h[c] = grandchild; + pos[grandchild & M] = c; + i = gc; + } + while (1) { + const size_t left = 2 * i + 1; + if (left >= size) break; + size_t c = left; + if (left + 1 < size && lt(h[left + 1], h[left])) c = left + 1; + const uint64_t child = h[c]; + h[i] = child; + pos[child & M] = i; + i = c; + } + while (i > 0) { + const size_t up = ((i + 1) >> 1) - 1; + const uint64_t parent = h[up]; + if (lt(parent, object)) break; + h[i] = parent; + pos[parent & M] = i; + i = up; + } + h[i] = object; + pos[object & M] = i; + } + + // areaBits: bit pattern of the new area + void Update(int32_t item, uint64_t areaBits) { + const size_t i = pos[item]; + const uint64_t cur = h[i] & hi; + if (cur > areaBits) up(i, areaBits | uint64_t(item)); + else if (cur < areaBits) down(i, areaBits | uint64_t(item)); + } +}; + +struct visPackScratch { + std::vector h; + std::vector pos; + std::vector link; + + void prepare(size_t n) { + if (pos.size() < n) { + h.resize(n + 8); + pos.resize(n); + link.resize(n); + } + } + // heap storage at an address = 8 (mod 64): the children 2i+1, 2i+2 of a + // slot share 16 bytes, its grandchildren 32 and great-grandchildren a line + uint64_t *heapBase() { + uint64_t *p = h.data(); + while ((reinterpret_cast(p) & 63) != 8) p++; + return p; + } + void trim() { + if (pos.size() > (1 << 16)) { + std::vector().swap(h); + std::vector().swap(pos); + std::vector().swap(link); + } + } +}; + +thread_local visPackScratch visPackTls; + +inline uint64_t visBits(double d) { uint64_t u; std::memcpy(&u, &d, 8); return u; } +inline double visDouble(uint64_t u) { double d; std::memcpy(&d, &u, 8); return d; } + +} + +// The same reduction as in visvalingamRun() below with visPackHeap; area[] holds +// the initial doubled areas, threshold is already doubled, K as above. Returns +// the linked list of the remaining points (from item 0). +template +static const visFastLink *visvalingamPacked(const GeometryType &ls, double threshold, size_t retain, const double *area, unsigned K) { + const size_t n = ls.size(); + visPackScratch &s = visPackTls; + s.prepare(n); + const Point *p = &ls[0]; + visFastLink *link = s.link.data(); + const uint64_t M = (uint64_t(1) << K) - 1; + visPackHeap heap{s.heapBase(), s.pos.data(), ~M, M}; + const int32_t endItem = n - 1; + const uint64_t inf = visBits(INFINITY); + heap.PushBuild(inf | 0); + for (size_t i = 1; i < n - 1; i++) heap.PushBuild(visBits(area[i]) | uint64_t(i)); + heap.PushBuild(inf | uint64_t(endItem)); + heap.fixPos(); + for (int32_t i = 0; i <= endItem; i++) link[i] = visFastLink{i - 1, i + 1}; + link[endItem].next = -1; + + size_t removed = 0; + while (heap.size > 0) { + const uint64_t current = heap.h[0]; + const uint64_t currentArea = current & ~M; + if (visDouble(currentArea) > threshold) break; + if (n - removed <= retain) break; + heap.Pop(); + + const int32_t item = int32_t(current & M); + const visFastLink cl = link[item]; + const int32_t nextItem = cl.next; + const int32_t prevItem = cl.prev; + const int32_t prevPrev = link[prevItem].prev; + const int32_t nextNext = link[nextItem].next; + link[prevItem].next = nextItem; + link[nextItem].prev = prevItem; + removed++; + + // max of the areas = max of the bit patterns + if (prevPrev != -1) { + const uint64_t a = visBits(doubleTriangleArea(p[prevPrev], p[prevItem], p[nextItem])); + heap.Update(prevItem, std::max(a, currentArea)); + } + if (nextNext != -1) { + const uint64_t a = visBits(doubleTriangleArea(p[prevItem], p[nextItem], p[nextNext])); + heap.Update(nextItem, std::max(a, currentArea)); + } + } + + return link; +} + +// Runs the reduction. Returns the linked list of the remaining points (from +// item 0, in the thread's scratch space; visTrim() when done with it), or +// nullptr with status 0 when nothing is removed, 2 when the caller must use +// visvalingamReference. +template +static const visFastLink *visvalingamRun(const GeometryType &ls, double threshold, size_t retain, int &status) { + const size_t n = ls.size(); + status = 2; + // Fewer than 3 points or retain < 2 (the reference's list head / end item + // handling differs there), or coordinates so large that an area could + // overflow to NaN (with |coordinate| <= 1e100 all areas are finite, which + // the heap above needs). + if (n < 3 || retain < 2) return nullptr; + // K: bits for the item numbers 0..n-1 in visPackHeap; packable: every + // coordinate is an integer of absolute value below lim (see visPackHeap) + unsigned K = 1; + while ((size_t(1) << K) < n) K++; + // (small inputs are faster without it: the check costs more than it saves) + bool packable = n >= 16 && K <= 40; + if (packable) { + const double lim = std::ldexp(1.0, (50 - int(K)) / 2); + for (const Point &q : ls) { + const double ax = std::abs(q.x()), ay = std::abs(q.y()); + // the conversions only see values in [-1, lim) + const double cx = ax < lim ? ax : -1.0, cy = ay < lim ? ay : -1.0; + if (!(double(int64_t(cx)) == ax && double(int64_t(cy)) == ay)) { packable = false; break; } + } + } + // (packable implies every |coordinate| < lim <= 2^25) + if (!packable) { + bool small = true; + for (const Point &q : ls) small &= std::abs(q.x()) <= 1e100 && std::abs(q.y()) <= 1e100; + if (!small) return nullptr; + } + status = 0; + threshold *= 2; + // n <= retain: the first pop stops the reduction (by its area or by the + // retain test), nothing is removed. + if (n <= retain) return nullptr; + + visFastScratch &s = visFastTls; + s.prepare(n); + const Point *p = &ls[0]; + + // If every initial area exceeds the threshold, so does the heap minimum + // (the list head and end have area infinity, and the threshold is below + // some finite area): the first pop stops the reduction. + double *area = s.area.data(); + bool allAbove = true; + for (size_t i = 1; i < n - 1; i++) { + area[i] = doubleTriangleArea(p[i - 1], p[i], p[i + 1]); + allAbove &= area[i] > threshold; + } + if (allAbove) return nullptr; + status = 1; + if (packable) return visvalingamPacked(ls, threshold, retain, area, K); + + // Item 0 is the list head (point 0), items 1..n-1 the other points, pushed + // in the reference's order: head, 1..n-2, end. + visFastLink *link = s.link.data(); + visFastHeap heap{s.heapBase(), s.pos.data()}; + const int32_t endItem = n - 1; + heap.PushBuild(0, INFINITY); + for (size_t i = 1; i < n - 1; i++) heap.PushBuild(i, area[i]); + heap.PushBuild(endItem, INFINITY); + heap.fixPos(); + for (int32_t i = 0; i <= endItem; i++) link[i] = visFastLink{i - 1, i + 1}; + link[endItem].next = -1; + + // The head and end are never popped: they have area infinity, all other + // areas are finite, so when they are the minimum only they are left and + // the retain test (2 <= retain) stops first. + size_t removed = 0; + while (heap.size > 0) { + const visFastEntry current = heap.h[0]; + if (current.area > threshold) break; + if (n - removed <= retain) break; + heap.Pop(); + + const visFastLink cl = link[current.item]; + const int32_t nextItem = cl.next; + const int32_t prevItem = cl.prev; + const int32_t prevPrev = link[prevItem].prev; + const int32_t nextNext = link[nextItem].next; + link[prevItem].next = nextItem; + link[nextItem].prev = prevItem; + removed++; + + if (prevPrev != -1) { + const double a = doubleTriangleArea(p[prevPrev], p[prevItem], p[nextItem]); + heap.Update(prevItem, std::max(a, current.area)); + } + if (nextNext != -1) { + const double a = doubleTriangleArea(p[prevItem], p[nextItem], p[nextNext]); + heap.Update(nextItem, std::max(a, current.area)); + } + } + + return link; +} + +static void visTrim() { + visFastTls.trim(); + visPackTls.trim(); +} + +template +GeometryType visvalingam(const GeometryType &ls, double threshold, size_t retain) { + int status; + const visFastLink *link = visvalingamRun(ls, threshold, retain, status); + if (status == 2) return visvalingamReference(ls, threshold, retain); + if (status == 0) return ls; + size_t m = 0; + for (int32_t item = 0; item != -1; item = link[item].next) m++; + GeometryType output; + output.reserve(m); + for (int32_t item = 0; item != -1; item = link[item].next) output.emplace_back(ls[item]); + visTrim(); + return output; +} + +// As visvalingam(), on ls itself: the remaining points are moved to the front +// (they keep their order, so each one moves to an index <= its own). +template +static void visvalingamInPlace(GeometryType &ls, double threshold, size_t retain) { + int status; + const visFastLink *link = visvalingamRun(ls, threshold, retain, status); + if (status == 2) { ls = visvalingamReference(ls, threshold, retain); return; } + if (status == 0) return; + size_t m = 0; + for (int32_t item = 0; item != -1; item = link[item].next) ls[m++] = ls[item]; + ls.resize(m); + visTrim(); +} + Linestring simplifyVis(const Linestring &ls, double max_distance) { if (ls.size()<3) return ls; bool isClosed = ls[0].x()==ls[ls.size()-1].x() && ls[0].y()==ls[ls.size()-1].y(); @@ -298,10 +825,16 @@ MultiPolygon simplifyVis(const MultiPolygon &mp, double max_distance) { // allValid: set to repair_multi_polygon's result, i.e. true when every polygon // of the output is individually valid. MultiPolygon simplifyVis(const MultiPolygon &mp, double max_distance, bool &allValid) { - MultiPolygon output; - output.reserve(mp.size()); - for (const auto &p : mp) { - output.emplace_back(simplifyVis(p, max_distance)); + MultiPolygon output = mp; + simplifyVisInPlace(output, max_distance, allValid); + return output; +} +// The same as mp = simplifyVis(mp, max_distance, allValid), without new rings +void simplifyVisInPlace(MultiPolygon &mp, double max_distance, bool &allValid) { + const double threshold = max_distance * max_distance * 4; + for (auto &p : mp) { + visvalingamInPlace(p.outer(), threshold, 4); + for (auto &ring : p.inners()) visvalingamInPlace(ring, threshold, 4); } // Per-ring simplification can leave the multipolygon invalid. Use the // area-preserving repair instead of a bare make_valid: on large/complex @@ -309,6 +842,5 @@ MultiPolygon simplifyVis(const MultiPolygon &mp, double max_distance, bool &allV // almost all of its area (which showed up as missing lake tiles at low // zoom). repair_multi_polygon keeps the simplified geometry untouched if a // repair would not preserve the covered area. - allValid = repair_multi_polygon(output); - return output; + allValid = repair_multi_polygon(mp); } diff --git a/test/append_vector.test.cpp b/test/append_vector.test.cpp index b3104428..f0d5d704 100644 --- a/test/append_vector.test.cpp +++ b/test/append_vector.test.cpp @@ -21,6 +21,12 @@ MU_TEST(test_append_vector) { mu_check(vec[25] == 25); + const AppendVector& constVec = vec; + mu_check(constVec[25] == 25); + mu_check(constVec[8191] == 8191); + mu_check(constVec[8192] == 8192); + mu_check(constVec[9999] == 9999); + const AppendVector::Iterator& it = vec.begin(); mu_check(*it == 0); mu_check(*(it + 1) == 1); diff --git a/test/correct.test.cpp b/test/correct.test.cpp new file mode 100644 index 00000000..7178d173 --- /dev/null +++ b/test/correct.test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include "external/minunit.h" +#include "geometry/correct.hpp" + +typedef boost::geometry::model::d2::point_xy point_t; +typedef boost::geometry::model::polygon polygon_t; +typedef boost::geometry::model::multi_polygon multi_polygon_t; + +static bool near(double a, double b) { return std::abs(a - b) < 1e-9; } + +MU_TEST(test_correct_merges_overlapping_holes) { + // Two overlapping holes are merged into one; a disjoint hole is kept as is + polygon_t input; + boost::geometry::read_wkt( + "POLYGON((0 0,0 10,10 10,10 0,0 0)," + "(1 1,3 1,3 3,1 3,1 1),(2 2,4 2,4 4,2 4,2 2),(6 6,7 6,7 7,6 7,6 6))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 2); + mu_check(near(boost::geometry::area(output), 100 - 7 - 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_merges_overlapping_outers) { + multi_polygon_t input; + boost::geometry::read_wkt( + "MULTIPOLYGON(((0 0,0 2,2 2,2 0,0 0)),((1 1,1 3,3 3,3 1,1 1)),((5 5,5 6,6 6,6 5,5 5)))", input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 2); + mu_check(near(boost::geometry::area(output), 4 + 4 - 1 + 1)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST(test_correct_many_disjoint_holes) { + // 400 disjoint holes, e.g. a lake with many islands: all are kept + std::string wkt = "POLYGON((0 0,0 100,100 100,100 0,0 0)"; + for (int x = 0; x < 20; x++) + for (int y = 0; y < 20; y++) { + double x0 = 1 + x * 5, y0 = 1 + y * 5; + wkt += ",(" + std::to_string(x0) + " " + std::to_string(y0) + "," + std::to_string(x0 + 2) + " " + std::to_string(y0) + "," + + std::to_string(x0 + 2) + " " + std::to_string(y0 + 2) + "," + std::to_string(x0) + " " + std::to_string(y0 + 2) + "," + + std::to_string(x0) + " " + std::to_string(y0) + ")"; + } + wkt += ")"; + polygon_t input; + boost::geometry::read_wkt(wkt, input); + multi_polygon_t output; + geometry::correct(input, output); + + mu_check(output.size() == 1); + mu_check(output[0].inners().size() == 400); + mu_check(near(boost::geometry::area(output), 100 * 100 - 400 * 4)); + mu_check(boost::geometry::is_valid(output)); +} + +MU_TEST_SUITE(test_suite_correct) { + MU_RUN_TEST(test_correct_merges_overlapping_holes); + MU_RUN_TEST(test_correct_merges_overlapping_outers); + MU_RUN_TEST(test_correct_many_disjoint_holes); +} + +int main() { + MU_RUN_SUITE(test_suite_correct); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/geom.test.cpp b/test/geom.test.cpp index e33b5990..2a31d97d 100644 --- a/test/geom.test.cpp +++ b/test/geom.test.cpp @@ -116,6 +116,44 @@ MU_TEST(test_precheck_many_holes) { checkPrecheck(mp, false); } +static Ring squareRing(double x, double y, double size) { + Ring r; + geom::append(r, Point(x, y)); geom::append(r, Point(x + size, y)); + geom::append(r, Point(x + size, y + size)); geom::append(r, Point(x, y + size)); + geom::append(r, Point(x, y)); + return r; +} + +MU_TEST(test_precheck_large_polygon) { + // A 40000-vertex outer ring (a circle) with 3600 holes: far more than walking the + // outer ring for every hole would allow (the old 2^26 work cap) + Polygon p; + const int n = 40000; + for (int i = 0; i <= n; i++) { + const double a = 2 * M_PI * (i % n) / n; + geom::append(p.outer(), Point(100 + 100 * std::cos(a), 100 + 100 * std::sin(a))); + } + for (int i = 0; i < 60; i++) + for (int j = 0; j < 60; j++) + p.inners().push_back(squareRing(40 + i * 2, 40 + j * 2, 1)); + MultiPolygon mp; + mp.push_back(p); + geom::correct(mp); + checkPrecheck(mp, true); + + // A hole outside the outer ring + MultiPolygon outside = mp; + outside[0].inners().push_back(squareRing(300, 100, 1)); + geom::correct(outside); + checkPrecheck(outside, false); + + // A hole nested in a hole on the other side of the polygon + MultiPolygon nested = mp; + nested[0].inners().insert(nested[0].inners().begin(), squareRing(40.25, 40.25, 0.5)); + geom::correct(nested); + checkPrecheck(nested, false); +} + MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_concave); MU_RUN_TEST(test_rect_clip_holes); @@ -124,6 +162,7 @@ MU_TEST_SUITE(test_suite_geom) { MU_RUN_TEST(test_rect_clip_touch_from_inside); MU_RUN_TEST(test_precheck_overlapping_bboxes); MU_RUN_TEST(test_precheck_many_holes); + MU_RUN_TEST(test_precheck_large_polygon); } int main() { diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp new file mode 100644 index 00000000..b9ff9599 --- /dev/null +++ b/test/ordered_batches.test.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include +#include +#include "external/minunit.h" +#include "ordered_batches.h" + +MU_TEST(test_ordered_batches_single_thread) { + std::vector written; + OrderedBatches batches([&](int& item) { written.push_back(item); }, [](const int&) { return 1; }); + + // batch 1 finishes first, so it's held until batch 0 is written + batches.begin(1); + mu_check(batches.inBatch()); + batches.add(10); + batches.add(11); + batches.end(); + mu_check(!batches.inBatch()); + mu_check(written.empty()); + mu_check(!batches.allWritten()); + + batches.begin(0); + batches.add(0); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11 })); + mu_check(batches.allWritten()); + + // an empty batch still moves the order on + batches.begin(3); + batches.add(30); + batches.end(); + batches.begin(2); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11, 30 })); + mu_check(batches.allWritten()); +} + +MU_TEST(test_ordered_batches_threads) { + // Batches are handed out in order and finish in any order; with a small limit on what + // may be held, threads also have to wait for room + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written; + OrderedBatches batches([&](uint64_t& item) { written.push_back(item); }, [](const uint64_t&) { return 1; }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(written.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < written.size(); i++) if (written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST(test_ordered_batches_flush) { + // With flushItem: every item is written and then flushed, both in order, flushes never + // overlap, and a flush can be done by a thread other than the one that wrote the batch + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written, flushed; + std::atomic flushingNow(0); + std::atomic overlap(false), flushedBeforeWritten(false); + std::atomic writtenCount(0); + OrderedBatches batches( + [&](uint64_t& item) { written.push_back(item); writtenCount++; }, + [](const uint64_t&) { return 1; }, + [&](uint64_t& item) { + if (flushingNow++ != 0) overlap = true; + if (item >= writtenCount) flushedBeforeWritten = true; + flushed.push_back(item); + if (item % 7 == 0) std::this_thread::yield(); + flushingNow--; + }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(!overlap); + mu_check(!flushedBeforeWritten); + mu_check(written.size() == numBatches * itemsPerBatch); + mu_check(flushed.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < flushed.size(); i++) if (flushed[i] != i || written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST_SUITE(test_suite_ordered_batches) { + MU_RUN_TEST(test_ordered_batches_single_thread); + MU_RUN_TEST(test_ordered_batches_threads); + MU_RUN_TEST(test_ordered_batches_flush); +} + +int main() { + MU_RUN_SUITE(test_suite_ordered_batches); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/shp_mem_tiles.test.cpp b/test/shp_mem_tiles.test.cpp new file mode 100644 index 00000000..b4629162 --- /dev/null +++ b/test/shp_mem_tiles.test.cpp @@ -0,0 +1,66 @@ +#include +#include "shp_mem_tiles.h" +#include "external/minunit.h" + +bool verbose = false; + +namespace { + MultiPolygon square(double x, double y, double size) { + MultiPolygon mp; + Polygon p; + geom::append(p.outer(), Point(x, y)); + geom::append(p.outer(), Point(x, y + size)); + geom::append(p.outer(), Point(x + size, y + size)); + geom::append(p.outer(), Point(x + size, y)); + geom::append(p.outer(), Point(x, y)); + mp.push_back(p); + return mp; + } + + Box box(double x1, double y1, double x2, double y2) { + return Box(Point(x1, y1), Point(x2, y2)); + } +} + +MU_TEST(test_may_intersect) { + Declutter declutter; + ShpMemTiles shp(1, 14, declutter); + shp.CreateNamedLayerIndex("layer"); + + // Two small squares at opposite corners: the shape's bounding box, (0,0)-(10,10), + // is almost all empty, like that of a country with far-flung islands. + MultiPolygon mp = square(0, 0, 1); + mp.push_back(square(9, 9, 1)[0]); + shp.StoreGeometry(0, "layer", POLYGON_, mp, true, true, "corners", 0, 0, 0); + + // Outside the shape's bounding box + mu_check(!shp.mayIntersect("layer", box(20, 20, 21, 21))); + + // A small box (a few z15 tiles) in the empty middle: checked tile by tile + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + // A large box (tens of thousands of z15 tiles) in the empty middle: checked at once + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + + // Small and large boxes that touch a square + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 5, 5))); + mu_check(shp.mayIntersect("layer", box(5, 5, 9.5, 9.5))); + + // A large box that surrounds a square without containing a vertex of it + mu_check(shp.mayIntersect("layer", box(-1, -1, 2, 2))); + + // Answers stay the same once tiles have been checked and cached + mu_check(!shp.mayIntersect("layer", box(5, 5, 5.001, 5.001))); + mu_check(!shp.mayIntersect("layer", box(3, 3, 7, 7))); + mu_check(shp.mayIntersect("layer", box(0.5, 0.5, 0.501, 0.501))); +} + +MU_TEST_SUITE(test_suite_shp_mem_tiles) { + MU_RUN_TEST(test_may_intersect); +} + +int main() { + MU_RUN_SUITE(test_suite_shp_mem_tiles); + MU_REPORT(); + return MU_EXIT_CODE; +} diff --git a/test/tile_coordinates_set.test.cpp b/test/tile_coordinates_set.test.cpp index 87a054b8..973cc13c 100644 --- a/test/tile_coordinates_set.test.cpp +++ b/test/tile_coordinates_set.test.cpp @@ -45,10 +45,71 @@ MU_TEST(test_tile_coordinates_set) { mu_check(z2.test(1, 1) == true); mu_check(z2.test(2, 2) == false); } + + // forEachSet on a wrapped set visits the same tiles as a full scan, in x-major, y-minor order + { + PreciseTileCoordinatesSet z3(3); + z3.set(0, 0); z3.set(0, 5); z3.set(2, 1); z3.set(2, 7); z3.set(7, 3); + LossyTileCoordinatesSet z5(5, z3); + + std::vector> expected, visited; + for (TileCoordinate x = 0; x < 32; x++) + for (TileCoordinate y = 0; y < 32; y++) + if (z5.test(x, y)) expected.push_back({x, y}); + z5.forEachSet([&](TileCoordinate x, TileCoordinate y) { visited.push_back({x, y}); }); + + mu_check(expected.size() == 5 * 16); + mu_check(visited == expected); + } +} + +static bool sameTiles(const TileCoordinatesSet& a, const TileCoordinatesSet& b) { + std::vector> ta, tb; + a.forEachSet([&](TileCoordinate x, TileCoordinate y) { ta.emplace_back(x, y); }); + b.forEachSet([&](TileCoordinate x, TileCoordinate y) { tb.emplace_back(x, y); }); + return ta == tb && a.size() == b.size(); +} + +MU_TEST(test_tile_coordinates_set_box) { + // setBox must set exactly the tiles that set() would, including boxes that + // start and end mid-word, span several words, or are clipped at the edge + const TileCoordinate boxes[][4] = { + { 0, 0, 0, 0 }, { 3, 9, 5, 70 }, { 100, 101, 0, 1023 }, { 7, 7, 60, 200 }, + { 1000, 1023, 990, 1023 }, { 1020, 1023, 1000, 1023 } + }; + for (const auto& b : boxes) { + PreciseTileCoordinatesSet bySet(10), byBox(10); + for (TileCoordinate x = b[0]; x <= b[1]; x++) + for (TileCoordinate y = b[2]; y <= b[3]; y++) + bySet.set(x, y); + byBox.setBox(b[0], b[1], b[2], b[3]); + mu_check(sameTiles(bySet, byBox)); + } + // Partly outside the 1024x1024 grid: the part inside is set + PreciseTileCoordinatesSet edge(10); + edge.setBox(1020, 1030, 1022, 1030); + mu_check(edge.size() == 4 * 2); + mu_check(edge.test(1023, 1023)); +} + +MU_TEST(test_tile_coordinates_set_merge) { + PreciseTileCoordinatesSet a(12), b(12), both(12); + for (TileCoordinate i = 0; i < 4096; i += 37) { + a.set(i, (i * 7) % 4096); + both.set(i, (i * 7) % 4096); + } + for (TileCoordinate i = 0; i < 4096; i += 53) { + b.set((i * 3) % 4096, i); + both.set((i * 3) % 4096, i); + } + a.merge(b); + mu_check(sameTiles(a, both)); } MU_TEST_SUITE(test_suite_tile_coordinates_set) { MU_RUN_TEST(test_tile_coordinates_set); + MU_RUN_TEST(test_tile_coordinates_set_box); + MU_RUN_TEST(test_tile_coordinates_set_merge); } int main() {