diff --git a/include/attribute_store.h b/include/attribute_store.h index 295c495c..849b3889 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include "pooled_string.h" @@ -30,11 +32,16 @@ class AttributeKeyStore { uint16_t key2index(const std::string& key); const std::string& getKey(uint16_t index) const; const std::string& getKeyUnsafe(uint16_t index) const; - void finalize() { finalized = true; } + // Position of the key in alphabetical order. Unlike the key index, which is + // handed out in first-use order (i.e. depends on thread scheduling), it is + // the same in every run. Only valid after finalize(). + uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; } + void finalize(); std::atomic keys2indexSize; private: bool finalized; + std::vector ranks; mutable std::mutex keys2indexMutex; // NB: we use a deque, not a vector, because a deque never invalidates // pointers to its members as long as you only push_back @@ -439,6 +446,24 @@ struct AttributeStore { } attrSet->forEachPair(fn); } + // As forEachPairIndexUnsafe, but in an order that is the same in every run + // (by key name), not by pair index, which depends on thread scheduling. + template + void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const { + boost::container::small_vector order; + forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); }); + if (order.size() > 1) + std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) { + return comparePairsUnsafe(a, b) < 0; + }); + for (const uint32_t pairIndex : order) + fn(pairIndex); + } + // Order and hash attribute pairs/sets by content (key names and values), not by + // index, so that they are the same in every run. Only valid after finalize(). + int comparePairsUnsafe(uint32_t a, uint32_t b) const; + int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const; + uint64_t contentHashUnsafe(AttributeIndex index) const; void reset(); // used for testing size_t size() const; void reportSize() const; diff --git a/include/osm_store.h b/include/osm_store.h index 4f50cd5f..1f9915ec 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -154,22 +154,31 @@ class RelationScanStore { RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + // Keep each member's parent relations ordered by relation id: relations are scanned on + // several threads, so appending would make the order NextRelation() returns depend on + // thread scheduling. Entries of the same relation keep their member order. + static void insertByRelation(std::vector>& list, RelationID relid, uint16_t roleId) { + const auto it = std::upper_bound(list.begin(), list.end(), relid, + [](RelationID id, const std::pair& entry) { return id < entry.first; }); + list.emplace(it, relid, roleId); + } + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForWays[shard][wayid], relid, roleId); } void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); - relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForNodes[shard][nodeId], relid, roleId); } void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); std::lock_guard lock(mutex[0]); - relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + insertByRelation(relationsForRelations[relationId], relid, roleId); } void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); diff --git a/include/stable_hash.h b/include/stable_hash.h new file mode 100644 index 00000000..e1af029f --- /dev/null +++ b/include/stable_hash.h @@ -0,0 +1,42 @@ +/*! \file */ +#ifndef _STABLE_HASH_H +#define _STABLE_HASH_H + +#include +#include +#include + +// Hash functions that give the same result on every platform, compiler and +// library version (unlike std::hash or boost::hash), for orderings that the +// output depends on. + +inline uint64_t stableMix(uint64_t x) { + // splitmix64 finaliser + x += 0x9e3779b97f4a7c15ull; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebull; + return x ^ (x >> 31); +} + +inline uint64_t stableCombine(uint64_t seed, uint64_t value) { + return stableMix(seed ^ stableMix(value)); +} + +inline uint64_t stableCombine(uint64_t seed, double value) { + if (value == 0) value = 0; // -0.0 and 0.0 compare equal, so hash them equally + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return stableCombine(seed, bits); +} + +inline uint64_t stableCombine(uint64_t seed, const char* data, std::size_t length) { + // FNV-1a over the bytes + uint64_t h = 0xcbf29ce484222325ull; + for (std::size_t i = 0; i < length; i++) { + h ^= static_cast(data[i]); + h *= 0x100000001b3ull; + } + return stableCombine(stableCombine(seed, static_cast(length)), h); +} + +#endif //_STABLE_HASH_H diff --git a/include/tile_data.h b/include/tile_data.h index 3e8d20e9..5729d338 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -348,9 +348,11 @@ class TileDataSource { std::vector getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ); + void orderStoredGeometryTies(std::vector& data) const; virtual Geometry buildWayGeometry(OutputGeometryType const geomType, NodeID const objectID, const TileBbox &bbox); virtual LatpLon buildNodeGeometry(NodeID const objectID, const TileBbox &bbox) const; diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6e661c8d..f2a3d77d 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -1,4 +1,5 @@ #include "attribute_store.h" +#include "stable_hash.h" #include #include @@ -55,6 +56,15 @@ const std::string& AttributeKeyStore::getKeyUnsafe(uint16_t index) const { return keys[index]; } +void AttributeKeyStore::finalize() { + finalized = true; + // keys2index is ordered by key string + ranks.assign(keys.size(), 0); + uint16_t rank = 0; + for (const auto& it : keys2index) + ranks[it.second] = ++rank; +} + // AttributePair void AttributePair::ensureStringIsOwned() { // Before we store an AttributePair in our long-term storage, we need @@ -557,6 +567,52 @@ void AttributeStore::reset() { pairIndexGeneration++; } +int AttributeStore::comparePairsUnsafe(uint32_t a, uint32_t b) const { + if (a == b) return 0; + const AttributePair& p = pairStore.getPairUnsafe(a); + const AttributePair& q = pairStore.getPairUnsafe(b); + const uint16_t pRank = keyStore.getRankUnsafe(p.keyIndex), qRank = keyStore.getRankUnsafe(q.keyIndex); + if (pRank != qRank) return pRank < qRank ? -1 : 1; + if (p.minzoom != q.minzoom) return p.minzoom < q.minzoom ? -1 : 1; + if (p.valueType != q.valueType) return p.valueType < q.valueType ? -1 : 1; + if (p.hasStringValue()) { + if (p.pooledString() < q.pooledString()) return -1; + if (q.pooledString() < p.pooledString()) return 1; + return 0; + } + if (p.doubleValue_ < q.doubleValue_) return -1; + if (q.doubleValue_ < p.doubleValue_) return 1; + return 0; +} + +int AttributeStore::compareContentUnsafe(AttributeIndex a, AttributeIndex b) const { + if (a == b) return 0; + boost::container::small_vector x, y; + forEachPairIndexCanonicalUnsafe(a, [&](uint32_t pairIndex) { x.push_back(pairIndex); }); + forEachPairIndexCanonicalUnsafe(b, [&](uint32_t pairIndex) { y.push_back(pairIndex); }); + for (size_t i = 0; i < x.size() && i < y.size(); i++) { + const int c = comparePairsUnsafe(x[i], y[i]); + if (c != 0) return c; + } + if (x.size() != y.size()) return x.size() < y.size() ? -1 : 1; + return 0; +} + +uint64_t AttributeStore::contentHashUnsafe(AttributeIndex index) const { + // Sum of the pair hashes, so the (index dependent) pair order doesn't matter + uint64_t rv = 0; + forEachPairUnsafe(index, [&](const AttributePair& p) { + uint64_t h = stableCombine(keyStore.getRankUnsafe(p.keyIndex), static_cast(p.minzoom)); + h = stableCombine(h, static_cast(p.valueType)); + if (p.hasStringValue()) + h = stableCombine(h, p.pooledString().data(), p.pooledString().size()); + else + h = stableCombine(h, p.doubleValue_); + rv += h; + }); + return rv; +} + void AttributeStore::finalize() { finalized = true; keyStore.finalize(); diff --git a/src/output_object.cpp b/src/output_object.cpp index 3d5703ca..bc4c1855 100644 --- a/src/output_object.cpp +++ b/src/output_object.cpp @@ -42,8 +42,8 @@ static void writeAttributesTo( FeatureBuilder& fbuilder, char zoom ) { - attributeStore.forEachPairUnsafe(attributes, [&](const AttributePair& pair) { - const AttributePair* it = &pair; + attributeStore.forEachPairIndexCanonicalUnsafe(attributes, [&](uint32_t pairIndex) { + const AttributePair* it = &attributeStore.pairStore.getPairUnsafe(pairIndex); if (it->minzoom > zoom) return; // TODO: consider taking a data view that is stable @@ -179,7 +179,7 @@ const uint32_t* MvtAttributeMemo::resolve(const AttributeStore& attributeStore, if (pairs.used >= MEMO_MAX_PAIRS) pairs.clear(); const uint32_t begin = entries.size(); - attributeStore.forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { + attributeStore.forEachPairIndexCanonicalUnsafe(index, [&](uint32_t pairIndex) { uint32_t key, value; if (const Table::Slot* s = pairs.find(pairIndex)) { key = s->a; diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 89b367e0..093669e0 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,7 @@ #include "tile_data.h" #include "coordinates_geom.h" #include "leased_store.h" +#include "stable_hash.h" #include using namespace std; @@ -511,22 +513,128 @@ void populateTilesAtZoom( void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ); std::vector TileDataSource::getObjectsForTile( const std::vector& sortOrders, + const AttributeStore& attributeStore, unsigned int zoom, TileCoordinates coordinates ) { std::vector data; collectObjectsForTile(zoom, coordinates, data); collectLargeObjectsForTile(zoom, coordinates, data); - sortOutputObjectIDs(sortOrders, data); + sortOutputObjectIDs(sortOrders, attributeStore, data); data.erase(unique(data.begin(), data.end()), data.end()); + orderStoredGeometryTies(data); return data; } +namespace { +struct GeometryTieKey { + double x, y; // first point + uint64_t points; // number of points + uint64_t hash; // of all points, only computed when the above are equal + OutputObjectID o; +}; + +// Adds a ring/linestring to the key: first point and point count, or (withHash) the hash of all points +template +void addToTieKey(GeometryTieKey& key, const R& ring, bool withHash) { + if (withHash) { + key.hash = stableCombine(key.hash, static_cast(ring.size())); + for (const auto& p : ring) { + key.hash = stableCombine(key.hash, static_cast(p.x())); + key.hash = stableCombine(key.hash, static_cast(p.y())); + } + return; + } + if (key.points == 0 && !ring.empty()) { + key.x = ring.front().x(); + key.y = ring.front().y(); + } + key.points += ring.size(); +} + +void geometryTieKey(const TileDataSource& source, GeometryTieKey& key, bool withHash) { + const NodeID objectID = key.o.oo.objectID; + switch (key.o.oo.geomType) { + case POINT_: { + const Point& p = source.retrievePoint(objectID); + addToTieKey(key, std::array{p}, withHash); + break; + } + case LINESTRING_: + addToTieKey(key, source.retrieveLinestring(objectID), withHash); + break; + case MULTILINESTRING_: + for (const auto& ls : source.retrieveMultiLinestring(objectID)) + addToTieKey(key, ls, withHash); + break; + case POLYGON_: + for (const auto& poly : source.retrieveMultiPolygon(objectID)) { + addToTieKey(key, poly.outer(), withHash); + for (const auto& inner : poly.inners()) + addToTieKey(key, inner, withHash); + } + break; + } +} +} + +// Geometries stored at runtime (relations, shapefiles, ...) have an objectID that +// depends on which thread stored them, so the sort ties objects that share layer, +// z_order, type, attributes and id in an arbitrary order. Order such runs by +// geometry instead. OSM ways and nodes carry their OSM id in the objectID, and +// sort after stored geometries, so they never need this. +void TileDataSource::orderStoredGeometryTies(std::vector& data) const { + const NodeID storedLimit = 1ull << TILE_DATA_ID_SIZE; + auto tied = [](const OutputObjectID& a, const OutputObjectID& b) { + return a.oo.layer == b.oo.layer && a.oo.z_order == b.oo.z_order && a.oo.geomType == b.oo.geomType && + a.oo.attributes == b.oo.attributes && a.id == b.id; + }; + + std::vector keys; + size_t i = 0; + while (i < data.size()) { + size_t j = i + 1; + if (data[i].oo.objectID < storedLimit) + while (j < data.size() && data[j].oo.objectID < storedLimit && tied(data[i], data[j])) j++; + if (j - i < 2) { i = j; continue; } + + keys.clear(); + for (size_t k = i; k < j; k++) { + keys.push_back(GeometryTieKey{0, 0, 0, 0, data[k]}); + geometryTieKey(*this, keys.back(), false); + } + auto byPoint = [](const GeometryTieKey& a, const GeometryTieKey& b) { + if (a.x != b.x) return a.x < b.x; + if (a.y != b.y) return a.y < b.y; + return a.points < b.points; + }; + std::sort(keys.begin(), keys.end(), byPoint); + + // Still tied: hash all points (identical geometries may stay in any order) + for (size_t a = 0; a < keys.size(); ) { + size_t b = a + 1; + while (b < keys.size() && !byPoint(keys[a], keys[b])) b++; + if (b - a > 1) { + for (size_t k = a; k < b; k++) geometryTieKey(*this, keys[k], true); + std::sort(keys.begin() + a, keys.begin() + b, [](const GeometryTieKey& p, const GeometryTieKey& q) { + if (p.hash != q.hash) return p.hash < q.hash; + return p.o.oo.objectID < q.o.oo.objectID; + }); + } + a = b; + } + + for (size_t k = i; k < j; k++) data[k] = keys[k - i].o; + i = j; + } +} + // ------------------------------------ // Add geometries to tile/large indices diff --git a/src/tile_sorting.cpp b/src/tile_sorting.cpp index 64588163..cc79e20d 100644 --- a/src/tile_sorting.cpp +++ b/src/tile_sorting.cpp @@ -208,26 +208,75 @@ template void sortOutputObjects( typename AppendVectorNS::AppendVector::Iterator end ); +namespace { +struct AttributeKeyCacheEntry { + AttributeIndex index; + uint64_t key; +}; +// Direct-mapped, per thread: neighbouring tiles share most attribute sets +const size_t ATTRIBUTE_KEY_CACHE_SIZE = 1 << 12; +thread_local std::vector tlsAttributeKeys; +thread_local const AttributeStore* tlsAttributeKeysStore = nullptr; + +struct KeyedOutputObjectID { + uint64_t attributeKey; + OutputObjectID o; +}; + +uint64_t attributeKey(const AttributeStore& attributeStore, AttributeIndex index) { + if (tlsAttributeKeysStore != &attributeStore) { + tlsAttributeKeys.assign(ATTRIBUTE_KEY_CACHE_SIZE, AttributeKeyCacheEntry{~0u, 0}); + tlsAttributeKeysStore = &attributeStore; + } + uint32_t slot = index; + slot ^= slot >> 16; slot *= 0x7feb352dU; slot ^= slot >> 15; + AttributeKeyCacheEntry& entry = tlsAttributeKeys[slot & (ATTRIBUTE_KEY_CACHE_SIZE - 1)]; + if (entry.index != index) + entry = AttributeKeyCacheEntry{index, attributeStore.contentHashUnsafe(index)}; + return entry.key; +} +} + void sortOutputObjectIDs( const std::vector& sortOrders, + const AttributeStore& attributeStore, std::vector& data ) { - // Lexicographic comparison, with the order of: layer, geomType, attributes, and objectID. + // Lexicographic comparison, with the order of: layer, geomType, attributes, id and objectID. // Note that attributes is preferred to objectID. // It is to arrange objects with the identical attributes continuously. // Such objects will be merged into one object, to reduce the size of output. - boost::sort::pdqsort(data.begin(), data.end(), [&sortOrders](const OutputObjectID& x, const OutputObjectID& y) -> bool { - if (x.oo.layer < y.oo.layer) return true; - if (x.oo.layer > y.oo.layer) return false; - if (x.oo.z_order < y.oo.z_order) return sortOrders[x.oo.layer]; - if (x.oo.z_order > y.oo.z_order) return !sortOrders[x.oo.layer]; - if (x.oo.geomType < y.oo.geomType) return true; - if (x.oo.geomType > y.oo.geomType) return false; - if (x.oo.attributes < y.oo.attributes) return true; - if (x.oo.attributes > y.oo.attributes) return false; - if (x.oo.objectID < y.oo.objectID) return true; - return false; + // + // Attribute sets are compared by content, not by AttributeIndex: the index is + // handed out in insertion order, which depends on thread scheduling. The same + // goes for the objectID of stored geometries, so the OSM id goes first; + // TileDataSource::orderStoredGeometryTies handles the remaining ties. + std::vector keyed; + keyed.reserve(data.size()); + for (const OutputObjectID& o : data) + keyed.push_back(KeyedOutputObjectID{attributeKey(attributeStore, o.oo.attributes), o}); + + boost::sort::pdqsort(keyed.begin(), keyed.end(), [&sortOrders, &attributeStore](const KeyedOutputObjectID& a, const KeyedOutputObjectID& b) -> bool { + const OutputObject& x = a.o.oo; + const OutputObject& y = b.o.oo; + if (x.layer < y.layer) return true; + if (x.layer > y.layer) return false; + if (x.z_order < y.z_order) return sortOrders[x.layer]; + if (x.z_order > y.z_order) return !sortOrders[x.layer]; + if (x.geomType < y.geomType) return true; + if (x.geomType > y.geomType) return false; + if (x.attributes != y.attributes) { + if (a.attributeKey != b.attributeKey) return a.attributeKey < b.attributeKey; + const int c = attributeStore.compareContentUnsafe(x.attributes, y.attributes); + if (c != 0) return c < 0; + return x.attributes < y.attributes; + } + if (a.o.id != b.o.id) return a.o.id < b.o.id; + return x.objectID < y.objectID; }); + + for (size_t i = 0; i < keyed.size(); i++) + data[i] = keyed[i].o; } void sortTileCoordinates( diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..7e5b8f89 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -548,7 +548,7 @@ int main(const int argc, const char* argv[]) { std::vector> data; for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); + data.emplace_back(source->getObjectsForTile(sortOrders, attributeStore, zoom, coords)); } outputProc(sharedData, sources, attributeStore, data, coords, zoom);