Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,16 @@ test: \
test_append_vector \
test_attribute_store \
test_config_validator \
test_correct \
test_deque_map \
test_geom \
test_helpers \
test_options_parser \
test_ordered_batches \
test_pbf_reader \
test_pooled_string \
test_relation_roles \
test_shp_mem_tiles \
test_significant_tags \
test_sorted_node_store \
test_sorted_way_store \
Expand Down Expand Up @@ -193,6 +196,10 @@ src/config_schema.h: resources/config-schema.json

src/config_validator.o: src/config_schema.h

test_correct: \
test/correct.test.o
$(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct

test_deque_map: \
test/deque_map.test.o
$(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map
Expand Down Expand Up @@ -223,6 +230,10 @@ test_options_parser: \
test/options_parser.test.o
$(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser

test_ordered_batches: \
test/ordered_batches.test.o
$(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches

test_osm_store: \
test/osm_store.test.o
$(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store
Expand Down Expand Up @@ -262,6 +273,22 @@ test_sorted_way_store: \
test/sorted_way_store.test.o
$(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store

test_shp_mem_tiles: \
src/attribute_store.o \
src/coordinates.o \
src/coordinates_geom.o \
src/declutter.o \
src/geom.o \
src/mmap_allocator.o \
src/output_object.o \
src/pooled_string.o \
src/shp_mem_tiles.o \
src/tile_coordinates_set.o \
src/tile_data.o \
src/tile_sorting.o \
test/shp_mem_tiles.test.o
$(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles

test_tile_coordinates_set: \
src/tile_coordinates_set.o \
test/tile_coordinates_set.test.o
Expand Down
4 changes: 4 additions & 0 deletions include/append_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ namespace AppendVectorNS {
return el;
}

const T& operator [](int idx) const {
return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE];
}

Iterator begin() {
return Iterator(*this, 0, 0);
}
Expand Down
27 changes: 26 additions & 1 deletion include/attribute_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#include <atomic>
#include <boost/functional/hash.hpp>
#include <boost/container/flat_map.hpp>
#include <boost/container/small_vector.hpp>
#include <algorithm>
#include <vector>
#include <protozero/data_view.hpp>
#include "pooled_string.h"
Expand All @@ -30,11 +32,16 @@ class AttributeKeyStore {
uint16_t key2index(const std::string& key);
const std::string& getKey(uint16_t index) const;
const std::string& getKeyUnsafe(uint16_t index) const;
void finalize() { finalized = true; }
// Position of the key in alphabetical order. Unlike the key index, which is
// handed out in first-use order (i.e. depends on thread scheduling), it is
// the same in every run. Only valid after finalize().
uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; }
void finalize();
std::atomic<uint32_t> keys2indexSize;

private:
bool finalized;
std::vector<uint16_t> ranks;
mutable std::mutex keys2indexMutex;
// NB: we use a deque, not a vector, because a deque never invalidates
// pointers to its members as long as you only push_back
Expand Down Expand Up @@ -439,6 +446,24 @@ struct AttributeStore {
}
attrSet->forEachPair(fn);
}
// As forEachPairIndexUnsafe, but in an order that is the same in every run
// (by key name), not by pair index, which depends on thread scheduling.
template<class F>
void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const {
boost::container::small_vector<uint32_t, 16> order;
forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); });
if (order.size() > 1)
std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) {
return comparePairsUnsafe(a, b) < 0;
});
for (const uint32_t pairIndex : order)
fn(pairIndex);
}
// Order and hash attribute pairs/sets by content (key names and values), not by
// index, so that they are the same in every run. Only valid after finalize().
int comparePairsUnsafe(uint32_t a, uint32_t b) const;
int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const;
uint64_t contentHashUnsafe(AttributeIndex index) const;
void reset(); // used for testing
size_t size() const;
void reportSize() const;
Expand Down
7 changes: 6 additions & 1 deletion include/clip_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ class TileBbox;
extern thread_local uint64_t clipCacheBatchId;
// Set when the current batch can't reuse clips (a single tile), so add() is skipped.
extern thread_local bool clipCacheNoStore;
// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces):
// the batch cache is then neither read nor written.
extern thread_local bool clipCacheBypass;

template <class T>
class ClipCache {
Expand All @@ -36,6 +39,8 @@ class ClipCache {
}

Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const {
if (clipCacheBypass)
return Entry();
// Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first)
LocalStore& s = local(zoom, TileCoordinates(x, y));
Entry rv;
Expand All @@ -59,7 +64,7 @@ class ClipCache {
void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) {
// The point of caching is to reuse the clip, so caching at the terminal zoom is
// pointless.
if (bbox.zoom == baseZoom || clipCacheNoStore)
if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass)
return;

// Levels beyond MAX_LEVELS could never be looked up by a child tile anyway
Expand Down
1 change: 1 addition & 0 deletions include/coordinates_geom.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class TileBbox {
std::pair<int,int> scaleLatpLon(double latp, double lon) const;
void scaleRing(Ring &dst, Ring const &src) const;
Ring scaleRing(Ring const &src) const;
void scaleRingNoBacktrack(Ring &dst, Ring const &src) const;
void scaleGeometry(MultiPolygon &dst, MultiPolygon const &src) const;
MultiPolygon scaleGeometry(MultiPolygon const &src) const;
std::pair<double, double> floorLatpLon(double latp, double lon) const;
Expand Down
92 changes: 62 additions & 30 deletions include/geometry/correct.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
* ----------------------------------------------------------------------------
*/

#include <functional>
#include <unordered_map>
#include <utility>
#include <vector>
#include <boost/geometry.hpp>
Expand All @@ -23,13 +25,18 @@ namespace geometry {

namespace impl {

template<typename C, typename T>
static inline void result_combine(C &result, T &&new_element)
// boxes[i] is the envelope of result[i]. Polygons whose envelopes are disjoint
// cannot intersect, so the box test skips most of the costly intersects() calls
// (otherwise quadratic in the number of rings, e.g. a lake with 14000 islands).
template<typename C, typename B, typename T>
static inline void result_combine(C &result, std::vector<B> &boxes, T &&new_element)
{
result.push_back(std::forward<T>(new_element));
boxes.push_back(boost::geometry::return_envelope<B>(result.back()));

for(std::size_t i = 0; i < result.size() - 1; ) {
if(!boost::geometry::intersects(result[i], result.back())) {
if(boost::geometry::disjoint(boxes[i], boxes.back()) ||
!boost::geometry::intersects(result[i], result.back())) {
++i;
continue;
}
Expand All @@ -43,8 +50,10 @@ static inline void result_combine(C &result, T &&new_element)
}

result.back() = std::move(union_result[0]);
boxes.back() = boost::geometry::return_envelope<B>(result.back());
result.erase(result.begin() + i);
}
boxes.erase(boxes.begin() + i);
}
}

struct pseudo_vertice_key
Expand Down Expand Up @@ -199,27 +208,40 @@ static inline std::vector<ring_t> dissolve_generate_rings(

// Generate all polygons by tracing all the intersections
// Perform union to combine all polygons into single polygon again
while(!start_keys.empty()) {
ring_t new_ring;

// Store point in generated polygon
auto push_point = [&new_ring](auto const &p) {
if(new_ring.empty() || boost::geometry::comparable_distance(new_ring.back(), p) > 0)
new_ring.push_back(p);
};
// Position of each point in the ring being traced. Its points are distinct until
// it closes, so a point already seen closes it: looking that up here instead of
// scanning the whole ring for every new point keeps tracing linear, not quadratic.
struct point_hash {
std::size_t operator()(std::pair<double, double> const &p) const {
return std::hash<double>()(p.first) * 31 + std::hash<double>()(p.second);
}
};
std::unordered_map<std::pair<double, double>, std::size_t, point_hash> seen;

auto is_closed = [](ring_t &ring) {
if(ring.size() < 2) return false;
for(std::size_t i = 0; i < ring.size() - 1; ++i) {
if(boost::geometry::comparable_distance(ring[i], ring.back()) == 0) {
ring.erase(ring.begin(), ring.begin() + i);
return true;
}
while(!start_keys.empty()) {
ring_t new_ring;
seen.clear();
bool closed = false;

// Store point in generated polygon; when it repeats an earlier point, the
// ring is closed and the points before that one are dropped
auto push_point = [&new_ring, &seen, &closed](auto const &p) {
if(!new_ring.empty() && !(boost::geometry::comparable_distance(new_ring.back(), p) > 0))
return;
new_ring.push_back(p);
// + 0.0 turns -0.0 into 0.0, which compares equal to it
std::pair<double, double> key(boost::geometry::get<0>(p) + 0.0, boost::geometry::get<1>(p) + 0.0);
auto found = seen.find(key);
if(found == seen.end()) {
seen.emplace(key, new_ring.size() - 1);
} else {
new_ring.erase(new_ring.begin(), new_ring.begin() + found->second);
closed = true;
}

return false;
};

auto is_closed = [&closed](ring_t &) { return closed; };

auto start_iter = pseudo_vertices.find(*start_keys.begin());
auto i = start_iter;

Expand Down Expand Up @@ -302,13 +324,15 @@ template<
>
struct combine_non_zero_winding
{
inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly)
using box_t = boost::geometry::model::box<point_t>;

inline void operator()(multi_polygon_t &combined_outers, std::vector<box_t> &outer_boxes, multi_polygon_t &combined_inners, std::vector<box_t> &inner_boxes, polygon_t &poly)
{
if(boost::geometry::area(poly) > 0)
result_combine(combined_outers, std::move(poly));
result_combine(combined_outers, outer_boxes, std::move(poly));
else {
std::reverse(poly.outer().begin(), poly.outer().end());
result_combine(combined_inners, std::move(poly));
result_combine(combined_inners, inner_boxes, std::move(poly));
}
}
};
Expand All @@ -320,7 +344,9 @@ template<
>
struct combine_odd_even
{
inline void operator()(multi_polygon_t &combined_outers, multi_polygon_t &combined_inners, polygon_t &poly)
using box_t = boost::geometry::model::box<point_t>;

inline void operator()(multi_polygon_t &combined_outers, std::vector<box_t> &, multi_polygon_t &combined_inners, std::vector<box_t> &, polygon_t &poly)
{
if(boost::geometry::area(poly) < 0)
std::reverse(poly.outer().begin(), poly.outer().end());
Expand All @@ -346,11 +372,12 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub
// Calculate all outers and combine them if possible
multi_polygon_t combined_outers;
multi_polygon_t combined_inners;
std::vector<boost::geometry::model::box<point_t>> outer_boxes, inner_boxes;

for(auto &ring: outer_rings) {
polygon_t poly;
poly.outer() = std::move(ring);
combine(combined_outers, combined_inners, poly);
combine(combined_outers, outer_boxes, combined_inners, inner_boxes, poly);
}

// Calculate all inners and combine them if possible
Expand All @@ -362,8 +389,8 @@ static inline void correct(polygon_t const &input, multi_polygon_t &output, doub
correct(poly, new_inners, remove_spike_min_area, combine);

for(auto &poly: new_inners) {
result_combine(combined_inners, std::move(poly));
}
result_combine(combined_inners, inner_boxes, std::move(poly));
}
}

// Cut out all inners from all the outers
Expand All @@ -379,13 +406,18 @@ template<
>
static inline void correct(multi_polygon_t const &input, multi_polygon_t &output, double remove_spike_min_area, combine_function_t combine)
{
using box_t = boost::geometry::model::box<point_t>;
std::vector<box_t> output_boxes;
for(auto const &polygon: output)
output_boxes.push_back(boost::geometry::return_envelope<box_t>(polygon));

for(auto const &polygon: input)
{
multi_polygon_t new_polygons;
correct(polygon, new_polygons, remove_spike_min_area, combine);

for(auto &new_polygon: new_polygons)
result_combine(output, std::move(new_polygon));
for(auto &new_polygon: new_polygons)
result_combine(output, output_boxes, std::move(new_polygon));
}
}

Expand Down
16 changes: 10 additions & 6 deletions include/leased_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#define _LEASED_STORE_H

#include "tile_data.h"
#include <tuple>

// LeasedStore implements "leases" for generated geometries.
//
Expand Down Expand Up @@ -45,23 +46,26 @@ inline std::vector<std::pair<size_t, TileDataSource::multi_polygon_store_t*>>& g

template <class T>
class LeasedStore {
std::vector<std::pair<TileDataSource*, std::pair<size_t, T*>>> leases;
std::vector<std::tuple<TileDataSource*, uint64_t, std::pair<size_t, T*>>> leases; // source, its leaseNumber, lease

public:
~LeasedStore() {
// Only to the sources still alive (see registerLeaseSource)
std::lock_guard<std::mutex> live(leaseSourcesMutex());
for (const auto& lease : leases) {
auto source = lease.first;
auto source = std::get<0>(lease);
if (!leaseSourceAlive(source, std::get<1>(lease))) continue;
std::lock_guard<std::mutex> lock(source->storeMutex);

std::vector<std::pair<size_t, T*>>& availableLeases = getAvailableLeases<T>(source);
availableLeases.push_back(lease.second);
availableLeases.push_back(std::get<2>(lease));
}
}

std::pair<size_t, T*> get(TileDataSource* source) {
for (const auto& lease : leases) {
if (lease.first == source)
return lease.second;
if (std::get<0>(lease) == source && std::get<1>(lease) == source->leaseNumber)
return std::get<2>(lease);
}

std::lock_guard<std::mutex> lock(source->storeMutex);
Expand All @@ -74,7 +78,7 @@ class LeasedStore {
std::pair<size_t, T*> entry = availableLeases.back();
availableLeases.pop_back();

leases.push_back(std::make_pair(source, entry));
leases.push_back(std::make_tuple(source, source->leaseNumber, entry));
return entry;
}

Expand Down
Loading
Loading