Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
36c9466
test: master + all open performance PRs + #981 + #942 + #983 (combine…
dktue Nov 7, 2026
a5c518c
test: add #986 (sort the tile list with at most 16 threads)
dktue Nov 7, 2026
154852c
perf: wake threads waiting for room only once there is room
dktue Nov 8, 2026
89208f3
perf: clip large polygons once per z8 tile and share the piece
dktue Nov 8, 2026
528bb99
perf(pmtiles): place and write tiles in two ordered steps
dktue Nov 8, 2026
d019a90
perf: let the tile threads compute the shared pieces
dktue Nov 8, 2026
697a6ff
build: fix Windows and macOS builds of the combined base
dktue Nov 15, 2026
4c0023f
fix: keep the allocator's region lists alive until thread-local destr…
dktue Nov 15, 2026
3f55964
perf: finalize the stores and close PMTiles on all cores (#989 + #990)
dktue Nov 15, 2026
463c11b
perf(lua): replay way_function/node_function from a memo; native Merc…
dktue Nov 15, 2026
1c14ba2
perf: faster reading phase (PBF decode, store lookups, relations, ins…
dktue Nov 16, 2026
3222454
perf: decide polygon validity without Boost where certain; faster mak…
dktue Nov 16, 2026
7d94aaf
perf: faster clipping, simplification and labels; reuse covered subtrees
dktue Nov 16, 2026
5db562e
perf: faster MVT encoding and per-tile sorting
dktue Nov 16, 2026
bc4c31b
perf: schedule the tile phase better and build the tile list in parallel
dktue Nov 16, 2026
7742db0
perf(pmtiles): direct I/O tile writer, parallel close; compression_level
dktue Nov 16, 2026
b6f49ad
perf: return as soon as the output is written; huge pages for the stores
dktue Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ else ifeq ($(TM_GIT_TAG)$(TM_GIT_DIRTY),$(TM_BASE_VERSION))
else
TM_VERSION ?= $(TM_BASE_VERSION)+g$(TM_GIT_SHA)$(TM_GIT_DIRTY)
endif
CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++14 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
CXXFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c++17 -pthread -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
CFLAGS ?= -O3 -Wall -Wno-unknown-pragmas -Wno-sign-compare -std=c99 -fPIE -DTM_VERSION=$(TM_VERSION) $(CONFIG)
DEPFLAGS := -MD -MP
DEPS := $(wildcard src/*.d src/external/*.d src/external/libdeflate/lib/*.d src/external/libdeflate/lib/*/*.d server/*.d test/*.d)
Expand Down Expand Up @@ -156,13 +156,16 @@ test: \
test_append_vector \
test_attribute_store \
test_config_validator \
test_correct \
test_deque_map \
test_geom \
test_helpers \
test_options_parser \
test_ordered_batches \
test_pbf_reader \
test_pooled_string \
test_relation_roles \
test_shp_mem_tiles \
test_significant_tags \
test_sorted_node_store \
test_sorted_way_store \
Expand Down Expand Up @@ -193,6 +196,11 @@ src/config_schema.h: resources/config-schema.json

src/config_validator.o: src/config_schema.h

test_correct: \
src/geom.o \
test/correct.test.o
$(CXX) $(CXXFLAGS) -o test.correct $^ $(INC) $(LIB) $(LDFLAGS) && ./test.correct

test_deque_map: \
test/deque_map.test.o
$(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map
Expand Down Expand Up @@ -223,6 +231,10 @@ test_options_parser: \
test/options_parser.test.o
$(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser

test_ordered_batches: \
test/ordered_batches.test.o
$(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches

test_osm_store: \
test/osm_store.test.o
$(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store
Expand Down Expand Up @@ -262,6 +274,22 @@ test_sorted_way_store: \
test/sorted_way_store.test.o
$(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store

test_shp_mem_tiles: \
src/attribute_store.o \
src/coordinates.o \
src/coordinates_geom.o \
src/declutter.o \
src/geom.o \
src/mmap_allocator.o \
src/output_object.o \
src/pooled_string.o \
src/shp_mem_tiles.o \
src/tile_coordinates_set.o \
src/tile_data.o \
src/tile_sorting.o \
test/shp_mem_tiles.test.o
$(CXX) $(CXXFLAGS) -o test.shp_mem_tiles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.shp_mem_tiles

test_tile_coordinates_set: \
src/tile_coordinates_set.o \
test/tile_coordinates_set.test.o
Expand Down
16 changes: 16 additions & 0 deletions docs/CONFIGURATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ It also includes these global settings:
* `basezoom` - the zoom level for which tilemaker will generate tiles internally (should usually be the same as `maxzoom`)
* `include_ids` - whether you want to store the OpenStreetMap IDs for each way/node within your vector tiles. This option is not compatible with the merging options defined by the `combine_xxx` settings (see the dedicated paragraph below)
* `compress` - for mbtiles output, whether to compress vector tiles (Any of "gzip","deflate" or "none"(default)). For pmtiles output, compression is hardcoded to gzip
* `compression_level` (optional) - how hard to compress the tiles, from 1 (fastest) to 12 (smallest); defaults to 6. On the planet with the OpenMapTiles profile, 1 gave a 1.8% larger file and a 3% shorter run
* `name`, `version` and `description` - about your project (these are written into the MBTiles file)
* `high_resolution` (optional) - whether to use extra coordinate precision at the maximum zoom level (makes tiles a bit bigger)
* `bounding_box` (optional) - the bounding box to output, in [minlon, minlat, maxlon, maxlat] order
Expand Down Expand Up @@ -210,6 +211,8 @@ To do that, you use these methods:
* `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
* `Score(number)`: set how important this point feature is (a whole number; anything else is rounded down), so that tilemaker can decide which zoom level to show it from. Only meaningful for points (i.e. features written with `Layer` for a node, or `LayerAsCentroid`) in a layer with `declutter_below` set - see 'Decluttering point features' below.
* `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires Boost 1.67+.
* `MercatorArea()`: return the area of the current object on the Web Mercator plane, in square metres at the scale of the equator (so it grows towards the poles, as the map does). It is cheap to compute and tells how much of a tile the object covers, which makes it the right measure for minimum zoom rules.
* `MinZoomByArea(limit, thresholds)`: set the minimum zoom of the current object from its `MercatorArea()`: the first zoom z from `limit` on whose threshold the area exceeds, where `thresholds` lists the thresholds for zoom 1, 2, 3...; if there is none, one more than the number of thresholds. Unlike the same rule written in Lua with `MercatorArea()`, this keeps `way_function` replayable (see `way_memo` above).
* `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).

The simplest possible function, to include roads/paths and nothing else, might look like this:
Expand Down Expand Up @@ -238,6 +241,19 @@ If your Lua file causes an error due to mistaken syntax, you can test it at the

`way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}`

`way_memo = true` lets tilemaker replay `way_function` instead of running it, for ways that read the same as an earlier way. This is only allowed if `way_function` is a function of what it reads with `Find`, `Holds`, `IsClosed` and `NextRelation` (no other state, such as global counters), and what it does is call tilemaker's functions (`Layer`, `Attribute`... `MinZoom`): tilemaker records which reads lead to which calls, and replays the calls for a way whose reads give the same results. Anything else (geometry queries such as `Area`, relation tags, `Id`...) makes ways with those reads run through Lua. A tag whose value is only tested for being empty and copied unchanged into some attributes can be declared in `way_memo_values`, e.g. `way_memo_values = { name = { "name:latin" } }`: ways are then replayed with their own value of the tag, instead of being told apart by it. An attribute that is set to the length of the value (in bytes, as Lua's `#value` or `value:len()`) is declared as `attribute = "length"`, e.g. `way_memo_values = { ref = { "ref", ref_length = "length" } }`.

Two more declarations let the memo replay ways that use some of the other functions:

* `way_memo_area = { 1000, 5000 }`: `way_function` uses `MercatorArea()` only to compare it (`<`, `<=`, `>`, `>=`, `==`, `~=`) with these numbers. The memo then tells ways apart by the results of these comparisons, not by the area (e.g. `if MercatorArea() < 1000 then MinZoom(8) else MinZoomByArea(0, thresholds) end`; `MinZoomByArea` needs no declaration, it is worked out for each way).
* `way_memo_relations = true`: `way_function` uses the relation id `NextRelation()` returns only to test whether there is one (`if not rel then break end`), and reads the relations' tags only with `FindInRelation` (no `RestartRelations`). The memo then tells ways apart by the roles and the relation tags read.

As with `way_memo_values`, tilemaker can't check these promises: if `way_function` does more with the area or the relations, ways are replayed with the wrong calls.

Multipolygon relations, which also run `way_function`, are replayed the same way (with a record of their own, apart from that of the ways): the promises above are about `way_function`, so they hold for them too. For a multipolygon, `NextRelation` and `FindInRelation` are about the relations it is a member of.

`node_memo = true` and `node_memo_values`, `node_memo_area`, `node_memo_relations` do the same for `node_function` (with its own record).

`init_function(name, is_first)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. `is_first` will be true only the first time `init_function` is called.

Other functions are described below and in RELATIONS.md.
Expand Down
20 changes: 20 additions & 0 deletions docs/RUNNING.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,26 @@ usage but runs faster.
* `--shard-stores`: Group temporary storage by area. Reduces RAM usage on large files (e.g.
whole planet) but runs slower.

On Linux, .pmtiles output is written with direct I/O (`O_DIRECT`), bypassing the page cache, which
is considerably faster on a fast disk. Where the file system doesn't support it, tilemaker
falls back to normal writes; setting the environment variable `TM_NO_DIRECT` forces normal writes.

A few more things make large runs faster on Linux:

* Once the output is complete, tilemaker returns at once and leaves freeing its memory to the
system in the background (after a planet run that takes several seconds). Use `--no-early-exit`
to make tilemaker wait until its memory is freed.
* tilemaker asks the kernel for transparent huge pages for its node and way stores, which cuts the
time spent on page faults. This only has an effect when `/sys/kernel/mm/transparent_hugepage/enabled`
is `madvise` or `always`. Set the environment variable `TM_NO_HUGEPAGES` to turn it off.
* With many threads, the jemalloc allocator is faster than glibc's, especially with huge pages for
its own memory too. It needs no rebuild: install it (e.g. `apt install libjemalloc2`) and run

LD_PRELOAD=libjemalloc.so.2 MALLOC_CONF=thp:always,metadata_thp:always tilemaker ...

On the planet with 96 threads, jemalloc made the run about 6% faster, and its huge-page settings
another 2%.

You can also tell tilemaker to only look at .pbf objects with certain tags. If you're making a
thematic map, this allows tilemaker to skip data it won't need. Specify this in your Lua file
like one of these three examples:
Expand Down
21 changes: 21 additions & 0 deletions include/append_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#define _APPEND_VECTOR_H

#include "mmap_allocator.h"
#include <algorithm>
#include <vector>
#include <queue>
#include <cstdint>
Expand Down Expand Up @@ -162,6 +163,10 @@ namespace AppendVectorNS {
return el;
}

const T& operator [](int idx) const {
return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE];
}

Iterator begin() {
return Iterator(*this, 0, 0);
}
Expand All @@ -184,6 +189,22 @@ namespace AppendVectorNS {
count++;
}

// The same as push_back of each of p[0, n)
void append(const T* p, size_t n) {
while (n > 0) {
auto& vec = vecs.back();
if (vec.capacity() == 0)
vec.reserve(APPEND_VECTOR_SIZE);
const size_t take = std::min(n, vec.capacity() - vec.size());
vec.insert(vec.end(), p, p + take);
p += take; n -= take; count += take;
if (vec.size() == vec.capacity()) {
vecs.push_back(std::vector<T, mmap_allocator<T>>());
vecs.back().reserve(APPEND_VECTOR_SIZE);
}
}
}

size_t count;
std::deque<std::vector<T, mmap_allocator<T>>> vecs;
};
Expand Down
58 changes: 55 additions & 3 deletions include/attribute_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#include <atomic>
#include <boost/functional/hash.hpp>
#include <boost/container/flat_map.hpp>
#include <boost/container/small_vector.hpp>
#include <algorithm>
#include <cstring>
#include <vector>
#include <protozero/data_view.hpp>
#include "pooled_string.h"
Expand All @@ -30,11 +33,16 @@ class AttributeKeyStore {
uint16_t key2index(const std::string& key);
const std::string& getKey(uint16_t index) const;
const std::string& getKeyUnsafe(uint16_t index) const;
void finalize() { finalized = true; }
// Position of the key in alphabetical order. Unlike the key index, which is
// handed out in first-use order (i.e. depends on thread scheduling), it is
// the same in every run. Only valid after finalize().
uint16_t getRankUnsafe(uint16_t index) const { return ranks[index]; }
void finalize();
std::atomic<uint32_t> keys2indexSize;

private:
bool finalized;
std::vector<uint16_t> ranks;
mutable std::mutex keys2indexMutex;
// NB: we use a deque, not a vector, because a deque never invalidates
// pointers to its members as long as you only push_back
Expand Down Expand Up @@ -152,14 +160,29 @@ struct AttributePair {
return true;
}

// Hashes the string 8 bytes at a time (the hash only picks shards and hash table slots)
static uint64_t hashBytes(uint64_t h, const char* p, size_t n) {
h ^= n * 0x9E3779B97F4A7C15ULL;
while (n >= 8) {
uint64_t w; memcpy(&w, p, 8);
h = (h ^ w) * 0xff51afd7ed558ccdULL; h ^= h >> 32;
p += 8; n -= 8;
}
if (n > 0) {
uint64_t w = 0; memcpy(&w, p, n);
h = (h ^ w) * 0xc4ceb9fe1a85ec53ULL; h ^= h >> 29;
}
h *= 0x94d049bb133111ebULL; h ^= h >> 31;
return h;
}

size_t hash() const {
std::size_t rv = minzoom;
boost::hash_combine(rv, keyIndex);
boost::hash_combine(rv, valueType);

if(hasStringValue()) {
const char* data = pooledString().data();
boost::hash_range(rv, data, data + pooledString().size());
rv = hashBytes(rv, pooledString().data(), pooledString().size());
} else if(hasFloatValue())
boost::hash_combine(rv, doubleValue_);
else if(hasIntValue())
Expand Down Expand Up @@ -216,6 +239,9 @@ class AttributePairStore {
friend class AttributeStore;
std::vector<DequeMap<AttributePair>> pairs;
bool finalized;
// Set (under its lock) once the hot shard is full: from then on it never changes, so it is
// looked up without the lock
std::atomic<bool> hotFull{false};
// We refer to all attribute pairs by index.
//
// Each shard is responsible for a portion of the key space.
Expand Down Expand Up @@ -439,6 +465,24 @@ struct AttributeStore {
}
attrSet->forEachPair(fn);
}
// As forEachPairIndexUnsafe, but in an order that is the same in every run
// (by key name), not by pair index, which depends on thread scheduling.
template<class F>
void forEachPairIndexCanonicalUnsafe(AttributeIndex index, F&& fn) const {
boost::container::small_vector<uint32_t, 16> order;
forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) { order.push_back(pairIndex); });
if (order.size() > 1)
std::sort(order.begin(), order.end(), [this](uint32_t a, uint32_t b) {
return comparePairsUnsafe(a, b) < 0;
});
for (const uint32_t pairIndex : order)
fn(pairIndex);
}
// Order and hash attribute pairs/sets by content (key names and values), not by
// index, so that they are the same in every run. Only valid after finalize().
int comparePairsUnsafe(uint32_t a, uint32_t b) const;
int compareContentUnsafe(AttributeIndex a, AttributeIndex b) const;
uint64_t contentHashUnsafe(AttributeIndex index) const;
void reset(); // used for testing
size_t size() const;
void reportSize() const;
Expand All @@ -453,6 +497,14 @@ struct AttributeStore {
void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom);
void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom);
void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom);
// The pair index that addAttributeWithKeyIndex adds to the set. It is the same for the same
// arguments for as long as the store lives, so callers may keep it.
uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom);
uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, double v, char minzoom);
uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, int v, char minzoom);
uint32_t pairIndexWithKeyIndex(uint16_t keyIndex, bool v, char minzoom);
// Counts an object whose AttributeIndex the caller knew without add() (for reportSize)
void noteLookup();

AttributeStore():
finalized(false),
Expand Down
7 changes: 6 additions & 1 deletion include/clip_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ class TileBbox;
extern thread_local uint64_t clipCacheBatchId;
// Set when the current batch can't reuse clips (a single tile), so add() is skipped.
extern thread_local bool clipCacheNoStore;
// Set while a shared piece of a large polygon is computed (TileDataSource::precomputeSharedPieces):
// the batch cache is then neither read nor written.
extern thread_local bool clipCacheBypass;

template <class T>
class ClipCache {
Expand All @@ -36,6 +39,8 @@ class ClipCache {
}

Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const {
if (clipCacheBypass)
return Entry();
// Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first)
LocalStore& s = local(zoom, TileCoordinates(x, y));
Entry rv;
Expand All @@ -59,7 +64,7 @@ class ClipCache {
void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) {
// The point of caching is to reuse the clip, so caching at the terminal zoom is
// pointless.
if (bbox.zoom == baseZoom || clipCacheNoStore)
if (bbox.zoom == baseZoom || clipCacheNoStore || clipCacheBypass)
return;

// Levels beyond MAX_LEVELS could never be looked up by a child tile anyway
Expand Down
Loading
Loading