Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions include/mbtiles.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#define _MBTILES_H

#include <string>
#include <atomic>
#include <mutex>
#include <vector>
#include "external/sqlite_modern_cpp.h"
Expand All @@ -27,6 +28,7 @@ class MBTiles {

std::shared_ptr<std::vector<PendingStatement>> pendingStatements1, pendingStatements2;
std::mutex pendingStatementsMutex;
std::atomic<size_t> pendingBytes;

void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge);
void flushPendingStatements();
Expand Down
28 changes: 25 additions & 3 deletions src/mbtiles.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,18 @@ using namespace std;

MBTiles::MBTiles():
pendingStatements1(std::make_shared<std::vector<PendingStatement>>()),
pendingStatements2(std::make_shared<std::vector<PendingStatement>>())
pendingStatements2(std::make_shared<std::vector<PendingStatement>>()),
pendingBytes(0)
{}

// At most this much is buffered while SQLite is busy; beyond it, workers wait
// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB).
static const size_t MAX_PENDING_BYTES = 256 << 20;

static size_t pendingCost(const std::string& data) {
return sizeof(PendingStatement) + data.size();
}

MBTiles::~MBTiles() {
if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open
}
Expand Down Expand Up @@ -71,6 +80,7 @@ void MBTiles::flushPendingStatements() {
while(!pendingStatements2->empty()) {
const PendingStatement& stmt = pendingStatements2->back();
insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge);
pendingBytes -= pendingCost(stmt.data);
pendingStatements2->pop_back();
}

Expand All @@ -85,11 +95,23 @@ void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) {
insertOrReplace(zoom, x, y, *data, isMerge);
flushPendingStatements();
m.unlock();
} else {
return;
}

{
// Else buffer the write for later, copying its binary blob.
const std::lock_guard<std::mutex> lock(pendingStatementsMutex);
pendingStatements1->push_back({zoom, x, y, *data, isMerge});
if (pendingBytes < MAX_PENDING_BYTES) {
pendingBytes += pendingCost(*data);
pendingStatements1->push_back({zoom, x, y, *data, isMerge});
return;
}
}

// The buffer is full: wait for SQLite, then help drain the buffer.
const std::lock_guard<std::mutex> lock(m);
insertOrReplace(zoom, x, y, *data, isMerge);
flushPendingStatements();
}

void MBTiles::closeForWriting() {
Expand Down
110 changes: 61 additions & 49 deletions src/tilemaker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -504,8 +504,13 @@ int main(const int argc, const char* argv[]) {

std::atomic<uint64_t> batchCounter(0);
const uint cutZoom = std::max<uint>(12, config.startZoom);
std::size_t batchSize = 0;
for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) {
// Batches are handed out as threads become free rather than all posted up
// front: a planet has ~14M of them, and their queued handlers held GBs.
std::mutex batchMutex;
std::size_t nextBatchStart = 0;
auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize) {
const std::lock_guard<std::mutex> lock(batchMutex);
startIndex = nextBatchStart;
// Compute how many tiles should be assigned to this batch --
// higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive.
batchSize = 0;
Expand All @@ -529,65 +534,72 @@ int main(const int argc, const char* argv[]) {

batchSize++;
}
nextBatchStart += batchSize;
return batchSize > 0;
};

boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() {
clipCacheBatchId = ++batchCounter;
// A single-tile batch has no descendants to reuse its clips.
clipCacheNoStore = batchSize == 1;
std::vector<std::string> tileTimings;
std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize);
for(std::size_t i = startIndex; i < endIndex; ++i) {
unsigned int zoom = tileCoordinates[i].first;
TileCoordinates coords = tileCoordinates[i].second;
for (unsigned int t = 0; t < options.threadNum; t++) {
boost::asio::post(pool, [&]() {
std::size_t startIndex, batchSize;
while (nextBatch(startIndex, batchSize)) {
clipCacheBatchId = ++batchCounter;
// A single-tile batch has no descendants to reuse its clips.
clipCacheNoStore = batchSize == 1;
std::vector<std::string> tileTimings;
std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize);
for(std::size_t i = startIndex; i < endIndex; ++i) {
unsigned int zoom = tileCoordinates[i].first;
TileCoordinates coords = tileCoordinates[i].second;

#ifdef CLOCK_MONOTONIC
timespec start, end;
if (options.logTileTimings)
clock_gettime(CLOCK_MONOTONIC, &start);
timespec start, end;
if (options.logTileTimings)
clock_gettime(CLOCK_MONOTONIC, &start);
#endif

std::vector<std::vector<OutputObjectID>> data;
for (auto source : sources) {
data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords));
}
outputProc(sharedData, sources, attributeStore, data, coords, zoom);
std::vector<std::vector<OutputObjectID>> data;
for (auto source : sources) {
data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords));
}
outputProc(sharedData, sources, attributeStore, data, coords, zoom);

#ifdef CLOCK_MONOTONIC
if (options.logTileTimings) {
clock_gettime(CLOCK_MONOTONIC, &end);
uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec;
std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms";
tileTimings.push_back(output);
}
if (options.logTileTimings) {
clock_gettime(CLOCK_MONOTONIC, &end);
uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec;
std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms";
tileTimings.push_back(output);
}
#endif
}
}

if (options.logTileTimings) {
const std::lock_guard<std::mutex> lock(io_mutex);
std::cout << std::endl;
for (const auto& output : tileTimings)
std::cout << output << std::endl;
}
if (options.logTileTimings) {
const std::lock_guard<std::mutex> lock(io_mutex);
std::cout << std::endl;
for (const auto& output : tileTimings)
std::cout << output << std::endl;
}

tilesWritten += (endIndex - startIndex);

if (io_mutex.try_lock()) {
uint64_t written = tilesWritten.load();

if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) {
lastTilesWritten = written;
// Show progress grouped by z6 (or lower)
size_t z = tileCoordinates[startIndex].first;
size_t x = tileCoordinates[startIndex].second.x;
size_t y = tileCoordinates[startIndex].second.y;
if (z > CLUSTER_ZOOM) {
x = x / (1 << (z - CLUSTER_ZOOM));
y = y / (1 << (z - CLUSTER_ZOOM));
z = CLUSTER_ZOOM;
tilesWritten += (endIndex - startIndex);

if (io_mutex.try_lock()) {
uint64_t written = tilesWritten.load();

if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) {
lastTilesWritten = written;
// Show progress grouped by z6 (or lower)
size_t z = tileCoordinates[startIndex].first;
size_t x = tileCoordinates[startIndex].second.x;
size_t y = tileCoordinates[startIndex].second.y;
if (z > CLUSTER_ZOOM) {
x = x / (1 << (z - CLUSTER_ZOOM));
y = y / (1 << (z - CLUSTER_ZOOM));
z = CLUSTER_ZOOM;
}
cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush;
}
cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush;
io_mutex.unlock();
}
io_mutex.unlock();
}
});
}
Expand Down
Loading