diff --git a/Makefile b/Makefile index 57227d4c..69c261bb 100644 --- a/Makefile +++ b/Makefile @@ -160,6 +160,7 @@ test: \ test_geom \ test_helpers \ test_options_parser \ + test_ordered_batches \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ @@ -223,6 +224,10 @@ test_options_parser: \ test/options_parser.test.o $(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser +test_ordered_batches: \ + test/ordered_batches.test.o + $(CXX) $(CXXFLAGS) -o test.ordered_batches $^ $(INC) $(LIB) $(LDFLAGS) && ./test.ordered_batches + test_osm_store: \ test/osm_store.test.o $(CXX) $(CXXFLAGS) -o test.osm_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.osm_store diff --git a/include/mbtiles.h b/include/mbtiles.h index ca671b07..8181fafa 100644 --- a/include/mbtiles.h +++ b/include/mbtiles.h @@ -3,9 +3,11 @@ #define _MBTILES_H #include +#include #include #include #include "external/sqlite_modern_cpp.h" +#include "ordered_batches.h" struct PendingStatement { int zoom; @@ -27,6 +29,7 @@ class MBTiles { std::shared_ptr> pendingStatements1, pendingStatements2; std::mutex pendingStatementsMutex; + std::atomic pendingBytes; void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge); void flushPendingStatements(); @@ -39,6 +42,9 @@ class MBTiles { void saveTile(int zoom, int x, int y, std::string *data, bool isMerge); void closeForWriting(); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + void openForReading(std::string &filename); void readBoundingBox(double &minLon, double &maxLon, double &minLat, double &maxLat); void readTileList(std::vector> &tileList); diff --git a/include/ordered_batches.h b/include/ordered_batches.h new file mode 100644 index 00000000..9531565f --- /dev/null +++ b/include/ordered_batches.h @@ -0,0 +1,157 @@ +/*! \file */ +#ifndef _ORDERED_BATCHES_H +#define _ORDERED_BATCHES_H + +#include +#include +#include +#include +#include +#include +#include +#include + +// Writes batches of output in the order they were handed out, not the order threads +// finish them, so the output file doesn't depend on thread scheduling. +// +// A thread collects a batch's items between begin() and end(). end() writes the batch +// if it's the next one, followed by any batches that were waiting for it; otherwise it +// holds the batch. Batches must be numbered 0, 1, 2, ... in the order they're handed out, +// and a thread calls waitForRoom() before it takes its next batch: that keeps what is +// held below the limit set with setMaxHeldBytes(). The batch everyone waits for is always +// being worked on by a thread that isn't waiting, so waitForRoom() always returns. +// +// writeItem is called by one thread at a time, with writeMutex() held, in batch order. +// If flushItem is given, a batch then goes through flushItem as well: also one thread at a time +// and in batch order, but without holding up writeItem. Whichever thread is free does it, so the +// two steps run side by side. A batch counts as held until it has been flushed. +template +class OrderedBatches { +public: + OrderedBatches(std::function writeItem, std::function itemBytes, + std::function flushItem = nullptr) + : writeItem(writeItem), itemBytes(itemBytes), flushItem(flushItem) { } + + void setMaxHeldBytes(uint64_t bytes) { + std::lock_guard lock(m); + maxHeldBytes = bytes; + } + + bool inBatch() const { return current().active; } + + void begin(uint64_t seq) { + Current& c = current(); + c.active = true; + c.seq = seq; + c.items.clear(); + } + + void add(T&& item) { current().items.emplace_back(std::move(item)); } + + // Hand the batch over. If it's the next one to be written and no other thread is + // writing, this thread writes it and then any held batches that follow it; meanwhile + // other threads just hand their batches over and carry on. + void end() { + Current& c = current(); + c.active = false; + uint64_t bytes = 0; + for (const auto& item : c.items) bytes += itemBytes(item); + std::unique_lock lock(m); + heldBytes += bytes; + held.emplace(c.seq, std::make_pair(bytes, std::move(c.items))); + c.items = std::vector(); + if (draining) { + flushQueued(lock); + return; + } + draining = true; + while (!held.empty() && held.begin()->first == nextSeq) { + auto batch = std::move(held.begin()->second); + held.erase(held.begin()); + lock.unlock(); + { + std::lock_guard writeLock(w); + for (auto& item : batch.second) writeItem(item); + } + lock.lock(); + nextSeq++; + if (flushItem) { + toFlush.emplace_back(std::move(batch)); + heldShrunk.notify_one(); // a waiting thread can flush it + continue; + } + heldBytes -= batch.first; + // Wake the waiting threads only once there is room. Woken for every batch + // written, they all found no room and went back to sleep, again and again: + // most of the system time of a 96-thread planet run. + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + draining = false; + flushQueued(lock); + } + + void waitForRoom() { + std::unique_lock lock(m); + while (heldBytes > maxHeldBytes) { + if (!toFlush.empty() && !flushing) flushQueued(lock); + else heldShrunk.wait(lock); + } + } + + bool allWritten() { + std::lock_guard lock(m); + return held.empty() && toFlush.empty() && !flushing; + } + + // Held while items are written; lock it to write anything outside a batch + std::mutex& writeMutex() { return w; } + +private: + struct Current { + bool active = false; + uint64_t seq = 0; + std::vector items; + }; + // Flush the batches waiting for flushItem, unless another thread already does; called and + // returns with m held + void flushQueued(std::unique_lock &lock) { + if (!flushItem || flushing) return; + flushing = true; + while (!toFlush.empty()) { + auto batch = std::move(toFlush.front()); + toFlush.pop_front(); + lock.unlock(); + for (auto& item : batch.second) flushItem(item); + batch.second = std::vector(); + lock.lock(); + heldBytes -= batch.first; + if (heldBytes <= maxHeldBytes) heldShrunk.notify_all(); + } + flushing = false; + // batches queued while finishing the last one are taken by the next thread that comes by; + // wake one in case all others wait for room + heldShrunk.notify_one(); + } + + // The batch this thread is working on (one per thread and item type) + static Current& current() { + static thread_local Current c; + return c; + } + + std::function writeItem; + std::function itemBytes; + std::function flushItem; + uint64_t maxHeldBytes = 1ull << 30; + std::mutex m; // guards the fields below + std::mutex w; // held while writing + std::condition_variable heldShrunk; + bool draining = false; + uint64_t nextSeq = 0; + uint64_t heldBytes = 0; + std::map>> held; // seq -> (bytes, items) + std::deque>> toFlush; // written in order, waiting for flushItem + bool flushing = false; // a thread is flushing +}; + +#endif //_ORDERED_BATCHES_H diff --git a/include/pmtiles.h b/include/pmtiles.h index 7e8040c8..fb2e6682 100644 --- a/include/pmtiles.h +++ b/include/pmtiles.h @@ -3,7 +3,12 @@ #define _PMTILES_H #include +#include +#include +#include +#include #include "external/pmtiles.hpp" +#include "ordered_batches.h" struct TileOffset { uint64_t offset : 40; @@ -28,6 +33,14 @@ struct TileOffset { // PMTiles v3 tile type for MapLibre Tiles (not yet in external/pmtiles.hpp) #define TILETYPE_MLT 0x6 +// A tile waiting to be written: tiny tiles keep their raw data, so that a repeat +// can reuse the earlier copy, and are only compressed if they're written +struct PendingTile { + uint64_t tileId; + bool isTiny; + std::string data; // after writeTile: the bytes flushTile appends to the file (empty if a repeat) +}; + class PMTiles { public: @@ -42,10 +55,14 @@ class PMTiles { void saveTile(int zoom, int x, int y, std::string &data); void close(std::string &metadata); + // Tiles are written in batch order, so the file doesn't depend on thread scheduling + OrderedBatches batches; + private: std::ofstream outputStream; - std::mutex fileMutex; // guards file writes, numTilesWritten - std::mutex indexMutex; // guards access to sparseIndex, denseIndex, tinyCache, numTilesAddressed + std::vector outputBuffer; // a large stream buffer: one thread writes all tiles + uint64_t tileDataEnd = 0; // bytes of tile data written so far (tellp() costs a flush and a seek per tile) + std::mutex fileMutex; // guards leaf directory writes in close() uint64_t leafStart = 0; uint64_t numTilesWritten = 0; uint64_t numTilesAddressed = 0; @@ -54,6 +71,8 @@ class PMTiles { std::vector denseIndex; std::unordered_map tinyCache; + void writeTile(PendingTile &tile); // with batches.writeMutex() held: places the tile + void flushTile(PendingTile &tile); // in the same order, by one thread at a time: writes its bytes void appendWithRLE(std::vector &entries, pmtiles::entryv3 &entry); void appendTileEntry(uint64_t tileId, TileOffset offset, std::vector &rootEntries, std::vector &entries); void flushEntries(std::vector &rootEntries, std::vector &entries); diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp index c056d3a6..1d2e653f 100644 --- a/src/mbtiles.cpp +++ b/src/mbtiles.cpp @@ -2,15 +2,30 @@ #include "helpers.h" #include #include +#include using namespace sqlite; using namespace std; MBTiles::MBTiles(): pendingStatements1(std::make_shared>()), - pendingStatements2(std::make_shared>()) + pendingStatements2(std::make_shared>()), + pendingBytes(0), + batches([this](PendingStatement& stmt) { + const std::lock_guard lock(m); + insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + }, + [](const PendingStatement& stmt) { return sizeof(PendingStatement) + stmt.data.size(); }) {} +// At most this much is buffered while SQLite is busy; beyond it, workers wait +// for the writer instead (a planet run otherwise buffered ~15M tiles, over 3 GB). +static const size_t MAX_PENDING_BYTES = 256 << 20; + +static size_t pendingCost(const std::string& data) { + return sizeof(PendingStatement) + data.size(); +} + MBTiles::~MBTiles() { if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open } @@ -71,6 +86,7 @@ void MBTiles::flushPendingStatements() { while(!pendingStatements2->empty()) { const PendingStatement& stmt = pendingStatements2->back(); insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge); + pendingBytes -= pendingCost(stmt.data); pendingStatements2->pop_back(); } @@ -80,19 +96,37 @@ void MBTiles::flushPendingStatements() { } void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) { + if (batches.inBatch()) { + batches.add({zoom, x, y, *data, isMerge}); + return; + } + // If the lock is available, write directly to SQLite. if (m.try_lock()) { insertOrReplace(zoom, x, y, *data, isMerge); flushPendingStatements(); m.unlock(); - } else { + return; + } + + { // Else buffer the write for later, copying its binary blob. const std::lock_guard lock(pendingStatementsMutex); - pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + if (pendingBytes < MAX_PENDING_BYTES) { + pendingBytes += pendingCost(*data); + pendingStatements1->push_back({zoom, x, y, *data, isMerge}); + return; + } } + + // The buffer is full: wait for SQLite, then help drain the buffer. + const std::lock_guard lock(m); + insertOrReplace(zoom, x, y, *data, isMerge); + flushPendingStatements(); } void MBTiles::closeForWriting() { + if (!batches.allWritten()) throw std::runtime_error("MBTiles: a batch of tiles was never written - please file an issue"); flushPendingStatements(); preparedStatements[0].used(true); preparedStatements[1].used(true); diff --git a/src/pmtiles.cpp b/src/pmtiles.cpp index c9561b59..dceb5af4 100644 --- a/src/pmtiles.cpp +++ b/src/pmtiles.cpp @@ -10,11 +10,19 @@ #include "helpers.h" TileOffset::TileOffset() { } -PMTiles::PMTiles() { } +// Placing tiles (offset, repeats, index) and writing their bytes are two steps, each in batch order +// and one thread at a time, but run by different threads side by side: one thread doing both was +// the bottleneck of the tile phase at 96 threads. +PMTiles::PMTiles() : + batches([this](PendingTile &tile) { writeTile(tile); }, + [](const PendingTile &tile) { return tile.data.size(); }, + [this](PendingTile &tile) { flushTile(tile); }) { } PMTiles::~PMTiles() { } void PMTiles::open(std::string &filename) { std::cout << "Creating pmtiles at " << filename << std::endl; + outputBuffer.resize(4 << 20); + outputStream.rdbuf()->pubsetbuf(outputBuffer.data(), outputBuffer.size()); outputStream.open(filename, std::ios::out | std::ios::trunc | std::ios::binary); // dummy header/root directory for now - we'll write it all later char header[HEADER_ROOT] = "PMTiles"; @@ -24,6 +32,7 @@ void PMTiles::open(std::string &filename) { // Finish writing the .pmtiles file void PMTiles::close(std::string &metadata) { std::cout << "\nClosing pmtiles file" << std::flush; + if (!batches.allWritten()) { throw std::runtime_error("PMTiles: a batch of tiles was never written - please file an issue"); } // add all tiles to directories, writing leaf directories as we go std::vector rootEntries; @@ -129,34 +138,50 @@ void PMTiles::flushEntries(std::vector &rootEntries, std::vect rootEntries.emplace_back(rootEntry); } -// Write a tile to file and store it in the index -// - if the tile is small and has already been written, reuse that instead -// - we're a bit fussy about mutexs because compress_string is expensive +// Save a tile: compress it here (compress_string is expensive), but leave tiny tiles +// raw until they're written, since most of them (e.g. sea) repeat an earlier one void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { - TileOffset offset; - bool isNew = false; - uint64_t tileId = pmtiles::zxy_to_tileid(zoom,x,y); + PendingTile tile; + tile.tileId = pmtiles::zxy_to_tileid(zoom,x,y); + tile.isTiny = data.size() lock(batches.writeMutex()); + writeTile(tile); + flushTile(tile); +} - // if it's a tiny tile (e.g. sea), see if we've written it already - std::unique_lock indexLock1(indexMutex); - if (data.size()second; + tile.data = std::string(); // already in the file } else { - indexLock1.unlock(); - std::string compressed = compress_string(data, Z_DEFAULT_COMPRESSION, true); - std::lock_guard lock(fileMutex); - // write to file - offset = TileOffset(static_cast(outputStream.tellp()) - HEADER_ROOT, compressed.size()); - outputStream.write(compressed.c_str(), compressed.size()); + std::string compressed = tile.isTiny ? compress_string(tile.data, Z_DEFAULT_COMPRESSION, true) : std::move(tile.data); + offset = TileOffset(tileDataEnd, compressed.size()); + tileDataEnd += compressed.size(); numTilesWritten++; - isNew = true; + // add to cache if tiny + if (tile.isTiny) { + if (tinyCache.size()>TINY_MAX_SIZE) tinyCache.clear(); + tinyCache.insert({ tile.data, offset }); + } + tile.data = std::move(compressed); // flushTile writes it } - + // store in index - std::lock_guard indexLock2(indexMutex); + uint64_t tileId = tile.tileId; numTilesAddressed++; if (isSparse) { sparseIndex[tileId] = offset; @@ -164,10 +189,4 @@ void PMTiles::saveTile(int zoom, int x, int y, std::string &data) { if (tileId >= denseIndex.size()) denseIndex.resize(tileId+10000, { 0, 0xffffff }); denseIndex[tileId] = offset; } - - // add to cache if tiny - if (isNew && data.size()TINY_MAX_SIZE) tinyCache.clear(); - tinyCache.insert({ data, offset }); - } } diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7a7ae1e5..1514898a 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -504,8 +504,15 @@ int main(const int argc, const char* argv[]) { std::atomic batchCounter(0); const uint cutZoom = std::max(12, config.startZoom); - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { + // Batches are handed out as threads become free rather than all posted up + // front: a planet has ~14M of them, and their queued handlers held GBs. + std::mutex batchMutex; + std::size_t nextBatchStart = 0; + uint64_t nextBatchSeq = 0; + auto nextBatch = [&](std::size_t &startIndex, std::size_t &batchSize, uint64_t &seq) { + const std::lock_guard lock(batchMutex); + startIndex = nextBatchStart; + seq = nextBatchSeq++; // Compute how many tiles should be assigned to this batch -- // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. batchSize = 0; @@ -529,65 +536,89 @@ int main(const int argc, const char* argv[]) { batchSize++; } + nextBatchStart += batchSize; + return batchSize > 0; + }; - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten, &batchCounter]() { - clipCacheBatchId = ++batchCounter; - // A single-tile batch has no descendants to reuse its clips. - clipCacheNoStore = batchSize == 1; - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; + // .pmtiles and .mbtiles are written in batch order, so the file doesn't depend on thread scheduling + const bool toPMTiles = options.outputMode == OptionsParser::OutputMode::PMTiles; + const bool toMBTiles = options.outputMode == OptionsParser::OutputMode::MBTiles; + // While an earlier batch is still being worked on, each thread may run ahead by about + // 64 MB of finished tiles. With less, threads often wait for a slow batch (mostly a + // dense z12 area), above all when two threads share a core and each runs at half speed. + const uint64_t maxHeldBytes = (64ull << 20) * options.threadNum; + if (toPMTiles) sharedData.pmtiles.batches.setMaxHeldBytes(maxHeldBytes); + if (toMBTiles) sharedData.mbtiles.batches.setMaxHeldBytes(maxHeldBytes); + for (unsigned int t = 0; t < options.threadNum; t++) { + boost::asio::post(pool, [&]() { + std::size_t startIndex, batchSize; + uint64_t seq; + while (true) { + if (toPMTiles) sharedData.pmtiles.batches.waitForRoom(); + if (toMBTiles) sharedData.mbtiles.batches.waitForRoom(); + if (!nextBatch(startIndex, batchSize, seq)) break; + if (toPMTiles) sharedData.pmtiles.batches.begin(seq); + if (toMBTiles) sharedData.mbtiles.batches.begin(seq); + clipCacheBatchId = ++batchCounter; + // A single-tile batch has no descendants to reuse its clips. + clipCacheNoStore = batchSize == 1; + std::vector tileTimings; + std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); + for(std::size_t i = startIndex; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; #ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); - } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); + } + outputProc(sharedData, sources, attributeStore, data, coords, zoom); #ifdef CLOCK_MONOTONIC - if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); - } + if (options.logTileTimings) { + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); + } #endif - } + } - if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; - } + if (options.logTileTimings) { + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; + } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - uint64_t written = tilesWritten.load(); - - if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { - lastTilesWritten = written; - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; + if (toPMTiles) sharedData.pmtiles.batches.end(); + if (toMBTiles) sharedData.mbtiles.batches.end(); + tilesWritten += (endIndex - startIndex); + + if (io_mutex.try_lock()) { + uint64_t written = tilesWritten.load(); + + if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) { + lastTilesWritten = written; + // Show progress grouped by z6 (or lower) + size_t z = tileCoordinates[startIndex].first; + size_t x = tileCoordinates[startIndex].second.x; + size_t y = tileCoordinates[startIndex].second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; + } + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << " \r" << std::flush; + io_mutex.unlock(); } - io_mutex.unlock(); } }); } diff --git a/test/ordered_batches.test.cpp b/test/ordered_batches.test.cpp new file mode 100644 index 00000000..b9ff9599 --- /dev/null +++ b/test/ordered_batches.test.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include +#include +#include "external/minunit.h" +#include "ordered_batches.h" + +MU_TEST(test_ordered_batches_single_thread) { + std::vector written; + OrderedBatches batches([&](int& item) { written.push_back(item); }, [](const int&) { return 1; }); + + // batch 1 finishes first, so it's held until batch 0 is written + batches.begin(1); + mu_check(batches.inBatch()); + batches.add(10); + batches.add(11); + batches.end(); + mu_check(!batches.inBatch()); + mu_check(written.empty()); + mu_check(!batches.allWritten()); + + batches.begin(0); + batches.add(0); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11 })); + mu_check(batches.allWritten()); + + // an empty batch still moves the order on + batches.begin(3); + batches.add(30); + batches.end(); + batches.begin(2); + batches.end(); + mu_check(written == std::vector({ 0, 10, 11, 30 })); + mu_check(batches.allWritten()); +} + +MU_TEST(test_ordered_batches_threads) { + // Batches are handed out in order and finish in any order; with a small limit on what + // may be held, threads also have to wait for room + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written; + OrderedBatches batches([&](uint64_t& item) { written.push_back(item); }, [](const uint64_t&) { return 1; }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(written.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < written.size(); i++) if (written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST(test_ordered_batches_flush) { + // With flushItem: every item is written and then flushed, both in order, flushes never + // overlap, and a flush can be done by a thread other than the one that wrote the batch + const uint64_t numBatches = 2000, itemsPerBatch = 5; + std::vector written, flushed; + std::atomic flushingNow(0); + std::atomic overlap(false), flushedBeforeWritten(false); + std::atomic writtenCount(0); + OrderedBatches batches( + [&](uint64_t& item) { written.push_back(item); writtenCount++; }, + [](const uint64_t&) { return 1; }, + [&](uint64_t& item) { + if (flushingNow++ != 0) overlap = true; + if (item >= writtenCount) flushedBeforeWritten = true; + flushed.push_back(item); + if (item % 7 == 0) std::this_thread::yield(); + flushingNow--; + }); + batches.setMaxHeldBytes(20); + + std::mutex handOut; + uint64_t nextSeq = 0; + std::vector threads; + for (int t = 0; t < 8; t++) { + threads.emplace_back([&, t]() { + std::mt19937 rng(t); + while (true) { + batches.waitForRoom(); + uint64_t seq; + { + std::lock_guard lock(handOut); + if (nextSeq == numBatches) return; + seq = nextSeq++; + } + batches.begin(seq); + for (uint64_t i = 0; i < itemsPerBatch; i++) { + if (rng() % 4 == 0) std::this_thread::yield(); + batches.add(seq * itemsPerBatch + i); + } + batches.end(); + } + }); + } + for (auto& thread : threads) thread.join(); + + mu_check(batches.allWritten()); + mu_check(!overlap); + mu_check(!flushedBeforeWritten); + mu_check(written.size() == numBatches * itemsPerBatch); + mu_check(flushed.size() == numBatches * itemsPerBatch); + bool inOrder = true; + for (uint64_t i = 0; i < flushed.size(); i++) if (flushed[i] != i || written[i] != i) inOrder = false; + mu_check(inOrder); +} + +MU_TEST_SUITE(test_suite_ordered_batches) { + MU_RUN_TEST(test_ordered_batches_single_thread); + MU_RUN_TEST(test_ordered_batches_threads); + MU_RUN_TEST(test_ordered_batches_flush); +} + +int main() { + MU_RUN_SUITE(test_suite_ordered_batches); + MU_REPORT(); + return MU_EXIT_CODE; +}