diff --git a/docs/modules/ROOT/pages/how-to/async-executor.adoc b/docs/modules/ROOT/pages/how-to/async-executor.adoc index d781443..ba4f17e 100644 --- a/docs/modules/ROOT/pages/how-to/async-executor.adoc +++ b/docs/modules/ROOT/pages/how-to/async-executor.adoc @@ -128,9 +128,9 @@ Give every call whose result you need an `error_callback`. `executor.on_error(ca [[native-create-record]] == Native: `create_record` hands records over in batches -On the `arcadedb` (native) package, from 26.11.1.post4, the async executor collects the records that `create_record` receives for one type and kind, and hands up to 1,000 of them to the library at once as one columnar frame. Each record keeps its own callback and its own error callback. Before, every record was one call with one JSON text. A thread of the executor hands a batch over when its first record has waited 0.05 seconds, so a record is not held back when no other call reaches the executor. The first 128 records of an executor go one by one, as before, and so does a hand-over of fewer than 64 collected records, because a frame has a fixed cost that a handful of records does not repay. The `arcadedb-jvm` package does not change. +On the `arcadedb` (native) package, from 26.11.1.post4, the async executor collects the records that `create_record` receives for one type and kind, and hands up to 1,000 of them to the library at once as one columnar frame. Each record keeps its own callback and its own error callback. Before, every record was one call with one JSON text. A thread hands a batch over when its first record has waited 0.05 seconds, so a record is not held back when no other call reaches the executor. In 26.11.1.post4 every executor starts that thread with its first collected record; after post4 (ArcadeData/arcadedb-python#406) one thread of the process does it for all executors, and an executor starts none. The first 128 records of an executor go one by one, as before, and so does a hand-over of fewer than 64 collected records, because a frame has a fixed cost that a handful of records does not repay. The `arcadedb-jvm` package does not change. -An executor that receives only about 100 records pays for the per-record counter that lets the first records run one callback each: an async `create_record` of 100 records reads 1.060 [1.055, 1.067] on the laptop and 1.050 [1.045, 1.052] on the mini, that is about 6 percent more time for executors of 100 records. Executors of 1,000 records or more are faster. A linger thread shared by the process, tracked in ArcadeData/arcadedb-python#406, would remove this cost. +An executor that receives only about 100 records pays for the per-record counter that lets the first records run one callback each: an async `create_record` of 100 records reads 1.060 [1.055, 1.067] on the laptop and 1.050 [1.045, 1.052] on the mini, that is about 6 percent more time for executors of 100 records. Executors of 1,000 records or more are faster. The figures here are those of post4. The thread shared by the process (ArcadeData/arcadedb-python#406) is meant to remove part of this cost; it has not been timed yet. The table is one study of the `arcadedb` (native) package on a laptop and on a second Linux machine, the mini: the time for each operation of the post4 wheel against the 26.11.1.post3 wheel. The ratio is the median of the pair ratios (below 1 means less time) with its 95% interval. Setup: Python 3.12, one fresh process for each run, two cores for the process, the two builds alternated in both orders (ABBA), the first round dropped, and an A/A control (the post3 wheel twice) whose median ratio was between 0.994 and 1.002 on the laptop and between 0.989 and 1.002 on the mini. The post4 wheels were built by CI from commit e83947e, the commit of the tag, and the post3 wheels were the ones that CI built from the post3 commit. The files on PyPI were checked for the same engine as 26.11.1 (see xref:which-package.adoc#engine-of-post4[The engine of 26.11.1.post4]) and were not timed again. diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 715b092..d23b14d 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -9,9 +9,11 @@ import copy import ctypes +import heapq import itertools import json import logging +import os import threading import time import weakref @@ -473,17 +475,18 @@ def get_deferred_incoming_edge_count(self) -> int: # records per crossing of create_record (the same size as the commit interval of insert_many on this build) _ASYNC_BATCH = 1000 -# seconds a collected record may wait for the rest of its batch before a thread of the executor hands it over: a caller that creates +# seconds a collected record may wait for the rest of its batch before a thread of the process (_Linger) hands it over: a caller that creates # records and waits for a callback, or reads after a pause, with no other call on the executor, is never kept waiting longer _ASYNC_LINGER = 0.05 # the first records of an executor go one op each, exactly as in 26.11.1.post3, before it starts to collect: no copy, no buffer, no linger thread, no frame. Collecting has a fixed cost -# per executor (the linger thread that is started for it and the locks) and a frame has its own fixed cost on top (see _ASYNC_FRAME_MIN), so a small load repays neither. In an untimed +# per executor (the linger thread that each executor started when this was measured, and the locks) and a frame has its own fixed cost on top (see _ASYNC_FRAME_MIN), so a small load repays neither. In an untimed # micro of one executor call (a new executor, N records, wait_completion, native, cores 14 and 15) the first head of this change, which collected from the first record, read 1.6x slower # than post3 for 1 record and 1.2x for 16, and a direct phase of 32 with the frame floor still read 1.2x to 1.3x slower for 33 to 64 records. An executor of up to _ASYNC_DIRECT records is # therefore exactly post3. Beyond it the records collect, and a frame repays itself only from _ASYNC_DIRECT + 64 records, 192: an executor of 129 to 191 records collects a few records, # hands them over one op each and pays the collection cost without a frame, about 0.2 ms more than post3 whatever its size (10% to 15% of its time in the micro). The cost is near # constant per executor, so the relative slowdown falls as this number rises; 128 was chosen after a direct phase of 96 put the band on the 100-record case, and it is not tuned further. -# A linger thread shared by all executors of the process would remove the cost (ArcadeData/arcadedb-python#406); 0 turns the direct phase off. +# The linger thread is now one for the whole process (_Linger, ArcadeData/arcadedb-python#406): an executor starts no thread of its own. Whether that removes the cost of the band (and so +# whether this number can fall or go) is not measured yet; the cost of a copy and a lock per collected record stays. 0 turns the direct phase off. _ASYNC_DIRECT = 128 # a hand-over of fewer collected records than this goes one op each, as in post3: a columnar frame has a fixed cost per crossing (the untimed micro put it near 270 us on this build, # against about 12 us per record for the per-record op and about 3 us per record inside a frame, so a frame repays itself from about 30 records), and without this floor an executor of @@ -500,6 +503,81 @@ def _snapshot(props: dict) -> dict: } +class _Linger: + """The one thread of the process that keeps the due times of collected batches (``create_record``) and hands a batch over when + its time is up. + + An executor starts no thread: the first record of a batch registers the batch here with its due time (:meth:`add`). A flush point + that hands the batch over earlier leaves the entry to expire; it is dropped, without a thread, when its time comes. When a batch + is due and still the executor's current one, this thread starts a short-lived thread for the hand-over (``_linger_fire``), so that + a hand-over that blocks in the engine (a full queue, back pressure) holds back no other executor. The locks have one order: an + executor's ``_buf_lock``, then this lock. This thread never holds its lock while it touches an executor. + """ + + def __init__(self): + self._wake = threading.Condition(threading.Lock()) + self._heap: list = ( + [] + ) # (due, serial, executor), the earliest first; the serial keeps the executors from being compared + self._serial = itertools.count() + self._thread = None + + def add(self, executor, due: float): + with self._wake: + if self._thread is None or not self._thread.is_alive(): + thread = threading.Thread( + target=self._run, name="arcadedb-async-linger-timer", daemon=True + ) + thread.start() + self._thread = thread + entry = (due, next(self._serial), executor) + heapq.heappush(self._heap, entry) + if self._heap[0] is entry: # the new earliest: the thread sleeps too long + self._wake.notify() + + def _run(self): + while True: + with self._wake: + while True: + if not self._heap: + self._wake.wait() + continue + left = self._heap[0][0] - time.monotonic() + if left > 0: + self._wake.wait(left) + continue + due, _, executor = heapq.heappop(self._heap) + break + if executor._linger_pending(due): + try: + threading.Thread( + target=executor._linger_fire, + args=(due,), + name="arcadedb-async-linger", + daemon=True, + ).start() + except ( + RuntimeError + ): # no thread to be had: hand over here, a delay for the others is better than a batch that stays + executor._linger_fire(due) + executor = ( + None # an executor that is dropped is not kept alive by this frame + ) + + +_linger = _Linger() + + +def _linger_after_fork(): + """A forked child has none of the threads of its parent, and the lock of ``_linger`` may be held by one of them: start afresh.""" + global _linger + _linger = _Linger() + + +if hasattr(os, "register_at_fork"): # not on Windows + os.register_at_fork(after_in_child=_linger_after_fork) + + class NativeAsyncExecutor: """The database's async executor (one per database) with the JPype wrapper's names; callbacks run on the engine's writer threads and take the GIL, as the JPype proxies do.""" @@ -522,10 +600,8 @@ def __init__(self, database): self._buf: list = [] self._buf_key = None self._buf_lock = threading.RLock() - # the linger thread (started with the first collected record) hands a batch over when its first record has waited - # _ASYNC_LINGER; a failure it meets is kept and raised by the next call that hands records over - self._linger = threading.Condition(self._buf_lock) - self._linger_thread = None + # the linger thread of the process (_Linger) hands a batch over when its first record has waited _ASYNC_LINGER (_linger_due, + # set with the first record of each batch); a failure it meets is kept and raised by the next call that hands records over self._linger_due = 0.0 self._linger_stop = False self._linger_error = None @@ -653,8 +729,8 @@ def create_record(self, document, callback=None, error_callback=None): afterwards is stored as it was at this call) and cross the library boundary ``_ASYNC_BATCH`` at a time, as one columnar frame with the callback ids of each record (not one JSON op per record) when at least ``_ASYNC_FRAME_MIN`` (64) of them are handed over together, else one op each as in post3. Every other call on the executor (``wait_completion``, ``is_processing``, ``query``, ``command``, ``kill``, ``close``, ...) hands the collected records over first, so the engine - queue keeps the order of the calls and a record is never held back from a ``wait_completion`` that follows its call. A thread of - the executor hands a batch over when its first record has waited ``_ASYNC_LINGER`` (50 ms), so a caller that waits for a + queue keeps the order of the calls and a record is never held back from a ``wait_completion`` that follows its call. A thread + that the process shares between its executors (``_Linger``) hands a batch over when its first record has waited ``_ASYNC_LINGER`` (50 ms), so a caller that waits for a callback or reads after a pause, with no other call, is not kept waiting; a failure that thread meets is raised by the next call that hands records over. Records whose shape the frame cannot carry (different keys, nested values, numpy scalars, mixed-type columns, integers beyond 64 bits) @@ -688,35 +764,29 @@ def create_record(self, document, callback=None, error_callback=None): def _start_linger(self): """Under ``_buf_lock``: the batch that has just got its first record is handed over at the latest after ``_ASYNC_LINGER``.""" self._linger_due = time.monotonic() + _ASYNC_LINGER - if self._linger_thread is None: - self._linger_thread = threading.Thread( - target=self._linger_loop, name="arcadedb-async-linger", daemon=True - ) - self._linger_thread.start() - else: - self._linger.notify() + _linger.add(self, self._linger_due) + + def _linger_pending(self, due): + """Without the lock, for the thread of the process: is the batch that was registered with ``due`` still waiting? A false answer + is final (the batch was handed over, a later batch has its own entry, or the executor is closed); a true one is checked again + under the lock by :meth:`_linger_fire`.""" + return self._linger_due == due and bool(self._buf) and not self._linger_stop - def _linger_loop(self): + def _linger_fire(self, due): + """The hand-over of the batch that was registered with ``due``, on a thread of its own (see :class:`_Linger`).""" with self._buf_lock: - while not self._linger_stop: - if not self._buf: - self._linger.wait() - continue - left = self._linger_due - time.monotonic() - if left > 0: - self._linger.wait(left) - continue - try: - self._flush_buffer() - except ( - BaseException - ) as e: # no caller to raise it to: the next call that hands records over raises it - self._linger_error = e + if self._linger_stop or not self._buf or self._linger_due != due: + return + try: + self._flush_buffer() + except ( + BaseException + ) as e: # no caller to raise it to: the next call that hands records over raises it + self._linger_error = e def _stop_linger(self): with self._buf_lock: self._linger_stop = True - self._linger.notify_all() def _queue_record(self, type_name, kind, props, callback, error_callback): cb = self._record_callback(callback, kind) if callback is not None else 0 diff --git a/tests/test_native_async_create_batch_unit.py b/tests/test_native_async_create_batch_unit.py index a2945f6..4033393 100644 --- a/tests/test_native_async_create_batch_unit.py +++ b/tests/test_native_async_create_batch_unit.py @@ -13,6 +13,7 @@ """ import json +import threading import time import pytest @@ -98,6 +99,8 @@ def library(monkeypatch): monkeypatch.setattr(extras, "_call", fake) monkeypatch.setattr(extras, "_ensure_upcall", lambda: None) monkeypatch.setattr(bulk, "_usable", lambda: True) + # a linger thread of its own for each test: the due times of another test (3600 s) are not in this one's way + monkeypatch.setattr(extras, "_linger", extras._Linger()) return fake @@ -403,19 +406,161 @@ def test_a_failure_of_the_linger_flush_is_raised_by_the_next_call( executor.wait_completion() # raised once, not again -def test_close_stops_the_linger_thread(library, executor, monkeypatch): +def test_close_leaves_nothing_for_the_linger_thread_to_hand_over( + library, executor, monkeypatch +): monkeypatch.setattr(extras, "_ASYNC_LINGER", 3600.0, raising=False) executor.create_record(_Doc("Item", {"id": 1})) - thread = executor._linger_thread - assert thread is not None and thread.is_alive() + assert extras._linger._thread.is_alive() + due = executor._linger_due executor.close() - assert _wait_for( - lambda: not thread.is_alive() - ), "the linger thread outlived close()" assert library.order()[0] == ( "create_record", None, ) # the record was handed over by close(), before the engine closed + assert not executor._linger_pending(due) + executor._linger_fire( + due + ) # the entry of the closed executor, when its time comes: nothing + assert library.order().count(("create_record", None)) == 1 + + +def test_an_executor_starts_no_thread_of_its_own(library, monkeypatch): + # ArcadeData/arcadedb-python#406: the thread that each executor started with its first collected record is the one of the process + first = extras.NativeAsyncExecutor(_Db()) + first.create_record(_Doc("Item", {"id": 0})) + before = threading.active_count() + executors = [extras.NativeAsyncExecutor(_Db()) for _ in range(20)] + for executor in executors: + for doc in _rows(3): + executor.create_record(doc) + assert threading.active_count() == before + assert ( + len(extras._linger._heap) == 21 + ) # one entry for the first record of each batch + for executor in executors: + executor.wait_completion() + assert len(library.batches()) == 20 + + +def test_many_executors_get_their_batches_handed_over_in_time(library, monkeypatch): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + executors = [extras.NativeAsyncExecutor(_Db()) for _ in range(10)] + for index, executor in enumerate(executors): + for doc in _rows(3, f"T{index}"): + executor.create_record(doc) + assert _wait_for(lambda: len(library.batches()) == 10) + assert sorted(b[2]["type"] for b in library.batches()) == sorted( + f"T{i}" for i in range(10) + ) + assert all(b[2]["n"] == 3 for b in library.batches()) + + +def test_a_batch_that_a_flush_point_handed_over_is_not_handed_over_again( + library, executor, monkeypatch +): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + fired = [] + real = executor._linger_fire + monkeypatch.setattr( + executor, "_linger_fire", lambda due: (fired.append(due), real(due)) + ) + for doc in _rows(3): + executor.create_record(doc) + executor.wait_completion() + assert library.order() == [("create_records", 3), ("wait", None)] + assert _wait_for(lambda: not extras._linger._heap), "the entry never came due" + time.sleep(0.05) + assert fired == [] # a stale entry starts no thread + assert library.order() == [("create_records", 3), ("wait", None)] + + +def test_the_entry_of_an_earlier_batch_does_not_cut_a_later_batch_short( + library, executor, monkeypatch +): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.1, raising=False) + fired = [] + real = executor._linger_fire + monkeypatch.setattr( + executor, "_linger_fire", lambda due: (fired.append(due), real(due)) + ) + for doc in _rows(2): + executor.create_record(doc) + executor.wait_completion() # the first batch is handed over by this call; its entry stays + first_due = executor._linger_due + for doc in _rows(2): + executor.create_record(doc) + second_due = executor._linger_due + assert second_due > first_due + assert _wait_for(lambda: library.batches() and len(library.batches()) == 2) + assert fired == [second_due] # only the entry of the batch that was still waiting + assert time.monotonic() >= second_due # and not before its own time + + +def test_the_order_of_the_calls_holds_across_linger_hand_overs_and_flush_points( + library, executor, monkeypatch +): + # one op each (a frame of a few records is a column layout; the id of each record shows in the per-record op) + monkeypatch.setattr(extras, "_ASYNC_FRAME_MIN", 1000) + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + sent = 0 + for round_ in range(4): + for _ in range(3): + executor.create_record(_Doc("Item", {"id": sent})) + sent += 1 + if round_ % 2 == 0: + assert _wait_for(lambda: len(library.per_record()) == sent) + else: + executor.is_processing() # a flush point of its own + executor.wait_completion() + assert [o[2]["props"]["id"] for o in library.per_record()] == list(range(sent)) + + +def test_a_hand_over_that_blocks_holds_back_no_other_executor(library, monkeypatch): + # the engine can block a hand-over (a full queue): the linger thread of the process must not wait for it + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + entered, release = threading.Event(), threading.Event() + + def gated(prefix, name, handle, op, text, *rest): + if json.loads(text).get("type") == "Blocked": + entered.set() + assert release.wait(30), "the test never released the blocked hand-over" + return library(prefix, name, handle, op, text, *rest) + + monkeypatch.setattr(extras, "_call", gated) + blocked, other = extras.NativeAsyncExecutor(_Db()), extras.NativeAsyncExecutor( + _Db() + ) + try: + for doc in _rows(3, "Blocked"): + blocked.create_record(doc) + assert entered.wait(10), "the first hand-over was never started" + for doc in _rows(3, "Free"): + other.create_record(doc) + assert _wait_for( + lambda: any(b[2]["type"] == "Free" for b in library.batches()) + ), "an executor waited for the hand-over of another" + finally: + release.set() + assert _wait_for(lambda: any(b[2]["type"] == "Blocked" for b in library.batches())) + + +def test_a_forked_child_starts_a_linger_thread_afresh(library, executor, monkeypatch): + monkeypatch.setattr(extras, "_ASYNC_LINGER", 3600.0, raising=False) + executor.create_record(_Doc("Item", {"id": 1})) + parents = extras._linger + assert parents._thread is not None and len(parents._heap) == 1 + extras._linger_after_fork() # what os.register_at_fork runs in the child + children = extras._linger + assert children is not parents + assert children._thread is None and children._heap == [] + monkeypatch.setattr(extras, "_ASYNC_LINGER", 0.02, raising=False) + child_executor = extras.NativeAsyncExecutor(_Db()) + for doc in _rows(3): + child_executor.create_record(doc) + assert _wait_for( + lambda: library.batches() + ), "the child's batch was never handed over" # the direct phase (ArcadeData/arcadedb-python #388, #393): the first _ASYNC_DIRECT (128) records of an executor go one op each, exactly as in post3. @@ -433,7 +578,8 @@ def test_an_executor_of_up_to_128_records_sends_one_op_each_and_collects_nothing executor.create_record( doc, callback=seen.append, error_callback=lambda e: seen.append(e) ) - assert executor._buf == [] and executor._linger_thread is None + assert executor._buf == [] and extras._linger._thread is None + assert extras._linger._heap == [] assert library.batches() == [] ops = library.per_record() assert len(ops) == n