From 3744b19dc13faa3db2c1133e353c3ce7d5322d35 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:20:44 +0200 Subject: [PATCH 01/66] feat: run popen Message IO on Trio host threads Move coordinator and worker framed protocol IO into dedicated Trio host threads for local popen + import bootstrap, while keeping the sync Channel/Gateway API and WorkerPool remote_exec. Adds a trio dependency; disable with EXECNET_TRIO_HOST=0. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- CHANGELOG.rst | 4 + doc/implnotes.rst | 43 ++- pyproject.toml | 3 + src/execnet/_trio_host.py | 527 ++++++++++++++++++++++++++++++++++++ src/execnet/_trio_worker.py | 117 ++++++++ src/execnet/gateway.py | 17 +- src/execnet/gateway_base.py | 43 ++- src/execnet/multi.py | 18 +- uv.lock | 61 +++++ 9 files changed, 818 insertions(+), 15 deletions(-) create mode 100644 src/execnet/_trio_host.py create mode 100644 src/execnet/_trio_worker.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7af6272..eac3c8d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ ------------------ * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. +* Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and + worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. + Other gateway types and greenlet execmodels keep the legacy thread path. + 2.1.2 (2025-11-11) ------------------ diff --git a/doc/implnotes.rst b/doc/implnotes.rst index d13d9e87..3dae82d6 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -7,7 +7,7 @@ capable of receiving and executing code, and routing data through channels. Gateways operate on InputOutput objects offering -a write and a read(n) method. + a write and a read(n) method. Once bootstrapped a higher level protocol based on Messages is used. Messages are serialized @@ -15,18 +15,43 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -After bootstrapping the BaseGateway opens a receiver thread which +Trio host-thread IO (popen / import bootstrap) +---------------------------------------------- + +For local ``popen`` gateways that use import-based bootstrap +(same installed ``execnet`` + ``trio`` on both sides), Message +protocol IO runs inside a dedicated OS thread hosting a Trio +event loop (``execnet._trio_host.TrioHost``): + +* Coordinator: ``trio.lowlevel.open_process`` plus async framed + reader/writer tasks per gateway (one host thread per ``Group``). +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio + streams; ``remote_exec`` still runs on the existing WorkerPool + (including ``main_thread_only`` primary-thread integration). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from +non-host threads wait until the frame is written (so abrupt +``os._exit`` cannot drop queued data). Sends from the Trio host +thread (receiver callbacks) only enqueue, to avoid deadlocking +the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types +(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) +still use the legacy thread receiver and sync ``Popen`` path. + +Legacy thread model +------------------- + +After bootstrapping, ``BaseGateway`` opens a receiver thread which accepts encoded messages and triggers actions to interpret them. Sending of channel data items happens directly through write operations to InputOutput objects so there is no -separate thread. +separate send thread. -Code execution messages are put into an execqueue from -which they will be taken for execution. gateway.serve() -will take and execute such items, one by one. This means -that by incoming default execution is single-threaded. +Code execution messages are scheduled on a WorkerPool. +On the worker, ``serve()`` integrates the main thread as the +primary executor when using the ``thread`` / ``main_thread_only`` +models. The receiver thread terminates if the remote side sends a gateway termination message or if the IO-connection drops. -It puts an end symbol into the execqueue so -that serve() can cleanly finish as well. diff --git a/pyproject.toml b/pyproject.toml index fcd2b4b5..709f887b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ description = "execnet: rapid multi-Python deployment" readme = {"file" = "README.rst", "content-type" = "text/x-rst"} license = "MIT" requires-python = ">=3.10" +dependencies = [ + "trio>=0.32", +] authors = [ { name = "holger krekel and others" }, ] diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py new file mode 100644 index 00000000..6ece94c4 --- /dev/null +++ b/src/execnet/_trio_host.py @@ -0,0 +1,527 @@ +"""Trio host thread for execnet Message-protocol IO. + +Coordinator and worker both run framed read/write loops here. +Sync Channel/Gateway APIs talk to this host via thread-safe queues and +``trio.from_thread``. +""" + +from __future__ import annotations + +import os +import queue +import subprocess +import threading +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol +from typing import TypeVar + +import trio + +from .gateway_base import ExecModel +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import trace + +if TYPE_CHECKING: + from .gateway_base import BaseGateway + +T = TypeVar("T") + +_CLOSE_WRITE = object() +_ENABLED_ENV = "EXECNET_TRIO_HOST" + + +def trio_host_enabled() -> bool: + """Return whether the Trio IO path should be used when applicable.""" + value = os.environ.get(_ENABLED_ENV, "1").strip().lower() + return value not in ("0", "false", "no", "off") + + +def should_use_trio_popen(spec: Any) -> bool: + """PoC: Trio path only for local popen with import bootstrap.""" + if not trio_host_enabled(): + return False + if not getattr(spec, "popen", False): + return False + if getattr(spec, "via", None): + return False + if getattr(spec, "python", None): + return False + # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. + execmodel = getattr(spec, "execmodel", None) + if execmodel not in (None, "thread", "main_thread_only"): + return False + return True + + +class AsyncByteIO(Protocol): + async def read_exact(self, n: int) -> bytes: ... + + async def write_all(self, data: bytes) -> None: ... + + async def aclose_read(self) -> None: ... + + async def aclose_write(self) -> None: ... + + +async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: + buf = bytearray() + while len(buf) < n: + chunk = await stream.receive_some(n - len(buf)) + if not chunk: + raise EOFError("expected %d bytes, got %d" % (n, len(buf))) + buf += chunk + return bytes(buf) + + +async def read_message(io: AsyncByteIO) -> Message: + header = await io.read_exact(9) + msgtype, channel, payload = Message.from_header(header) + data = await io.read_exact(payload) if payload else b"" + return Message.from_parts(msgtype, channel, data) + + +class ProcessStreamsIO: + """Async IO over a Trio Process stdin/stdout pair.""" + + def __init__(self, process: trio.Process) -> None: + assert process.stdin is not None + assert process.stdout is not None + self.process = process + self._stdin = process.stdin + self._stdout = process.stdout + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stdout, n) + + async def write_all(self, data: bytes) -> None: + await self._stdin.send_all(data) + + async def aclose_read(self) -> None: + await self._stdout.aclose() + + async def aclose_write(self) -> None: + await self._stdin.aclose() + + +class FdStreamsIO: + """Async IO over OS file descriptors (worker stdio pipes).""" + + def __init__(self, read_fd: int, write_fd: int) -> None: + self._read = trio.lowlevel.FdStream(read_fd) + self._write = trio.lowlevel.FdStream(write_fd) + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._read, n) + + async def write_all(self, data: bytes) -> None: + await self._write.send_all(data) + + async def aclose_read(self) -> None: + await self._read.aclose() + + async def aclose_write(self) -> None: + await self._write.aclose() + + +class SyncIOHandle: + """Sync IO facade for Group.terminate wait/kill/close_write.""" + + remoteaddress: str + + def __init__( + self, + execmodel: ExecModel, + session: ProtocolSession, + *, + remoteaddress: str | None = None, + ) -> None: + self.execmodel = execmodel + self._session = session + if remoteaddress is not None: + self.remoteaddress = remoteaddress + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio IO handle") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio IO handle") + + def close_read(self) -> None: + self._session.request_close_read() + + def close_write(self) -> None: + self._session.request_close_write() + + def wait(self) -> int | None: + return self._session.wait_process() + + def kill(self) -> None: + self._session.kill_process() + + +class ProtocolSession: + """Reader/writer tasks for one gateway connection.""" + + def __init__( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + host: TrioHost, + ) -> None: + self.gateway = gateway + self.io = io + self.process = process + self.host = host + self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._done = threading.Event() + self._process_exitcode: int | None = None + self._process_done = threading.Event() + self._send_closed = False + self._lock = threading.Lock() + + def enqueue_message(self, message: Message) -> None: + """Enqueue a frame; wait until written when safe to block. + + Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist + crash tests) cannot drop already-"sent" data still in the queue. + + The Trio host thread (receiver callbacks) must not wait — that would + deadlock the writer task on the same event loop. + """ + wait = not self.host.is_host_thread() + done = threading.Event() if wait else None + errors: list[BaseException] = [] + with self._lock: + if self._send_closed or self._done.is_set(): + raise OSError("cannot send (already closed?)") + self._outbound.put((message.pack(), done, errors)) + if done is None: + return + if not done.wait(timeout=120.0): + raise OSError("cannot send (write timed out)") + if errors: + raise OSError("cannot send (already closed?)") from errors[0] + + def request_close_write(self) -> None: + with self._lock: + if self._send_closed: + return + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + + def request_close_read(self) -> None: + # Reader observes EOF / cancel; nothing required from callers. + return + + def wait_done(self, timeout: float | None = None) -> bool: + return self._done.wait(timeout) + + def wait_process(self) -> int | None: + if self.process is None: + return None + + async def _wait() -> int | None: + assert self.process is not None + # Always await wait() so the child is reaped (no zombies). + code = await self.process.wait() + self._process_exitcode = code + self._process_done.set() + return code + + try: + return self.host.call(_wait) + except Exception: + self._process_done.wait() + return self._process_exitcode + + def kill_process(self) -> None: + if self.process is None: + return + + async def _kill() -> None: + assert self.process is not None + with trio.move_on_after(5): + self.process.kill() + self._process_exitcode = await self.process.wait() + self._process_done.set() + + try: + self.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + def is_alive(self) -> bool: + return not self._done.is_set() + + async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + if self.process is not None: + nursery.start_soon(self._supervisor) + task_status.started() + await self._reader() + nursery.cancel_scope.cancel() + finally: + await self._finish() + + async def _reader(self) -> None: + gateway = self.gateway + + def log(*msg: object) -> None: + gateway._trace("[trio-receiver]", *msg) + + log("RECEIVER: starting") + try: + while True: + msg = await read_message(self.io) + log("received", msg) + with gateway._receivelock: + msg.received(gateway) + del msg + except GatewayReceivedTerminate: + log("GATEWAY_TERMINATE") + except EOFError as exc: + log("EOF without prior gateway termination message") + gateway._error = exc + except Exception as exc: + log(gateway._geterrortext(exc)) + log("finishing receiver") + + async def _writer(self) -> None: + while True: + # abandon_on_cancel: queue.get blocks forever until a sentinel; + # nursery shutdown must not wait on it. + item = await trio.to_thread.run_sync( + self._outbound.get, abandon_on_cancel=True + ) + if item is _CLOSE_WRITE: + try: + await self.io.aclose_write() + except Exception as exc: + self.gateway._trace("aclose_write failed", exc) + return + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + + async def _supervisor(self) -> None: + assert self.process is not None + try: + self._process_exitcode = await self.process.wait() + finally: + self._process_done.set() + + async def _finish(self) -> None: + gateway = self.gateway + # Unblock a writer thread left in queue.get after cancel. + with self._lock: + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + gateway._trace("[trio-receiver] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done.set() + gateway._trace("[trio-receiver] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + try: + await self.io.aclose_read() + except Exception: + pass + try: + await self.io.aclose_write() + except Exception: + pass + + +class TrioHost: + """Dedicated OS thread running ``trio.run`` for protocol IO.""" + + def __init__(self, name: str = "execnet-trio-host") -> None: + self._name = name + self._thread: threading.Thread | None = None + self._token: trio.lowlevel.TrioToken | None = None + self._nursery: trio.Nursery | None = None + self._ready = threading.Event() + self._shutdown: trio.Event | None = None + self._started = False + + def start(self) -> None: + if self._started: + return + self._thread = threading.Thread(target=self._run, name=self._name, daemon=True) + self._thread.start() + if not self._ready.wait(timeout=30): + raise RuntimeError("TrioHost failed to start") + self._started = True + + def is_host_thread(self) -> bool: + return self._thread is not None and threading.current_thread() is self._thread + + def _run(self) -> None: + trio.run(self._main) + + async def _main(self) -> None: + self._token = trio.lowlevel.current_trio_token() + self._shutdown = trio.Event() + try: + async with trio.open_nursery() as nursery: + self._nursery = nursery + self._ready.set() + await self._shutdown.wait() + nursery.cancel_scope.cancel() + finally: + self._nursery = None + + def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + async def start_session( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + ) -> ProtocolSession: + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + session = ProtocolSession(gateway, io, process=process, host=self) + await self._nursery.start(session.task) + return session + + def stop(self, timeout: float | None = 5.0) -> None: + if not self._started or self._token is None or self._shutdown is None: + return + + def _set() -> None: + assert self._shutdown is not None + self._shutdown.set() + + try: + trio.from_thread.run_sync(_set, trio_token=self._token) + except Exception: + pass + if self._thread is not None: + self._thread.join(timeout=timeout) + self._started = False + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +async def bootstrap_import_async( + process: trio.Process, + *, + importdir: str, + execmodel: str, + gateway_id: str, +) -> ProcessStreamsIO: + """Send import-bootstrap source and wait for the handshake byte.""" + io = ProcessStreamsIO(process) + sources = [ + "import sys", + "if %r not in sys.path:" % importdir, + " sys.path.insert(0, %r)" % importdir, + "from execnet._trio_worker import serve_popen_trio", + "sys.stdout.write('1')", + "sys.stdout.flush()", + "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), + ] + source = "\n".join(sources) + await io.write_all((repr(source) + "\n").encode("utf-8")) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") + return io + + +class _TempIO: + """Placeholder IO used only while constructing a Trio-backed Gateway.""" + + def __init__(self, execmodel: ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio temp IO") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio temp IO") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def makegateway_popen_trio(group: Any, spec: Any) -> Any: + """Create a popen Gateway using Trio process + protocol IO on both sides.""" + import execnet + + from . import gateway_io + from .gateway_bootstrap import importdir + + host: TrioHost = group._ensure_trio_host() + args = gateway_io.popen_args(spec) + remote_execmodel = spec.execmodel + + async def _create_and_attach() -> Any: + process = await open_popen_process(args) + try: + async_io = await bootstrap_import_async( + process, + importdir=importdir, + execmodel=remote_execmodel, + gateway_id=spec.id, + ) + except BaseException: + with trio.move_on_after(5): + process.kill() + await process.wait() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, async_io, process=process) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py new file mode 100644 index 00000000..70338f1c --- /dev/null +++ b/src/execnet/_trio_worker.py @@ -0,0 +1,117 @@ +"""Worker-side Trio networking entry for popen/import bootstrap.""" + +from __future__ import annotations + +import os +import sys + +from . import gateway_base +from .gateway_base import WorkerGateway +from .gateway_base import get_execmodel +from .gateway_base import trace + + +def _prepare_protocol_fds() -> tuple[int, int]: + """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. + + Returns ``(read_fd, write_fd)`` for the Message protocol (child reads + coordinator stdin writes; child writes go to coordinator stdout reads). + """ + if not hasattr(os, "dup"): # pragma: no cover - jython legacy + raise RuntimeError("Trio worker requires os.dup") + + try: + devnull = os.devnull + except AttributeError: + devnull = "NUL" if os.name == "nt" else "/dev/null" + + # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) + read_fd = os.dup(0) + fd = os.open(devnull, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + + # Protocol write end: former stdout + write_fd = os.dup(1) + fd = os.open(devnull, os.O_WRONLY) + os.dup2(fd, 1) + + if os.name == "nt": + # Match init_popen_io: keep a stderr handle then point fd 2 at null. + sys.stderr = os.fdopen(os.dup(2), "w", 1) + os.dup2(fd, 2) + os.close(fd) + + # Replace sys.stdin/out with the null fds (closefd=False). + sys.stdin = os.fdopen(0, "r", 1, closefd=False) + sys.stdout = os.fdopen(1, "w", 1, closefd=False) + return read_fd, write_fd + + +class _WorkerIOStub: + """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" + + def __init__(self, execmodel: gateway_base.ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not used on Trio worker") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not used on Trio worker") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def serve_popen_trio(id: str, execmodel: str = "thread") -> None: + """Serve a WorkerGateway with Message IO on a Trio host thread.""" + from . import _trio_host + + model = get_execmodel(execmodel) + read_fd, write_fd = _prepare_protocol_fds() + # Keep the historic trace token so tests looking for workergateway still pass. + trace(f"creating workergateway on trio id={id!r}") + + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + try: + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + hasprimary = model.backend in ("thread", "main_thread_only") + gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + gateway._executetask_complete = None + if model.backend == "main_thread_only": + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + + async def _start() -> _trio_host.ProtocolSession: + async_io = _trio_host.FdStreamsIO(read_fd, write_fd) + return await host.start_session(gateway, async_io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if hasprimary: + trace("integrating as primary thread (trio worker)") + gateway._execpool.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 1d3eb59d..360b9865 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,11 +26,21 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__(self, io: IO, spec: XSpec) -> None: + def __init__( + self, + io: IO, + spec: XSpec, + *, + trio_session: object | None = None, + defer_receive: bool = False, + ) -> None: """:private:""" super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - self._initreceive() + if trio_session is not None: + self._attach_trio_session(trio_session) + elif not defer_receive: + self._initreceive() @property def remoteaddress(self) -> str: @@ -91,6 +101,9 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" + session = self._trio_session + if session is not None: + return session.is_alive() return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 73dc7175..0829a33d 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -563,6 +563,23 @@ def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.channelid = channelid self.data = data + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + @staticmethod def from_io(io: ReadIO) -> Message: try: @@ -571,12 +588,11 @@ def from_io(io: ReadIO) -> Message: raise EOFError("empty read") except EOFError as e: raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = struct.unpack("!bii", header) + msgtype, channel, payload = Message.from_header(header) return Message(msgtype, channel, io.read(payload)) def to_io(self, io: WriteIO) -> None: - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - io.write(header + self.data) + io.write(self.pack()) def received(self, gateway: BaseGateway) -> None: handler = self._types[self.msgcode][1] @@ -1125,6 +1141,7 @@ def readline(self) -> str: class BaseGateway: _sysex = sysex id = "" + _trio_session: Any = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1137,11 +1154,18 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.__trace = trace self._geterrortext = geterrortext self._receivepool = WorkerPool(self.execmodel) + self._trio_session = None def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) + def _attach_trio_session(self, session: Any) -> None: + """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + self._trio_session = session + def _initreceive(self) -> None: + if self._trio_session is not None: + return self._receivepool.spawn(self._thread_receiver) def _thread_receiver(self) -> None: @@ -1181,6 +1205,15 @@ def _terminate_execution(self) -> None: def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return try: message.to_io(self._io) self._trace("sent", message) @@ -1204,6 +1237,10 @@ def newchannel(self) -> Channel: def join(self, timeout: float | None = None) -> None: """Wait for receiverthread to terminate.""" self._trace("waiting for receiver thread to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + return self._receivepool.waitall(timeout) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4dbf8b89..8079f4c4 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -52,6 +52,7 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] + self._trio_host: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -62,6 +63,14 @@ def __init__( self.makegateway(xspec) atexit.register(self._cleanup_atexit) + def _ensure_trio_host(self) -> Any: + if self._trio_host is None: + from . import _trio_host + + self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") + self._trio_host.start() + return self._trio_host + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -143,7 +152,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend - if spec.via: + from . import _trio_host + + if _trio_host.should_use_trio_popen(spec): + gw = _trio_host.makegateway_popen_trio(self, spec) + elif spec.via: assert not spec.socket master = self[spec.via] proxy_channel = master.remote_exec(gateway_io) @@ -207,6 +220,9 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._trio_host is not None: + self._trio_host.stop(timeout=1.0) + self._trio_host = None def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination diff --git a/uv.lock b/uv.lock index 893f1543..fc2da46b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -333,6 +342,9 @@ wheels = [ [[package]] name = "execnet" source = { editable = "." } +dependencies = [ + { name = "trio" }, +] [package.optional-dependencies] testing = [ @@ -358,6 +370,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, { name = "tox", marker = "extra == 'testing'" }, + { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] provides-extras = ["testing"] @@ -723,6 +736,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -977,6 +1002,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1072,6 +1115,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/3d/7ba55871e9d794d40b6c8424f2e5d1b267ea5d9a4bd2175e08b57960ba13/tox-4.58.0-py3-none-any.whl", hash = "sha256:dcae21f5f015f3a67658e35644cce0d1aa0dedcd06f3927f95d84e1717f6cea5", size = 223298, upload-time = "2026-07-21T13:10:34.731Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + [[package]] name = "trove-classifiers" version = "2025.5.9.12" From beb36b92d05bab211017e5324519e05eb37faba7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:26:19 +0200 Subject: [PATCH 02/66] feat: schedule worker remote_exec from the Trio nursery Replace WorkerPool on the Trio popen worker with TrioWorkerExec: thread-model tasks run via trio.to_thread, main_thread_only hands off to the process main thread. Also wake the protocol writer with a Trio Event instead of blocking to_thread queue.get. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- doc/implnotes.rst | 5 +- src/execnet/_trio_host.py | 81 +++++++++++++----- src/execnet/_trio_worker.py | 159 +++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 7 ++ 4 files changed, 217 insertions(+), 35 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 3dae82d6..c8229865 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -26,8 +26,9 @@ event loop (``execnet._trio_host.TrioHost``): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` still runs on the existing WorkerPool - (including ``main_thread_only`` primary-thread integration). + streams; ``remote_exec`` is scheduled from the Trio nursery + (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 6ece94c4..b0db9ad7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -178,6 +178,7 @@ def __init__( self.process = process self.host = host self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._wake: trio.Event | None = None self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() @@ -200,6 +201,7 @@ def enqueue_message(self, message: Message) -> None: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") self._outbound.put((message.pack(), done, errors)) + self._wake_writer() if done is None: return if not done.wait(timeout=120.0): @@ -207,12 +209,25 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def _wake_writer(self) -> None: + wake = self._wake + if wake is None: + return + if self.host.is_host_thread(): + wake.set() + else: + try: + self.host.call_sync(wake.set) + except Exception: + pass + def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -294,31 +309,47 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: + self._wake = trio.Event() while True: - # abandon_on_cancel: queue.get blocks forever until a sentinel; - # nursery shutdown must not wait on it. - item = await trio.to_thread.run_sync( - self._outbound.get, abandon_on_cancel=True - ) - if item is _CLOSE_WRITE: + while True: try: - await self.io.aclose_write() - except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + item = self._outbound.get_nowait() + except queue.Empty: + break + if not await self._writer_handle_item(item): + return + # Reset wake before re-check to avoid losing a notification. + self._wake = trio.Event() + try: + item = self._outbound.get_nowait() + except queue.Empty: + await self._wake.wait() + continue + if not await self._writer_handle_item(item): return - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) + + async def _writer_handle_item(self, item: object) -> bool: + """Handle one outbound queue item. Return False when writer should stop.""" + if item is _CLOSE_WRITE: try: - await self.io.write_all(blob) + await self.io.aclose_write() except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() + self.gateway._trace("aclose_write failed", exc) + return False + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + return True async def _supervisor(self) -> None: assert self.process is not None @@ -329,10 +360,10 @@ async def _supervisor(self) -> None: async def _finish(self) -> None: gateway = self.gateway - # Unblock a writer thread left in queue.get after cancel. with self._lock: self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown @@ -403,6 +434,14 @@ def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: raise RuntimeError("TrioHost is not running") return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: + """Schedule a task on the root nursery (must be called on the host thread).""" + if not self.is_host_thread(): + raise RuntimeError("start_soon requires the Trio host thread") + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + self._nursery.start_soon(async_fn, *args) + async def start_session( self, gateway: BaseGateway, diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 70338f1c..6d247194 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -1,15 +1,151 @@ -"""Worker-side Trio networking entry for popen/import bootstrap.""" +"""Worker-side Trio networking and exec scheduling for popen/import bootstrap.""" from __future__ import annotations import os +import queue import sys +import threading +from typing import TYPE_CHECKING +from typing import Any + +import trio from . import gateway_base +from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import loads_internal from .gateway_base import trace +if TYPE_CHECKING: + from . import _trio_host + from .gateway_base import Channel + from .gateway_base import ExecModel + +ExecItem = tuple[Any, ...] + + +class TrioWorkerExec: + """Schedule ``remote_exec`` work from the Trio host nursery. + + * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). + * ``main_thread_only``: hand off to the process main thread (GUI-safe). + """ + + def __init__( + self, + host: _trio_host.TrioHost, + gateway: WorkerGateway, + *, + main_thread_only: bool, + ) -> None: + self.host = host + self.gateway = gateway + self.main_thread_only = main_thread_only + self._lock = threading.Lock() + self._running = 0 + self._shutting_down = False + self._idle = threading.Event() + self._idle.set() + self._primary_q: queue.SimpleQueue[ + tuple[Channel, ExecItem, threading.Event] | None + ] = queue.SimpleQueue() + self._primary_wake = threading.Event() + # Serialize main_thread_only admission (wait+clear) so two tasks cannot + # both observe the idle Event before either clears it. + self._admit_lock = trio.Lock() + + def active_count(self) -> int: + with self._lock: + return self._running + + def _track_start(self) -> None: + with self._lock: + self._running += 1 + self._idle.clear() + + def _track_finish(self) -> None: + with self._lock: + self._running -= 1 + if self._running == 0: + self._idle.set() + + def schedule(self, channel: Channel, sourcetask: bytes) -> None: + """Called from the Trio receiver while holding ``_receivelock``. + + Must not block: deadlock checks and exec run in a nursery task. + """ + item = loads_internal(sourcetask) + assert isinstance(item, tuple) + with self._lock: + if self._shutting_down: + channel.close("execution disallowed") + return + # Already on the Trio host thread (Message handler). + self.host.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + + def _wait_slot() -> bool: + return complete.wait(timeout=1) + + async with self._admit_lock: + if not await trio.to_thread.run_sync( + _wait_slot, abandon_on_cancel=True + ): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return + complete.clear() + + self._track_start() + try: + if self.main_thread_only: + done = threading.Event() + self._primary_q.put((channel, item, done)) + self._primary_wake.set() + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + else: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + finally: + self._track_finish() + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running main_thread_only exec tasks.""" + while True: + self._primary_wake.wait() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + self._primary_wake.clear() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + continue + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + with self._lock: + self._shutting_down = True + self._primary_q.put(None) + self._primary_wake.set() + + def waitall(self, timeout: float | None = None) -> bool: + return self._idle.wait(timeout) + def _prepare_protocol_fds() -> tuple[int, int]: """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. @@ -25,24 +161,20 @@ def _prepare_protocol_fds() -> tuple[int, int]: except AttributeError: devnull = "NUL" if os.name == "nt" else "/dev/null" - # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) read_fd = os.dup(0) fd = os.open(devnull, os.O_RDONLY) os.dup2(fd, 0) os.close(fd) - # Protocol write end: former stdout write_fd = os.dup(1) fd = os.open(devnull, os.O_WRONLY) os.dup2(fd, 1) if os.name == "nt": - # Match init_popen_io: keep a stderr handle then point fd 2 at null. sys.stderr = os.fdopen(os.dup(2), "w", 1) os.dup2(fd, 2) os.close(fd) - # Replace sys.stdin/out with the null fds (closefd=False). sys.stdin = os.fdopen(0, "r", 1, closefd=False) sys.stdout = os.fdopen(1, "w", 1, closefd=False) return read_fd, write_fd @@ -51,7 +183,7 @@ def _prepare_protocol_fds() -> tuple[int, int]: class _WorkerIOStub: """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" - def __init__(self, execmodel: gateway_base.ExecModel) -> None: + def __init__(self, execmodel: ExecModel) -> None: self.execmodel = execmodel def read(self, numbytes: int) -> bytes: @@ -74,7 +206,7 @@ def kill(self) -> None: def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO on a Trio host thread.""" + """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" from . import _trio_host model = get_execmodel(execmodel) @@ -88,10 +220,13 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - hasprimary = model.backend in ("thread", "main_thread_only") - gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec gateway._executetask_complete = None - if model.backend == "main_thread_only": + if main_thread_only: gateway._executetask_complete = model.Event() gateway._executetask_complete.set() @@ -103,9 +238,9 @@ async def _start() -> _trio_host.ProtocolSession: gateway._attach_trio_session(session) try: - if hasprimary: + if main_thread_only: trace("integrating as primary thread (trio worker)") - gateway._execpool.integrate_as_primary_thread() + trio_exec.integrate_as_primary_thread() gateway.join() except KeyboardInterrupt: # Match WorkerGateway.serve(): swallow in the worker. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0829a33d..8cf3cb09 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1245,7 +1245,14 @@ def join(self, timeout: float | None = None) -> None: class WorkerGateway(BaseGateway): + _trio_exec: Any = None + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = getattr(self, "_trio_exec", None) + if trio_exec is not None: + trio_exec.schedule(channel, sourcetask) + return + if self._execpool.execmodel.backend == "main_thread_only": assert self._executetask_complete is not None # It's necessary to wait for a short time in order to ensure From 37a184ff425f1f560e5da21d8c4ac4ceace069f0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 21:53:03 +0200 Subject: [PATCH 03/66] refactor: drop eventlet and gevent execmodels Remove the EventletExecModel and GeventExecModel backends and their get_execmodel branches, leaving only the stdlib thread and main_thread_only models. Narrow the test execmodel fixture, drop the gevent test dependency and the eventlet/gevent mypy overrides, and scrub the docs of the removed backends. This is phase 1 of moving execnet onto Trio: the ExecModel surface is kept intact for now and collapsed further once Trio drives IO on both sides. Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 23 ++--- pyproject.toml | 8 -- src/execnet/gateway_base.py | 121 ------------------------- src/execnet/multi.py | 2 +- testing/conftest.py | 12 +-- testing/test_channel.py | 2 - testing/test_gateway.py | 30 ------- testing/test_multi.py | 2 +- testing/test_xspec.py | 4 +- uv.lock | 172 +----------------------------------- 10 files changed, 15 insertions(+), 361 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index 78672647..723de83f 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -57,10 +57,6 @@ Examples for valid gateway specifications same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. -* ``popen//execmodel=eventlet`` specifies a subprocess that uses the - same interpreter as the one it is initiated from but will run the - other side using eventlet for handling IO and dispatching threads. - * ``socket=192.168.1.4:8888`` specifies a Python Socket server process that listens on ``192.168.1.4:8888`` @@ -138,30 +134,29 @@ processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. -threading models: gevent, eventlet, thread, main_thread_only +threading models: thread, main_thread_only ==================================================================== .. versionadded:: 1.2 (status: experimental!) -execnet supports "main_thread_only", "thread", "eventlet" and "gevent" -as thread models on each of the two sides. You need to decide which -model to use before you create any gateways:: +execnet supports "thread" and "main_thread_only" as thread models on +each of the two sides. You need to decide which model to use before +you create any gateways:: # content of threadmodel.py import execnet - # locally use "eventlet", remotely use "thread" model - execnet.set_execmodel("eventlet", "thread") + # locally use "thread", remotely use "main_thread_only" model + execnet.set_execmodel("thread", "main_thread_only") gw = execnet.makegateway() print (gw) print (gw.remote_status()) print (gw.remote_exec("channel.send(1)").receive()) -You need to have eventlet installed in your environment and then -you can execute this little test file:: +You can execute this little test file:: $ python threadmodel.py - - + + 1 How to execute in the main thread diff --git a/pyproject.toml b/pyproject.toml index 709f887b..2c73a32b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ testing = [ "tox", "hatch", "uv", - "gevent", ] [dependency-groups] @@ -122,10 +121,3 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false - -[[tool.mypy.overrides]] -module = [ - "eventlet.*", - "gevent.thread.*", -] -ignore_missing_imports = true diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8cf3cb09..c9933fe0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -179,123 +179,6 @@ class MainThreadOnlyExecModel(ThreadExecModel): backend = "main_thread_only" -class EventletExecModel(ExecModel): - backend = "eventlet" - - @property - def queue(self): - import eventlet - - return eventlet.queue - - @property - def subprocess(self): - import eventlet.green.subprocess - - return eventlet.green.subprocess - - @property - def socket(self): - import eventlet.green.socket - - return eventlet.green.socket - - def get_ident(self) -> int: - import eventlet.green.thread - - return eventlet.green.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import eventlet - - eventlet.sleep(delay) - - def start(self, func, args=()) -> None: - import eventlet - - eventlet.spawn_n(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import eventlet.green.os - - return eventlet.green.os.fdopen(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def RLock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def Event(self): - import eventlet.green.threading - - return eventlet.green.threading.Event() - - -class GeventExecModel(ExecModel): - backend = "gevent" - - @property - def queue(self): - import gevent.queue - - return gevent.queue - - @property - def subprocess(self): - import gevent.subprocess - - return gevent.subprocess - - @property - def socket(self): - import gevent - - return gevent.socket - - def get_ident(self) -> int: - import gevent.thread - - return gevent.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import gevent - - gevent.sleep(delay) - - def start(self, func, args=()) -> None: - import gevent - - gevent.spawn(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import gevent.fileobject - - # Prefer FileObject (FileObjectPosix on Unix). FileObjectThread keeps a - # native threadpool alive and can prevent interpreter shutdown, which - # hangs tests/scripts that open stdio via init_popen_io and then exit. - return gevent.fileobject.FileObject(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import gevent.lock - - return gevent.lock.RLock() - - def RLock(self): - import gevent.lock - - return gevent.lock.RLock() - - def Event(self): - import gevent.event - - return gevent.event.Event() - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend @@ -303,10 +186,6 @@ def get_execmodel(backend: str | ExecModel) -> ExecModel: return ThreadExecModel() elif backend == "main_thread_only": return MainThreadOnlyExecModel() - elif backend == "eventlet": - return EventletExecModel() - elif backend == "gevent": - return GeventExecModel() else: raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 8079f4c4..687f1840 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -138,7 +138,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread', 'main_thread_only', 'eventlet', 'gevent' execution model + execmodel=model 'thread' or 'main_thread_only' execution model chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. diff --git a/testing/conftest.py b/testing/conftest.py index c75f96c7..4d97bf35 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -128,10 +128,6 @@ def anypython(request: pytest.FixtureRequest) -> str: executable = getexecutable(name) if executable is None: pytest.skip(f"no {name} found") - if "execmodel" in request.fixturenames and name != "sys.executable": - backend = request.getfixturevalue("execmodel").backend - if backend not in ("thread", "main_thread_only"): - pytest.xfail(f"cannot run {backend!r} execmodel with bare {name}") return executable @@ -182,14 +178,8 @@ def gw( return gw -@pytest.fixture( - params=["thread", "main_thread_only", "eventlet", "gevent"], scope="session" -) +@pytest.fixture(params=["thread", "main_thread_only"], scope="session") def execmodel(request: pytest.FixtureRequest) -> ExecModel: - if request.param not in ("thread", "main_thread_only"): - pytest.importorskip(request.param) - if request.param in ("eventlet", "gevent") and sys.platform == "win32": - pytest.xfail(request.param + " does not work on win32") return get_execmodel(request.param) diff --git a/testing/test_channel.py b/testing/test_channel.py index d4712277..ba02b57b 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -214,8 +214,6 @@ def test_channel_callback_stays_active(self, gw: Gateway) -> None: def check_channel_callback_stays_active( self, gw: Gateway, earlyfree: bool = True ) -> Channel | None: - if gw.spec.execmodel == "gevent": - pytest.xfail("investigate gevent failure") # with 'earlyfree==True', this tests the "sendonly" channel state. l: list[int] = [] channel = gw.remote_exec( diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 634f237a..83a50f44 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -535,36 +535,6 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args == expected_args -@pytest.mark.parametrize( - "interleave_getstatus", - [ - pytest.param(True, id="interleave-remote-status"), - pytest.param( - False, - id="no-interleave-remote-status", - marks=pytest.mark.xfail( - reason="https://github.com/pytest-dev/execnet/issues/123", - ), - ), - ], -) -def test_regression_gevent_hangs( - group: execnet.Group, interleave_getstatus: bool -) -> None: - pytest.importorskip("gevent") - gw = group.makegateway("popen//execmodel=gevent") - - print(gw.remote_status()) - - def sendback(channel) -> None: - channel.send(1234) - - ch = gw.remote_exec(sendback) - if interleave_getstatus: - print(gw.remote_status()) - assert ch.receive(timeout=0.5) == 1234 - - def test_assert_main_thread_only( execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: diff --git a/testing/test_multi.py b/testing/test_multi.py index 12e0ed3d..f3166503 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -69,7 +69,7 @@ def test_Group_execmodel_setting(self) -> None: gm._gateways.append(1) # type: ignore[arg-type] try: with pytest.raises(ValueError): - gm.set_execmodel("eventlet") + gm.set_execmodel("main_thread_only") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop() diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f837b07a..5240d72d 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -64,8 +64,8 @@ def test_ssh_options(self) -> None: def test_execmodel(self) -> None: spec = XSpec("execmodel=thread") assert spec.execmodel == "thread" - spec = XSpec("execmodel=eventlet") - assert spec.execmodel == "eventlet" + spec = XSpec("execmodel=main_thread_only") + assert spec.execmodel == "main_thread_only" def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") diff --git a/uv.lock b/uv.lock index fc2da46b..5dbf4f10 100644 --- a/uv.lock +++ b/uv.lock @@ -332,7 +332,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,7 +348,6 @@ dependencies = [ [package.optional-dependencies] testing = [ - { name = "gevent" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -364,7 +363,6 @@ testing = [ [package.metadata] requires-dist = [ - { name = "gevent", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -387,118 +385,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] -[[package]] -name = "gevent" -version = "26.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, - { name = "zope-event" }, - { name = "zope-interface" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/b7/01a5880e01702f39fb09e3616c624054a0dc9a82561a865f3b1eff4bfc80/gevent-26.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2ba673dcbf7747513b58fa64ca7e9d6a828bc5c604d1552d23db89006d7911df", size = 2181491, upload-time = "2026-05-20T20:35:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fe/035ec5fa58a886740a744380118f03a90ac2da3f6c9cba248f28074ce40a/gevent-26.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:271b1474d81bb33036631adb16a35e5a1ee9dc414b05c999d6b01dc839a89975", size = 2212161, upload-time = "2026-05-20T20:43:25.678Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ea/ea87c08931c9e4c6c40bb05a2cb19c2d6f93fe6e0052f9152ea5ade6d037/gevent-26.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cd3dc60581687e2618286108f8e2f820d8446be4b34131065011c066e911d39c", size = 1768295, upload-time = "2026-05-20T21:17:29.438Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1d0e7287ae55700a8d25153ac736896bd9bcc3f85a12d374ef398db4b33c/gevent-26.5.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:dc7fa28b2d627f8e87595f39043b6dec71e8e7fb97e685e5506c47cf3ff8cb2e", size = 1862627, upload-time = "2026-05-20T21:15:59.365Z" }, - { url = "https://files.pythonhosted.org/packages/3a/4c/7f5ed67e52dfdef4ff91ae1a6fb28186d52e2496962edc8f17bdea9ab2c0/gevent-26.5.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:68c5fc21cef80268cdff88a4ae6c025fabb019b071f6f8ee4d20a7bccbddb873", size = 1804690, upload-time = "2026-05-20T21:30:51.713Z" }, - { url = "https://files.pythonhosted.org/packages/4c/75/0f5da6ca045f8a052203e1810058029f4b682507a789b413cac7d28bae28/gevent-26.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d325502eb0695708ef8c899f605573ed6847f3961f8159627dba267fbf3ce457", size = 2119054, upload-time = "2026-05-20T20:35:22.678Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f1/fcff7f7fad2bb33f3742db6b2145825a2191c0cd31d75789b0741fd28faf/gevent-26.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a11daf3a588b932c8bf965fb18444c69aff48badec88435e988cf8d67137075a", size = 1778784, upload-time = "2026-05-20T21:16:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/98/57/151314f00bdc6ba77333febb3e9dc97fdf94d79426559b4fa8332f0c2b6e/gevent-26.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1101b5ef82a3fb178550cfd80f32293dc8dd2f3d0828292223ebba29d6f76e33", size = 2145373, upload-time = "2026-05-20T20:43:27.255Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b5/7a02f711db62cbed1c1a00e1f9ff50eef95ccc78d4c04a0f93636655d1b7/gevent-26.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:5233109ad4f3af16393ba9888f238919a05ce15ce68d6831ac8a0da8dfb750ae", size = 1696576, upload-time = "2026-05-20T20:15:49.62Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/5022adc310697ef25c6fb22eb9bf0ebcad3427b51776e882709de9a8b6d7/gevent-26.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:3be804565168ffacebeb21af9f1cd689831a89f0f12fc0c3f423c730c3c9eb31", size = 1552095, upload-time = "2026-05-20T20:16:54.81Z" }, - { url = "https://files.pythonhosted.org/packages/37/0b/1a530b2db55c97cc0cf44116201f538f3033c04c1d2aca143979b412f4be/gevent-26.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:e80ad2a8a1e8bdaa5605e3bf4929e0cebf9ea7b8237c83362f7257698bb14280", size = 2929714, upload-time = "2026-05-20T20:13:24.656Z" }, - { url = "https://files.pythonhosted.org/packages/b9/df/32fe851ed5f68493f354e09b19bdebae0de1185be4db0b2988e71e737fd3/gevent-26.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fe42c037253580a3386fce275f8a2a845e540f5a729916934a732f13d42e72cc", size = 1784838, upload-time = "2026-05-20T21:17:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9a/21332674f9a10e8cdf13b41b52e9d663647a1c6e1dc3c62b07c0aeefd360/gevent-26.5.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:9f463c7d6f69d13b6fe8e3b832a6175a6e95328a940f38495d25496d1ae8ad88", size = 1880440, upload-time = "2026-05-20T21:16:00.881Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b1/5f8a4196113cf7f3fdd987b483f7e6b10c28ea3930c4727e31ba8cce51b6/gevent-26.5.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:96d5e96b1b14a4c1023dcfcc114533217f13febc3b6169254f23fc18d19fee29", size = 1831592, upload-time = "2026-05-20T21:30:53.832Z" }, - { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, - { url = "https://files.pythonhosted.org/packages/e4/32/602c499d54472f64e5cdf6013aeab5ce6aa6fed005387e8b4f2d22f5dc8d/gevent-26.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f519139354d5ca7625df9ddb1b2ffada885c14abc5b4dbae3682e967ddf79669", size = 1796906, upload-time = "2026-05-20T21:16:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/22/d5/4620797bbd9c88f4541188efc138b0d615f9834db540da36a2249ee929c5/gevent-26.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e49ce0de007dfd7412edbc2b5d41cce33b049bb1b7086f50be5a09e601bde603", size = 1699995, upload-time = "2026-05-20T20:15:39.311Z" }, - { url = "https://files.pythonhosted.org/packages/cb/83/ac3477dfc0f9fd80c88110102c73cefc35dcded2b248544f45a8fa5412df/gevent-26.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c5ff29495a2eed2a244de8150f21893d6c1b15d8b4b5719ab4bbfa06db1e28f", size = 1547433, upload-time = "2026-05-20T20:15:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/7d/47/5b992ab9c8037633cfd0fe698a97a878f59d8eb53c381e91e9a1a76fd215/gevent-26.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9b4d3f34c913d1a6bec6d030365a517f3b527a9773b12e58cf56c3339bbe96e6", size = 2952523, upload-time = "2026-05-20T20:13:04.698Z" }, - { url = "https://files.pythonhosted.org/packages/74/11/c7dfc773eb43331a682efed610b49df6e976331f1b0e1c592a0c35d29872/gevent-26.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1d8da4e799431feeb4c9e441ac7431f0baabb9106976790d884289d08ac08359", size = 1787044, upload-time = "2026-05-20T21:17:32.845Z" }, - { url = "https://files.pythonhosted.org/packages/ae/28/9812933dac93560f46910a9e834805fe76f822c408bd1c20cdf299d7c311/gevent-26.5.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:51becdb4c30a8f45c1c028ad7a97bf5a1ed141f74b159a31aa9cc6aa1e6263a6", size = 1882342, upload-time = "2026-05-20T21:16:02.645Z" }, - { url = "https://files.pythonhosted.org/packages/96/4b/514f248f69b2230b69b0bb17f4158b0b05dd4b2cb469a60ab206e9fe7496/gevent-26.5.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:c42bbcd3d453b08ad8915fd3feaf3d44a3562cdf1c7b208f9837149711e16d9d", size = 1834136, upload-time = "2026-05-20T21:30:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/53/67/f5f30716efca99b6200ae89a9303a7e94dae085b7de6f6d0033c52a37f4b/gevent-26.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bd3445e4fbeeb46690ed8efe94b8d1d46b14aa04af8866ae7a8da5997828d1c6", size = 2115349, upload-time = "2026-05-20T20:35:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/09/d8/60e8809bde7986e6c4e6d106080b3603fa09b3bb0255fed1a4d8282e3ca2/gevent-26.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b573d5b2826edc705f31f07da6889ad483a6a0d64944ebd8d32205f7c5bf46fb", size = 1799443, upload-time = "2026-05-20T21:16:41.928Z" }, - { url = "https://files.pythonhosted.org/packages/f8/41/b388b2b1f0a026ea30687e51ddf81dbb783dfb55fac0a16708d2821d99e5/gevent-26.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d53b1b28f2082a151bded2850b53f6baed02f742d2a1584029e8bd42d457fb4", size = 2141117, upload-time = "2026-05-20T20:43:30.694Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f3/ac9a4b0de487e390c5d53a908a9347c0df0102de2bbf3e8603087769191d/gevent-26.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:23569ce0c254eb821fc3dcfe250843dde8b3180b09bae9e222e41aa3fa4885b7", size = 1699862, upload-time = "2026-05-20T20:15:33.642Z" }, - { url = "https://files.pythonhosted.org/packages/2a/cf/1ef1fc9b390563c0f97702f94a557d1649b7bbb5724f9b86c2122747e92f/gevent-26.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:40cdcdb2e404b6c82b82a4576bdb33958f23fc2deb0d933e9e022b362001e647", size = 1545341, upload-time = "2026-05-20T20:16:26.229Z" }, - { url = "https://files.pythonhosted.org/packages/17/55/7d98d3888e7bb9ad4656420dec69232ecbbea48792aff9295d0ad7cf8435/gevent-26.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:75a0050e4b87f08ddee7e56f59e6014cd7fcdc3153046c09a847940515d12c85", size = 2968223, upload-time = "2026-05-20T20:13:17.223Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b4/e8e116fcbcb9dc0bf3acc50037f86e1204c217c8ed5defde68be11b3aab6/gevent-26.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fd1a0b83a04e19378d9466ae0ee2b5937cf1d7fbfdcb916b2aea82179a208574", size = 1793926, upload-time = "2026-05-20T21:17:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/7b267e9754b661defb93542e97731a4df21f8a40dc0f6c853faa717cf124/gevent-26.5.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:4c964c15076e76391d523ec24202f579a2535f7e301a40efb1656ae046d3eb69", size = 1887632, upload-time = "2026-05-20T21:16:04.158Z" }, - { url = "https://files.pythonhosted.org/packages/5c/50/b47d29e99449bd13b557ffa451401dc13d397a9923f562ef90a4e8514502/gevent-26.5.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:45d5438d1c84da5df7e832434627624709543630977332bb4e2d05ecca362cc9", size = 1838688, upload-time = "2026-05-20T21:30:57.979Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/5b54ccff11bc7d7bebd40a24571ccc115d5cdae4f6c32ab457b43b436e42/gevent-26.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:354f35924113abc954819216c2a6ee16751958c615681e0490946e31b437bd2f", size = 2120351, upload-time = "2026-05-20T20:35:32.699Z" }, - { url = "https://files.pythonhosted.org/packages/9c/70/30fd325c30e04b1e5174c61945e17421d53ddb2450366cc52cef234f8c4b/gevent-26.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a47cd2d32f6404212d374ad8014a3491d7477dcf0cc09c5a2308ad6d325fd663", size = 1806684, upload-time = "2026-05-20T21:16:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e8/fbf911ac3f9524ecfaed174d100fde671904ab8db92ceaf07faaebd13386/gevent-26.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:032157cebdedb84f2f52cdd980f2f5f2623eed6a8f083aadf44b44c47f628642", size = 2146606, upload-time = "2026-05-20T20:43:32.216Z" }, - { url = "https://files.pythonhosted.org/packages/9e/4d/284fcbbfde66fd978c2980c1fbe0eabd586af6e4b728649e9cf459e8b38f/gevent-26.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c414935ba5fc88359110968851d3616f119082c937390d00a1c0f4f59be814f", size = 1722497, upload-time = "2026-05-20T20:16:44.274Z" }, - { url = "https://files.pythonhosted.org/packages/15/d2/9f66eb53434704402be0ba733bf3320bf589671a4b76fac52a7d6077e972/gevent-26.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:2a0f5993a04b95a35b3a118b1a58ba272833f9b547b774001dea29f90620882f", size = 1574249, upload-time = "2026-05-20T20:15:50.873Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d5/b4c50adb761878e3c96642b9f79bf44cee3120f3df55cd40876f51d89866/gevent-26.5.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2e117df896a2660c9ebd4e2b5afc02dfd6e2ddf9b495e787e67c72d105432b09", size = 2971993, upload-time = "2026-05-20T20:12:50.845Z" }, - { url = "https://files.pythonhosted.org/packages/03/83/71c2a945e80198422d1d93dbe67355f249fb456b451bf9201199d3ef6a1a/gevent-26.5.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:af5ffe9c11ffb8a39b6bef2e8b722aa2043ae4980977915c6aa8c68b4bc26e46", size = 1796658, upload-time = "2026-05-20T21:17:35.968Z" }, - { url = "https://files.pythonhosted.org/packages/42/96/548ca77aed5cb9a44e855a6c23ebceeb3554a0ea9ca0c01c311878899a3e/gevent-26.5.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:7da34aef7e87c43dd3662e5785e79ed505c01399a7cb42876d2d8925969fd75f", size = 1891473, upload-time = "2026-05-20T21:16:05.657Z" }, - { url = "https://files.pythonhosted.org/packages/f6/4f/f48bd47d5287afb0fbcc56165f3ed47583f1803bad401653fe27e71ade2d/gevent-26.5.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:1c6293a7046bcc6f3d8972a74b19cd7a4cfd02d3881edf0fcf827aa514bd247b", size = 1841429, upload-time = "2026-05-20T21:30:59.907Z" }, - { url = "https://files.pythonhosted.org/packages/a0/72/1925215fc720d2561fa3ec8d4af5f098f8d0cbfa76a45fafed6e5ade7718/gevent-26.5.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d3bde0f140a275b2fa88e4b6516bda85551930e10bc2fd95e18c1b7d11cb780c", size = 2123895, upload-time = "2026-05-20T20:35:34.964Z" }, - { url = "https://files.pythonhosted.org/packages/83/59/0f584f6b1170c9a6abd9b70ccf5e9cc5ead34eabafabc0e21876ef0fe6f7/gevent-26.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e29fb4b17d9958ec8cb7f6339a111b29bc23f2c2efbef86189d1248bb4862d17", size = 1809047, upload-time = "2026-05-20T21:16:45.977Z" }, - { url = "https://files.pythonhosted.org/packages/82/88/61e854bfd98ac22eac78a97fc6db10de0f9ace46514072b435c217168729/gevent-26.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b2239df2f7570efa03736678f3f053bb1bdd22a8a16cd28a2feb7d32ea5f533f", size = 2150764, upload-time = "2026-05-20T20:43:33.781Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f5/af048b97433d7f9a7df7f5510b2c46918b7d073dcfb3bf6d0ef0e5a83dcc/gevent-26.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:aae214952fd38d27a42dc416bb70193962ec932384b63445d29bbb5817a1c042", size = 1722600, upload-time = "2026-05-20T20:19:56.81Z" }, - { url = "https://files.pythonhosted.org/packages/11/95/fb74a2299c6a2d78d9de12deaaac640ab5d2ef96a8e0f97a3ff84b9ca84b/gevent-26.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:f7067564f139e33bf26a31ee3b13d168d76eb99a44b85ced626652b158baa80c", size = 1574406, upload-time = "2026-05-20T20:17:12.125Z" }, -] - -[[package]] -name = "greenlet" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/c1/a82edae11d46c0d83481aacaa1e578fea21d94a1ef400afd734d47ad95ad/greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", size = 185797, upload-time = "2025-05-09T19:47:35.066Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/66/910217271189cc3f32f670040235f4bf026ded8ca07270667d69c06e7324/greenlet-3.2.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", size = 267395, upload-time = "2025-05-09T14:50:45.357Z" }, - { url = "https://files.pythonhosted.org/packages/a8/36/8d812402ca21017c82880f399309afadb78a0aa300a9b45d741e4df5d954/greenlet-3.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", size = 625742, upload-time = "2025-05-09T15:23:58.293Z" }, - { url = "https://files.pythonhosted.org/packages/7b/77/66d7b59dfb7cc1102b2f880bc61cb165ee8998c9ec13c96606ba37e54c77/greenlet-3.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", size = 637014, upload-time = "2025-05-09T15:24:47.025Z" }, - { url = "https://files.pythonhosted.org/packages/36/a7/ff0d408f8086a0d9a5aac47fa1b33a040a9fca89bd5a3f7b54d1cd6e2793/greenlet-3.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", size = 632874, upload-time = "2025-05-09T15:29:20.014Z" }, - { url = "https://files.pythonhosted.org/packages/a1/75/1dc2603bf8184da9ebe69200849c53c3c1dca5b3a3d44d9f5ca06a930550/greenlet-3.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", size = 631652, upload-time = "2025-05-09T14:53:30.961Z" }, - { url = "https://files.pythonhosted.org/packages/7b/74/ddc8c3bd4c2c20548e5bf2b1d2e312a717d44e2eca3eadcfc207b5f5ad80/greenlet-3.2.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", size = 580619, upload-time = "2025-05-09T14:53:42.049Z" }, - { url = "https://files.pythonhosted.org/packages/7e/f2/40f26d7b3077b1c7ae7318a4de1f8ffc1d8ccbad8f1d8979bf5080250fd6/greenlet-3.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", size = 1109809, upload-time = "2025-05-09T15:26:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/c5/21/9329e8c276746b0d2318b696606753f5e7b72d478adcf4ad9a975521ea5f/greenlet-3.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", size = 1133455, upload-time = "2025-05-09T14:53:55.823Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1e/0dca9619dbd736d6981f12f946a497ec21a0ea27262f563bca5729662d4d/greenlet-3.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", size = 294991, upload-time = "2025-05-09T15:05:56.847Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9f/a47e19261747b562ce88219e5ed8c859d42c6e01e73da6fbfa3f08a7be13/greenlet-3.2.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", size = 268635, upload-time = "2025-05-09T14:50:39.007Z" }, - { url = "https://files.pythonhosted.org/packages/11/80/a0042b91b66975f82a914d515e81c1944a3023f2ce1ed7a9b22e10b46919/greenlet-3.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", size = 628786, upload-time = "2025-05-09T15:24:00.692Z" }, - { url = "https://files.pythonhosted.org/packages/38/a2/8336bf1e691013f72a6ebab55da04db81a11f68e82bb691f434909fa1327/greenlet-3.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", size = 640866, upload-time = "2025-05-09T15:24:48.153Z" }, - { url = "https://files.pythonhosted.org/packages/f8/7e/f2a3a13e424670a5d08826dab7468fa5e403e0fbe0b5f951ff1bc4425b45/greenlet-3.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", size = 636752, upload-time = "2025-05-09T15:29:23.182Z" }, - { url = "https://files.pythonhosted.org/packages/fd/5d/ce4a03a36d956dcc29b761283f084eb4a3863401c7cb505f113f73af8774/greenlet-3.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", size = 636028, upload-time = "2025-05-09T14:53:32.854Z" }, - { url = "https://files.pythonhosted.org/packages/4b/29/b130946b57e3ceb039238413790dd3793c5e7b8e14a54968de1fe449a7cf/greenlet-3.2.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", size = 583869, upload-time = "2025-05-09T14:53:43.614Z" }, - { url = "https://files.pythonhosted.org/packages/ac/30/9f538dfe7f87b90ecc75e589d20cbd71635531a617a336c386d775725a8b/greenlet-3.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", size = 1112886, upload-time = "2025-05-09T15:27:01.304Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/4b7deeb1a1e9c32c1b59fdca1cac3175731c23311ddca2ea28a8b6ada91c/greenlet-3.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", size = 1138355, upload-time = "2025-05-09T14:53:58.011Z" }, - { url = "https://files.pythonhosted.org/packages/c5/eb/7551c751a2ea6498907b2fcbe31d7a54b602ba5e8eb9550a9695ca25d25c/greenlet-3.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", size = 295437, upload-time = "2025-05-09T15:00:57.733Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a1/88fdc6ce0df6ad361a30ed78d24c86ea32acb2b563f33e39e927b1da9ea0/greenlet-3.2.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", size = 270413, upload-time = "2025-05-09T14:51:32.455Z" }, - { url = "https://files.pythonhosted.org/packages/a6/2e/6c1caffd65490c68cd9bcec8cb7feb8ac7b27d38ba1fea121fdc1f2331dc/greenlet-3.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", size = 637242, upload-time = "2025-05-09T15:24:02.63Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/088af2cedf8823b6b7ab029a5626302af4ca1037cf8b998bed3a8d3cb9e2/greenlet-3.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", size = 651444, upload-time = "2025-05-09T15:24:49.856Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0116ab876bb0bc7a81eadc21c3f02cd6100dcd25a1cf2a085a130a63a26a/greenlet-3.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", size = 646067, upload-time = "2025-05-09T15:29:24.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/17/bb8f9c9580e28a94a9575da847c257953d5eb6e39ca888239183320c1c28/greenlet-3.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", size = 648153, upload-time = "2025-05-09T14:53:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ee/7f31b6f7021b8df6f7203b53b9cc741b939a2591dcc6d899d8042fcf66f2/greenlet-3.2.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", size = 603865, upload-time = "2025-05-09T14:53:45.738Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2d/759fa59323b521c6f223276a4fc3d3719475dc9ae4c44c2fe7fc750f8de0/greenlet-3.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", size = 1119575, upload-time = "2025-05-09T15:27:04.248Z" }, - { url = "https://files.pythonhosted.org/packages/30/05/356813470060bce0e81c3df63ab8cd1967c1ff6f5189760c1a4734d405ba/greenlet-3.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", size = 1147460, upload-time = "2025-05-09T14:54:00.315Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/b2a26a309a04fb844c7406a4501331b9400e1dd7dd64d3450472fd47d2e1/greenlet-3.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", size = 296239, upload-time = "2025-05-09T14:57:17.633Z" }, - { url = "https://files.pythonhosted.org/packages/89/30/97b49779fff8601af20972a62cc4af0c497c1504dfbb3e93be218e093f21/greenlet-3.2.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", size = 269150, upload-time = "2025-05-09T14:50:30.784Z" }, - { url = "https://files.pythonhosted.org/packages/21/30/877245def4220f684bc2e01df1c2e782c164e84b32e07373992f14a2d107/greenlet-3.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", size = 637381, upload-time = "2025-05-09T15:24:12.893Z" }, - { url = "https://files.pythonhosted.org/packages/8e/16/adf937908e1f913856b5371c1d8bdaef5f58f251d714085abeea73ecc471/greenlet-3.2.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", size = 651427, upload-time = "2025-05-09T15:24:51.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/49/6d79f58fa695b618654adac64e56aff2eeb13344dc28259af8f505662bb1/greenlet-3.2.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", size = 645795, upload-time = "2025-05-09T15:29:26.673Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e6/28ed5cb929c6b2f001e96b1d0698c622976cd8f1e41fe7ebc047fa7c6dd4/greenlet-3.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", size = 648398, upload-time = "2025-05-09T14:53:36.61Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/b200194e25ae86bc57077f695b6cc47ee3118becf54130c5514456cf8dac/greenlet-3.2.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", size = 606795, upload-time = "2025-05-09T14:53:47.039Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c8/ba1def67513a941154ed8f9477ae6e5a03f645be6b507d3930f72ed508d3/greenlet-3.2.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", size = 1117976, upload-time = "2025-05-09T15:27:06.542Z" }, - { url = "https://files.pythonhosted.org/packages/c3/30/d0e88c1cfcc1b3331d63c2b54a0a3a4a950ef202fb8b92e772ca714a9221/greenlet-3.2.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", size = 1145509, upload-time = "2025-05-09T14:54:02.223Z" }, - { url = "https://files.pythonhosted.org/packages/90/2e/59d6491834b6e289051b252cf4776d16da51c7c6ca6a87ff97e3a50aa0cd/greenlet-3.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421", size = 296023, upload-time = "2025-05-09T14:53:24.157Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/8a73aace5a5335a1cba56d0da71b7bd93e450f17d372c5b7c5fa547557e9/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", size = 629911, upload-time = "2025-05-09T15:24:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/48/08/c8b8ebac4e0c95dcc68ec99198842e7db53eda4ab3fb0a4e785690883991/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", size = 635251, upload-time = "2025-05-09T15:24:52.205Z" }, - { url = "https://files.pythonhosted.org/packages/37/26/7db30868f73e86b9125264d2959acabea132b444b88185ba5c462cb8e571/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", size = 632620, upload-time = "2025-05-09T15:29:28.051Z" }, - { url = "https://files.pythonhosted.org/packages/10/ec/718a3bd56249e729016b0b69bee4adea0dfccf6ca43d147ef3b21edbca16/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", size = 628851, upload-time = "2025-05-09T14:53:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/d1c79286a76bc62ccdc1387291464af16a4204ea717f24e77b0acd623b99/greenlet-3.2.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", size = 593718, upload-time = "2025-05-09T14:53:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/cd/41/96ba2bf948f67b245784cd294b84e3d17933597dffd3acdb367a210d1949/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", size = 1105752, upload-time = "2025-05-09T15:27:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/68/3b/3b97f9d33c1f2eb081759da62bd6162159db260f602f048bc2f36b4c453e/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", size = 1125170, upload-time = "2025-05-09T14:54:04.082Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b7d17d66c8d0f578d2885a3d8f565e9e4725eacc9d3fdc946d0031c055c4/greenlet-3.2.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", size = 269899, upload-time = "2025-05-09T14:54:01.581Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -984,15 +870,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", size = 15221, upload-time = "2022-08-13T16:22:44.457Z" }, ] -[[package]] -name = "setuptools" -version = "80.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/d2/ec1acaaff45caed5c2dedb33b67055ba9d4e96b091094df90762e60135fe/setuptools-80.8.0.tar.gz", hash = "sha256:49f7af965996f26d43c8ae34539c8d99c5042fbff34302ea151eaa9c207cd257", size = 1319720, upload-time = "2025-05-20T14:02:53.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/29/93c53c098d301132196c3238c312825324740851d77a8500a2462c0fd888/setuptools-80.8.0-py3-none-any.whl", hash = "sha256:95a60484590d24103af13b686121328cc2736bee85de8936383111e421b9edc0", size = 1201470, upload-time = "2025-05-20T14:02:51.348Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -1222,50 +1099,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] - -[[package]] -name = "zope-event" -version = "5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c2/427f1867bb96555d1d34342f1dd97f8c420966ab564d58d18469a1db8736/zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd", size = 17350, upload-time = "2023-06-23T06:28:35.709Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/42/f8dbc2b9ad59e927940325a22d6d3931d630c3644dae7e2369ef5d9ba230/zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", size = 6824, upload-time = "2023-06-23T06:28:32.652Z" }, -] - -[[package]] -name = "zope-interface" -version = "7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/71/e6177f390e8daa7e75378505c5ab974e0bf59c1d3b19155638c7afbf4b2d/zope.interface-7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce290e62229964715f1011c3dbeab7a4a1e4971fd6f31324c4519464473ef9f2", size = 208243, upload-time = "2024-11-28T08:47:29.781Z" }, - { url = "https://files.pythonhosted.org/packages/52/db/7e5f4226bef540f6d55acfd95cd105782bc6ee044d9b5587ce2c95558a5e/zope.interface-7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05b910a5afe03256b58ab2ba6288960a2892dfeef01336dc4be6f1b9ed02ab0a", size = 208759, upload-time = "2024-11-28T08:47:31.908Z" }, - { url = "https://files.pythonhosted.org/packages/28/ea/fdd9813c1eafd333ad92464d57a4e3a82b37ae57c19497bcffa42df673e4/zope.interface-7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550f1c6588ecc368c9ce13c44a49b8d6b6f3ca7588873c679bd8fd88a1b557b6", size = 254922, upload-time = "2024-11-28T09:18:11.795Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d3/0000a4d497ef9fbf4f66bb6828b8d0a235e690d57c333be877bec763722f/zope.interface-7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ef9e2f865721553c6f22a9ff97da0f0216c074bd02b25cf0d3af60ea4d6931d", size = 249367, upload-time = "2024-11-28T08:48:24.238Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e5/0b359e99084f033d413419eff23ee9c2bd33bca2ca9f4e83d11856f22d10/zope.interface-7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f926f0dcb058211a3bb3e0e501c69759613b17a553788b2caeb991bed3b61d", size = 254488, upload-time = "2024-11-28T08:48:28.816Z" }, - { url = "https://files.pythonhosted.org/packages/7b/90/12d50b95f40e3b2fc0ba7f7782104093b9fd62806b13b98ef4e580f2ca61/zope.interface-7.2-cp310-cp310-win_amd64.whl", hash = "sha256:144964649eba4c5e4410bb0ee290d338e78f179cdbfd15813de1a664e7649b3b", size = 211947, upload-time = "2024-11-28T08:48:18.831Z" }, - { url = "https://files.pythonhosted.org/packages/98/7d/2e8daf0abea7798d16a58f2f3a2bf7588872eee54ac119f99393fdd47b65/zope.interface-7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1909f52a00c8c3dcab6c4fad5d13de2285a4b3c7be063b239b8dc15ddfb73bd2", size = 208776, upload-time = "2024-11-28T08:47:53.009Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2a/0c03c7170fe61d0d371e4c7ea5b62b8cb79b095b3d630ca16719bf8b7b18/zope.interface-7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80ecf2451596f19fd607bb09953f426588fc1e79e93f5968ecf3367550396b22", size = 209296, upload-time = "2024-11-28T08:47:57.993Z" }, - { url = "https://files.pythonhosted.org/packages/49/b4/451f19448772b4a1159519033a5f72672221e623b0a1bd2b896b653943d8/zope.interface-7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:033b3923b63474800b04cba480b70f6e6243a62208071fc148354f3f89cc01b7", size = 260997, upload-time = "2024-11-28T09:18:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/5aa4461c10718062c8f8711161faf3249d6d3679c24a0b81dd6fc8ba1dd3/zope.interface-7.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a102424e28c6b47c67923a1f337ede4a4c2bba3965b01cf707978a801fc7442c", size = 255038, upload-time = "2024-11-28T08:48:26.381Z" }, - { url = "https://files.pythonhosted.org/packages/9f/aa/1a28c02815fe1ca282b54f6705b9ddba20328fabdc37b8cf73fc06b172f0/zope.interface-7.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25e6a61dcb184453bb00eafa733169ab6d903e46f5c2ace4ad275386f9ab327a", size = 259806, upload-time = "2024-11-28T08:48:30.78Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2c/82028f121d27c7e68632347fe04f4a6e0466e77bb36e104c8b074f3d7d7b/zope.interface-7.2-cp311-cp311-win_amd64.whl", hash = "sha256:3f6771d1647b1fc543d37640b45c06b34832a943c80d1db214a37c31161a93f1", size = 212305, upload-time = "2024-11-28T08:49:14.525Z" }, - { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, - { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3b/e309d731712c1a1866d61b5356a069dd44e5b01e394b6cb49848fa2efbff/zope.interface-7.2-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:3e0350b51e88658d5ad126c6a57502b19d5f559f6cb0a628e3dc90442b53dd98", size = 208961, upload-time = "2024-11-28T08:48:29.865Z" }, - { url = "https://files.pythonhosted.org/packages/49/65/78e7cebca6be07c8fc4032bfbb123e500d60efdf7b86727bb8a071992108/zope.interface-7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15398c000c094b8855d7d74f4fdc9e73aa02d4d0d5c775acdef98cdb1119768d", size = 209356, upload-time = "2024-11-28T08:48:33.297Z" }, - { url = "https://files.pythonhosted.org/packages/11/b1/627384b745310d082d29e3695db5f5a9188186676912c14b61a78bbc6afe/zope.interface-7.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:802176a9f99bd8cc276dcd3b8512808716492f6f557c11196d42e26c01a69a4c", size = 264196, upload-time = "2024-11-28T09:18:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f6/54548df6dc73e30ac6c8a7ff1da73ac9007ba38f866397091d5a82237bd3/zope.interface-7.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb23f58a446a7f09db85eda09521a498e109f137b85fb278edb2e34841055398", size = 259237, upload-time = "2024-11-28T08:48:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/b6/66/ac05b741c2129fdf668b85631d2268421c5cd1a9ff99be1674371139d665/zope.interface-7.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a71a5b541078d0ebe373a81a3b7e71432c61d12e660f1d67896ca62d9628045b", size = 264696, upload-time = "2024-11-28T08:48:41.161Z" }, - { url = "https://files.pythonhosted.org/packages/0a/2f/1bccc6f4cc882662162a1158cda1a7f616add2ffe322b28c99cb031b4ffc/zope.interface-7.2-cp313-cp313-win_amd64.whl", hash = "sha256:4893395d5dd2ba655c38ceb13014fd65667740f09fa5bb01caa1e6284e48c0cd", size = 212472, upload-time = "2024-11-28T08:49:56.587Z" }, -] From 60e6ce9dbb720e76aab84e6caed6c8c9cd2826fb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:50:05 +0200 Subject: [PATCH 04/66] feat: bootstrap same-python popen worker via `python -m` Launch the Trio popen worker as `python -m execnet._trio_worker ` instead of sending bootstrap source over the wire. The worker imports the installed execnet + trio, writes the `1` handshake after adopting its stdio fds, and serves; the coordinator just waits for the handshake. A rough major/minor version check warns on a real execnet mismatch between coordinator and worker while tolerating patch-level drift. Drops the import-bootstrap source send and the importdir/PYTHONPATH plumbing (no more uninstalled running). Foreign-python and remote transports stay on the legacy path pending the uv-provisioned bootstrap. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 18 +++++---- src/execnet/_trio_host.py | 79 ++++++++++++++++++------------------- src/execnet/_trio_worker.py | 57 +++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 49 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index c8229865..84492656 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -18,17 +18,21 @@ for standardizing or versioning the protocol. Trio host-thread IO (popen / import bootstrap) ---------------------------------------------- -For local ``popen`` gateways that use import-based bootstrap -(same installed ``execnet`` + ``trio`` on both sides), Message -protocol IO runs inside a dedicated OS thread hosting a Trio -event loop (``execnet._trio_host.TrioHost``): +For local same-interpreter ``popen`` gateways, Message protocol IO +runs inside a dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire: +the worker is launched as ``python -m execnet._trio_worker`` and +imports the installed ``execnet`` + ``trio`` (a rough major/minor +version check guards against an incompatible install): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). + It waits for the worker's ``b"1"`` handshake before starting the + Message protocol. * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` is scheduled from the Trio nursery - (``trio.to_thread`` for ``thread``, main-thread handoff for - ``main_thread_only``). + streams and writes the handshake byte; ``remote_exec`` is + scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, + main-thread handoff for ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b0db9ad7..ce9ef1cb 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,7 @@ import os import queue import subprocess +import sys import threading from collections.abc import Callable from typing import TYPE_CHECKING @@ -40,20 +41,19 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """PoC: Trio path only for local popen with import bootstrap.""" + """Trio path only for local same-interpreter popen (module bootstrap).""" if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False + # A foreign interpreter (python=) is handled by the future uv-provisioned + # remote path, not the same-interpreter module launch. if getattr(spec, "python", None): return False - # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - return True + return execmodel in (None, "thread", "main_thread_only") class AsyncByteIO(Protocol): @@ -273,7 +273,9 @@ async def _kill() -> None: def is_alive(self) -> bool: return not self._done.is_set() - async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + async def task( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: try: async with trio.open_nursery() as nursery: nursery.start_soon(self._writer) @@ -480,30 +482,27 @@ async def open_popen_process(args: list[str]) -> trio.Process: ) -async def bootstrap_import_async( - process: trio.Process, - *, - importdir: str, - execmodel: str, - gateway_id: str, -) -> ProcessStreamsIO: - """Send import-bootstrap source and wait for the handshake byte.""" - io = ProcessStreamsIO(process) - sources = [ - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet._trio_worker import serve_popen_trio", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). + The coordinator version is passed so the worker can do a rough compatibility + check against its own installed version. + """ + import execnet + + args = [sys.executable, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += [ + "-m", + "execnet._trio_worker", + f"{spec.id}-worker", + spec.execmodel, + execnet.__version__, ] - source = "\n".join(sources) - await io.write_all((repr(source) + "\n").encode("utf-8")) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") - return io + return args class _TempIO: @@ -532,25 +531,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a popen Gateway using Trio process + protocol IO on both sides.""" - import execnet + """Create a same-interpreter popen Gateway on the Trio IO path. - from . import gateway_io - from .gateway_bootstrap import importdir + The worker is launched as ``python -m execnet._trio_worker`` and imports the + installed execnet + trio; nothing is sent over the wire to bootstrap it. + """ + import execnet host: TrioHost = group._ensure_trio_host() - args = gateway_io.popen_args(spec) - remote_execmodel = spec.execmodel + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) try: - async_io = await bootstrap_import_async( - process, - importdir=importdir, - execmodel=remote_execmodel, - gateway_id=spec.id, - ) + async_io = ProcessStreamsIO(process) + ack = await async_io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") except BaseException: with trio.move_on_after(5): process.kill() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6d247194..2c297589 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -11,7 +11,6 @@ import trio -from . import gateway_base from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel @@ -211,6 +210,10 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: model = get_execmodel(execmodel) read_fd, write_fd = _prepare_protocol_fds() + # Bootstrap handshake: the coordinator waits for this byte on our stdout + # before starting the Message protocol. We are launched as a plain module + # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. + os.write(write_fd, b"1") # Keep the historic trace token so tests looking for workergateway still pass. trace(f"creating workergateway on trio id={id!r}") @@ -250,3 +253,55 @@ async def _start() -> _trio_host.ProtocolSession: # Trio's to_thread cache uses non-daemon threads that would otherwise # keep this disposable worker process alive after serve returns. os._exit(0) + + +def _rough_version(version: str) -> tuple[int, ...]: + """Leading numeric (major, minor) of a version string; ``()`` if unparsable.""" + parts: list[int] = [] + for chunk in version.split(".")[:2]: + number = "" + for char in chunk: + if char.isdigit(): + number += char + else: + break + if not number: + break + parts.append(int(number)) + return tuple(parts) + + +def _check_version(coordinator_version: str) -> None: + """Warn on a real (major/minor) execnet version mismatch across the wire. + + A minimal (patch-level) mismatch is tolerated. For same-interpreter popen + the versions are always identical; this guards the future remote paths. + """ + import execnet + + ours = _rough_version(execnet.__version__) + theirs = _rough_version(coordinator_version) + if ours and theirs and ours != theirs: + sys.stderr.write( + "WARNING: execnet version mismatch: coordinator %s worker %s\n" + % (coordinator_version, execnet.__version__) + ) + sys.stderr.flush() + + +def _main() -> None: + """Entry point for ``python -m execnet._trio_worker [ver]``. + + The worker imports execnet + trio from the environment; no source is sent + over the wire to bootstrap it. + """ + argv = sys.argv + worker_id = argv[1] if len(argv) > 1 else "worker" + execmodel = argv[2] if len(argv) > 2 else "thread" + if len(argv) > 3: + _check_version(argv[3]) + serve_popen_trio(id=worker_id, execmodel=execmodel) + + +if __name__ == "__main__": + _main() From dfe7391024badcd7ecc15d18e440d193cd9f5aae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:53:28 +0200 Subject: [PATCH 05/66] add claude settings --- .claude/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..489c3d39 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(uv run pytest:*)" + ] + } +} From 603dfc6551afdc66251a77463846ff8b1ff3674e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 00:21:13 +0200 Subject: [PATCH 06/66] feat: provision foreign-python popen workers via uv Route `popen//python=` through the Trio path. If the target interpreter already imports execnet + trio, launch the worker module directly on it (preserving sys.executable); otherwise provision an ephemeral environment with uv and run the worker there. New _provision.py builds the launch: - coordinator_requirement(): `execnet==` for a released coordinator, else a wheel built from the editable source (located via PEP 610 direct_url.json) and cached keyed by version. The wheel path is read from uv's "Successfully built" output rather than reconstructed, since the build-time version can differ from the import-time one. - uv_run_argv(): `uv run --no-project [--python X] --with python -u -m execnet._trio_worker ...`; trio comes in transitively. - target_has_execnet(): cached probe deciding direct vs provisioned. Falls back to the legacy source-copy path when the target lacks execnet and uv is unavailable. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_provision.py | 180 ++++++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 50 ++++++++--- 2 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 src/execnet/_provision.py diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py new file mode 100644 index 00000000..55a7a055 --- /dev/null +++ b/src/execnet/_provision.py @@ -0,0 +1,180 @@ +"""Coordinator-side worker provisioning via ``uv``. + +Non-same-interpreter Trio workers (foreign-python popen, later ssh/socket) are +launched inside an environment that ``uv`` provisions with a matching execnet + +trio. The worker itself is always ``python -m execnet._trio_worker`` and does +the usual ``b"1"`` stdio handshake; only the launch prefix differs. + +Delivery of execnet into that environment is version-aware: + +* released coordinator (``X.Y.Z``) -> ``uv run --with execnet==X.Y.Z`` +* dev coordinator (``X.Y.Z.devN+g...``) -> build a wheel from the editable + install's source tree, cache it keyed by version, and ``uv run --with `` + +trio is pulled transitively as an execnet dependency. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import tempfile +from functools import cache +from pathlib import Path +from typing import Any + +_RELEASED_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +def uv_available() -> bool: + """Whether the ``uv`` launcher is on PATH.""" + return shutil.which("uv") is not None + + +@cache +def target_has_execnet(python: str) -> bool: + """Whether interpreter ``python`` can already import execnet + trio. + + When true the worker can be launched directly on that interpreter + (preserving ``sys.executable``); otherwise it must be uv-provisioned. + """ + from .gateway_io import shell_split_path + + argv = [*shell_split_path(python), "-c", "import execnet, trio"] + try: + completed = subprocess.run( + argv, capture_output=True, timeout=30, check=False + ) + return completed.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def _version_slug(version: str) -> str: + """Filesystem-safe slug for a version string (may contain ``+``/``.``).""" + return re.sub(r"[^0-9A-Za-z]+", "_", version) + + +def _wheel_cache_dir() -> Path: + d = Path(tempfile.gettempdir()) / "execnet-bootstrap-wheels" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _editable_source_root() -> Path | None: + """Source tree of an editable execnet install, via PEP 610 ``direct_url.json``. + + Returns ``None`` when execnet is not installed editable (nothing to build). + """ + from importlib.metadata import PackageNotFoundError + from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname + + try: + dist = distribution("execnet") + except PackageNotFoundError: + return None + raw = dist.read_text("direct_url.json") + if not raw: + return None + info = json.loads(raw) + if not info.get("dir_info", {}).get("editable"): + return None + url = info.get("url", "") + if not url.startswith("file:"): + return None + return Path(url2pathname(urlparse(url).path)) + + +_BUILT_RE = re.compile(r"^Successfully built (?P.+\.whl)\s*$", re.MULTILINE) + + +def _parse_built_wheel(uv_build_stderr: str) -> Path | None: + """Extract the exact wheel path from ``uv build``'s ``Successfully built`` line. + + Building the filename ourselves is unsafe: the build-time version (dirty tree + -> ``.dYYYYMMDD``/differing dev count) can differ from the import-time + ``__version__``, so we trust the path uv reports. + """ + matches = _BUILT_RE.findall(uv_build_stderr) + if len(matches) != 1: + return None + return Path(matches[0].strip()) + + +def _build_wheel(version: str) -> Path: + """Build (and cache) a wheel of the editable execnet source for ``version``.""" + version_dir = _wheel_cache_dir() / _version_slug(version) + if version_dir.exists(): + cached = sorted(version_dir.glob("*.whl")) + if len(cached) == 1: + return cached[0] + + root = _editable_source_root() + if root is None: + raise RuntimeError( + f"cannot provision dev execnet {version!r}: no editable source tree " + "found (install a released execnet or an editable checkout)" + ) + version_dir.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + ["uv", "build", "--wheel", "-o", str(version_dir), str(root)], + check=True, + capture_output=True, + text=True, + ) + wheel = _parse_built_wheel(proc.stderr) + if wheel is None or not wheel.exists(): + raise RuntimeError( + f"could not determine wheel path from uv build for {version!r}:\n" + f"{proc.stderr}" + ) + return wheel + + +def coordinator_requirement() -> str: + """A ``uv --with`` requirement that installs this coordinator's execnet.""" + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return f"execnet=={version}" + return str(_build_wheel(version)) + + +def uv_run_argv( + *, python: str | None, requirement: str, module_args: list[str] +) -> list[str]: + """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. + + ``--no-project`` keeps the surrounding execnet checkout from being synced, so + the ephemeral env holds only the requirement (+ trio). + """ + argv = ["uv", "run", "--no-project"] + if python: + argv += ["--python", python] + argv += [ + "--with", + requirement, + "python", + "-u", + "-m", + "execnet._trio_worker", + *module_args, + ] + return argv + + +def uv_worker_argv(spec: Any) -> list[str]: + """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" + import execnet + + module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] + return uv_run_argv( + python=spec.python, + requirement=coordinator_requirement(), + module_args=module_args, + ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ce9ef1cb..d72b681d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -41,19 +41,27 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """Trio path only for local same-interpreter popen (module bootstrap).""" + """Trio path for local popen. + + Same-interpreter popen launches the worker module directly; a foreign + interpreter (``python=``) is provisioned via ``uv`` and only taken when + ``uv`` is available (otherwise the legacy source-copy path handles it). + """ if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False - # A foreign interpreter (python=) is handled by the future uv-provisioned - # remote path, not the same-interpreter module launch. - if getattr(spec, "python", None): - return False execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") + if execmodel not in (None, "thread", "main_thread_only"): + return False + if getattr(spec, "python", None): + from . import _provision + + # Direct launch if the interpreter already has execnet; else uv-provision. + return _provision.target_has_execnet(spec.python) or _provision.uv_available() + return True class AsyncByteIO(Protocol): @@ -486,13 +494,21 @@ def popen_module_args(spec: Any) -> list[str]: """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. No source is sent over the wire; the worker imports the installed execnet + - trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). - The coordinator version is passed so the worker can do a rough compatibility + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). The + coordinator version is passed so the worker can do a rough compatibility check against its own installed version. """ import execnet - args = [sys.executable, "-u"] + if getattr(spec, "python", None): + from .gateway_io import shell_split_path + + interpreter = shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") args += [ @@ -531,15 +547,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a same-interpreter popen Gateway on the Trio IO path. + """Create a popen Gateway on the Trio IO path. - The worker is launched as ``python -m execnet._trio_worker`` and imports the - installed execnet + trio; nothing is sent over the wire to bootstrap it. + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ import execnet + from . import _provision + host: TrioHost = group._ensure_trio_host() - args = popen_module_args(spec) + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) From 76563685af7204c19b07213b620260c833dcc138 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 07:32:11 +0200 Subject: [PATCH 07/66] test: add local ssh-connect harness + typing cleanups Add an in-process asyncssh server (asyncio-loop thread) with binary-safe command passthrough so the ssh transport can be tested against the system ssh client without an external host. Committed, intentionally-insecure ed25519 test keys back it (see testing/sshkeys/README.md); the fixture copies the client key to a 0600 temp file since git does not preserve it. asyncssh joins the testing extra. The initial test exercises the existing legacy ssh path, validating the harness before the Trio ssh transport. Also make mypy green at the source instead of casting at call sites: TrioHost.call is now generic (Callable[..., Awaitable[T]] -> T) and makegateway_popen_trio returns Gateway. Drop the stale types-gevent from the mypy hook. Co-Authored-By: Claude Opus 4.8 --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 + src/execnet/_provision.py | 4 +- src/execnet/_trio_host.py | 17 ++- src/execnet/gateway.py | 2 +- testing/sshkeys/README.md | 17 +++ testing/sshkeys/insecure_client_ed25519 | 7 + testing/sshkeys/insecure_client_ed25519.pub | 1 + testing/sshkeys/insecure_host_ed25519 | 7 + testing/sshkeys/insecure_host_ed25519.pub | 1 + testing/test_ssh_local.py | 147 ++++++++++++++++++++ uv.lock | 44 +++++- 12 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 testing/sshkeys/README.md create mode 100644 testing/sshkeys/insecure_client_ed25519 create mode 100644 testing/sshkeys/insecure_client_ed25519.pub create mode 100644 testing/sshkeys/insecure_host_ed25519 create mode 100644 testing/sshkeys/insecure_host_ed25519.pub create mode 100644 testing/test_ssh_local.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be4b6e07..37ccd44d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,3 @@ repos: additional_dependencies: - pytest - types-pywin32 - - types-gevent diff --git a/pyproject.toml b/pyproject.toml index 2c73a32b..853ba3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ testing = [ "tox", "hatch", "uv", + "asyncssh", ] [dependency-groups] @@ -121,3 +122,7 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false + +[[tool.mypy.overrides]] +module = ["asyncssh.*"] +ignore_missing_imports = true diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 55a7a055..b7f6ac56 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -44,9 +44,7 @@ def target_has_execnet(python: str) -> bool: argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: - completed = subprocess.run( - argv, capture_output=True, timeout=30, check=False - ) + completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) return completed.returncode == 0 except (OSError, subprocess.SubprocessError): return False diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index d72b681d..42b45047 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -12,11 +12,13 @@ import subprocess import sys import threading +from collections.abc import Awaitable from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from typing import Protocol from typing import TypeVar +from typing import cast import trio @@ -26,6 +28,7 @@ from .gateway_base import trace if TYPE_CHECKING: + from .gateway import Gateway from .gateway_base import BaseGateway T = TypeVar("T") @@ -251,7 +254,7 @@ def wait_process(self) -> int | None: async def _wait() -> int | None: assert self.process is not None # Always await wait() so the child is reaped (no zombies). - code = await self.process.wait() + code: int | None = await self.process.wait() self._process_exitcode = code self._process_done.set() return code @@ -434,15 +437,17 @@ async def _main(self) -> None: finally: self._nursery = None - def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return cast( + "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + ) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -546,7 +551,7 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Any: +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: """Create a popen Gateway on the Trio IO path. Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; @@ -565,7 +570,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Any: # same interpreter, or a python= that already has execnet args = popen_module_args(spec) - async def _create_and_attach() -> Any: + async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 360b9865..6e336d6f 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -103,7 +103,7 @@ def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session if session is not None: - return session.is_alive() + return bool(session.is_alive()) return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/testing/sshkeys/README.md b/testing/sshkeys/README.md new file mode 100644 index 00000000..d0133e2c --- /dev/null +++ b/testing/sshkeys/README.md @@ -0,0 +1,17 @@ +# Intentionally insecure test SSH keys + +These ed25519 keypairs are **committed on purpose** and are **not secret**. They +exist only so the local ssh-connect tests (`testing/test_ssh_local.py`) can run +an in-process asyncssh server that the system `ssh` client authenticates against, +without generating keys at runtime. + +- `insecure_host_ed25519[.pub]` — the test SSH **server** host key. +- `insecure_client_ed25519[.pub]` — the test **client** identity; its public key + is the server's sole authorized key. + +**Never** use these anywhere real. They grant nothing beyond a throwaway server +bound to `127.0.0.1` on an ephemeral port during a test run. + +Note: git does not preserve `0600` permissions, and OpenSSH refuses a +world-readable private key, so the tests copy the client key to a temp dir and +`chmod 0600` it before use. diff --git a/testing/sshkeys/insecure_client_ed25519 b/testing/sshkeys/insecure_client_ed25519 new file mode 100644 index 00000000..6f5650c8 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzwAAAKBzGIF1cxiB +dQAAAAtzc2gtZWQyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzw +AAAEBCIAXAZCBC6mZFORLaloyPr6HZsRkWpVxVd/vKSZpIhXoxkPC5tjkt9PGK4HGGg6ZF +lp5sdxO5EL9qEeAMMNfPAAAAHGV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1jbGllbnQB +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_client_ed25519.pub b/testing/sshkeys/insecure_client_ed25519.pub new file mode 100644 index 00000000..b4d824b9 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHoxkPC5tjkt9PGK4HGGg6ZFlp5sdxO5EL9qEeAMMNfP execnet-insecure-test-client diff --git a/testing/sshkeys/insecure_host_ed25519 b/testing/sshkeys/insecure_host_ed25519 new file mode 100644 index 00000000..0a31038e --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9RwAAAKDBS525wUud +uQAAAAtzc2gtZWQyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9Rw +AAAECXfzL6Ua9c9U+UkQn+Q7A1aKlBboBhABFALH+/ojmbMGqqgc+IMHZ5QmUZUIszZ5G1 +96W7XfzCcYsNEd3UAb1HAAAAGmV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1ob3N0AQID +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_host_ed25519.pub b/testing/sshkeys/insecure_host_ed25519.pub new file mode 100644 index 00000000..93d3cf93 --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGqqgc+IMHZ5QmUZUIszZ5G196W7XfzCcYsNEd3UAb1H execnet-insecure-test-host diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py new file mode 100644 index 00000000..196c5d76 --- /dev/null +++ b/testing/test_ssh_local.py @@ -0,0 +1,147 @@ +"""Local ssh-connect tests backed by an in-process asyncssh server. + +The coordinator shells out to the system ``ssh`` client, which connects to an +asyncssh server running in its own asyncio-loop thread; the server runs each +requested command as a subprocess with binary-safe stdio passthrough (the +execnet Message protocol needs raw bytes). +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import sys +import threading +from collections.abc import Iterator +from pathlib import Path + +import asyncssh +import pytest + +import execnet + +pytestmark = [ + pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" + ), + # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, + # surfaced by pytest's unraisable-exception hook during GC; harmless here. + pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), +] + +# Committed, intentionally-insecure test keys (see sshkeys/README.md). +SSHKEYS = Path(__file__).parent / "sshkeys" +HOST_KEY = SSHKEYS / "insecure_host_ed25519" +CLIENT_KEY = SSHKEYS / "insecure_client_ed25519" +CLIENT_PUBKEY = SSHKEYS / "insecure_client_ed25519.pub" + + +class SSHServerThread: + """asyncssh server on an ephemeral port, driven from its own asyncio thread.""" + + def __init__(self, client_key_path: str) -> None: + # OpenSSH refuses a world-readable private key; git does not preserve + # 0600, so the caller hands us a temp copy already chmod'd 0600. + self.client_key_path = client_key_path + self._loop: asyncio.AbstractEventLoop | None = None + self._server: asyncssh.SSHAcceptor | None = None + self.port: int | None = None + self._ready = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + async def _handle(self, process: asyncssh.SSHServerProcess) -> None: + proc = await asyncio.create_subprocess_shell( + process.command or "", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) + process.exit(await proc.wait()) + + async def _serve(self) -> None: + self._server = await asyncssh.listen( + "127.0.0.1", + 0, + server_host_keys=[str(HOST_KEY)], + authorized_client_keys=str(CLIENT_PUBKEY), + process_factory=self._handle, + encoding=None, # binary stdio + ) + self.port = self._server.get_port() + self._ready.set() + await self._server.wait_closed() + + def _run(self) -> None: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + try: + self._loop.run_until_complete(self._serve()) + finally: + # Drain asyncssh's shutdown coroutines so GC does not surface an + # "un-awaited coroutine" RuntimeWarning. + pending = asyncio.all_tasks(self._loop) + for task in pending: + task.cancel() + self._loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.close() + + def start(self) -> None: + self._thread.start() + assert self._ready.wait(timeout=10), "ssh server did not start" + + def stop(self) -> None: + if self._loop is not None and self._server is not None: + self._loop.call_soon_threadsafe(self._server.close) + self._thread.join(timeout=5) + + def write_ssh_config(self, path: str) -> None: + """Write an ssh config with a ``testhost`` alias pointing at this server.""" + with open(path, "w") as f: + f.write( + "Host testhost\n" + " HostName 127.0.0.1\n" + f" Port {self.port}\n" + " User testuser\n" + f" IdentityFile {self.client_key_path}\n" + " IdentitiesOnly yes\n" + " StrictHostKeyChecking no\n" + " UserKnownHostsFile /dev/null\n" + " LogLevel ERROR\n" + ) + + +@pytest.fixture +def ssh_server(tmp_path) -> Iterator[SSHServerThread]: + # OpenSSH rejects the committed key's checkout permissions; use a 0600 copy. + client_key = tmp_path / "client_ed25519" + client_key.write_bytes(CLIENT_KEY.read_bytes()) + client_key.chmod(0o600) + server = SSHServerThread(str(client_key)) + server.start() + yield server + server.stop() + + +@pytest.fixture +def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: + path = str(tmp_path / "ssh_config") + ssh_server.write_ssh_config(path) + return path + + +def test_ssh_roundtrip(ssh_config: str) -> None: + group = execnet.Group() + try: + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}//id=ssh" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) diff --git a/uv.lock b/uv.lock index 5dbf4f10..e4657f7b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncssh" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/7f/2d79247bacc562104f312d27efe541673aa177feac89de291bd61bca52be/asyncssh-2.24.0.tar.gz", hash = "sha256:4064c590e59ce2e8d82a2f66d35f3120d765828b4df5e3dbfb07b4a8c24686c9", size = 550148, upload-time = "2026-06-27T20:34:44.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/29/908ce0ca5e8cae76662e354a0f08df552d6d221844748b9e5ca06051cc44/asyncssh-2.24.0-py3-none-any.whl", hash = "sha256:9abd46300adcb6d4b73269b34c53cd0d17a138b9a22b5b38008ce7d5808734b7", size = 381237, upload-time = "2026-06-27T20:34:43.198Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -145,6 +158,8 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/e9/6d7724983b3d5a0908dbf74f64038ade77c18646ff6636ec7894fd392ce1/cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0", size = 183837, upload-time = "2026-07-06T21:32:09.655Z" }, + { url = "https://files.pythonhosted.org/packages/69/aa/24580a278de21fd7322635556334d9b535f1cbc00b0a3919447cdf464c65/cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd", size = 184226, upload-time = "2026-07-06T21:32:11.196Z" }, { url = "https://files.pythonhosted.org/packages/88/a9/02cae418ec4beb282ace11958d9d4737793439d561fadc7e6d56f2e2b354/cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46", size = 211107, upload-time = "2026-07-06T21:32:12.328Z" }, { url = "https://files.pythonhosted.org/packages/3b/30/c806937ed5e4c2c7ac30d9d6b76b5dc57ff8b75d83800d9bb11a8253cf2a/cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2", size = 218733, upload-time = "2026-07-06T21:32:13.67Z" }, { url = "https://files.pythonhosted.org/packages/f9/cf/398272b8bbfd58aa314fda5a7f1cdbb26d1d78ae324a11211521315dd1f0/cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd", size = 205543, upload-time = "2026-07-06T21:32:15.148Z" }, @@ -155,6 +170,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/75/74dfb7c3fc6ebbd408038476bd4c1d7e925c62614e7b9c534ecc34218288/cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd", size = 220341, upload-time = "2026-07-06T21:32:21.9Z" }, { url = "https://files.pythonhosted.org/packages/70/b6/9003c33a3e7d2c1306f5962e646457dcfe5a8cd8fce6bbe02d7af25db783/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f", size = 174578, upload-time = "2026-07-06T21:32:23.073Z" }, { url = "https://files.pythonhosted.org/packages/8a/26/710688310447531c7a22f857c7f79d9855ec18b03e04494ced723fb37e2f/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da", size = 185071, upload-time = "2026-07-06T21:32:24.671Z" }, + { url = "https://files.pythonhosted.org/packages/d3/67/85c89a59ba36a671e79638f44d466749f08179266a57e4f2ffdf92174072/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc", size = 183845, upload-time = "2026-07-06T21:32:26.32Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dd/e3b0baa2d3d6a857ac72b7efbf18e32e487c9cdafcc13049ad765495b15e/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7", size = 184186, upload-time = "2026-07-06T21:32:28.025Z" }, { url = "https://files.pythonhosted.org/packages/65/68/9f3ef890cf3c6ab97bd531c5677f67613d302165d16f8142b2811782a614/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93", size = 211892, upload-time = "2026-07-06T21:32:29.565Z" }, { url = "https://files.pythonhosted.org/packages/22/d7/1a74539db16d8bfd839ff1515948948efbb162e574650fd3d846896eea95/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2", size = 218793, upload-time = "2026-07-06T21:32:30.951Z" }, { url = "https://files.pythonhosted.org/packages/ec/d1/9a5b7169499e8e8d8e636de70b97ac7c9447104d2ff1a2cd94790cea5162/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c", size = 205737, upload-time = "2026-07-06T21:32:32.216Z" }, @@ -166,6 +183,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, @@ -178,6 +197,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, @@ -190,6 +211,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, @@ -199,6 +222,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, @@ -210,6 +235,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, @@ -219,6 +246,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, @@ -270,6 +299,7 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, @@ -281,6 +311,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, @@ -292,6 +325,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, @@ -303,10 +339,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] @@ -332,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,6 +388,7 @@ dependencies = [ [package.optional-dependencies] testing = [ + { name = "asyncssh" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -363,6 +404,7 @@ testing = [ [package.metadata] requires-dist = [ + { name = "asyncssh", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, From f153271a5f2feb4369b73ddf487c88d8c5c99439 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:04:26 +0200 Subject: [PATCH 08/66] feat: run ssh gateways on the Trio path Route ssh gateways through the Trio host: `ssh -C [-F cfg] ''` spawns the uv-provisioned worker module on the remote, then the coordinator does the usual b"1" handshake and attaches a Trio session. HostNotFound is raised when ssh exits 255. The popen and ssh factories now share _open_trio_gateway (spawn + handshake + attach); ssh_trio_args builds the shell-quoted remote worker command. test_ssh_roundtrip is parametrized over the trio and legacy paths against the in-process asyncssh server. test_sshconfig_config_parsing (white-box over legacy Popen2IOMaster) is pinned to EXECNET_TRIO_HOST=0, with a new ssh_trio_args test covering -F on the Trio path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 66 +++++++++++++++++------------- src/execnet/_trio_host.py | 85 +++++++++++++++++++++++++++++++++------ src/execnet/multi.py | 2 + testing/test_gateway.py | 13 ++++++ testing/test_ssh_local.py | 8 +++- 5 files changed, 131 insertions(+), 43 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84492656..84bcc065 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -15,34 +15,44 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -Trio host-thread IO (popen / import bootstrap) ----------------------------------------------- - -For local same-interpreter ``popen`` gateways, Message protocol IO -runs inside a dedicated OS thread hosting a Trio event loop -(``execnet._trio_host.TrioHost``). No source is sent over the wire: -the worker is launched as ``python -m execnet._trio_worker`` and -imports the installed ``execnet`` + ``trio`` (a rough major/minor -version check guards against an incompatible install): - -* Coordinator: ``trio.lowlevel.open_process`` plus async framed - reader/writer tasks per gateway (one host thread per ``Group``). - It waits for the worker's ``b"1"`` handshake before starting the - Message protocol. -* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams and writes the handshake byte; ``remote_exec`` is - scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, - main-thread handoff for ``main_thread_only``). - -Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from -non-host threads wait until the frame is written (so abrupt -``os._exit`` cannot drop queued data). Sends from the Trio host -thread (receiver callbacks) only enqueue, to avoid deadlocking -the writer task. - -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types -(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) -still use the legacy thread receiver and sync ``Popen`` path. +Trio host-thread IO +------------------- + +``popen`` and ``ssh`` gateways run their Message protocol IO inside a +dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire; the +worker is launched as ``python -m execnet._trio_worker`` and imports the +installed ``execnet`` + ``trio`` (a rough major/minor version check guards +against an incompatible install). How the worker environment is obtained +depends on the target: + +* Same-interpreter ``popen`` -> ``sys.executable -m execnet._trio_worker``. +* A ``python=`` interpreter that already has execnet -> that interpreter + directly (so ``sys.executable`` is preserved). +* A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via + ``uv`` (``execnet._provision``): ``uv run --with `` where ```` + is ``execnet==`` for a released coordinator or a locally-built, + version-cached wheel for a dev coordinator. + +Coordinator and worker roles: + +* Coordinator: ``trio.lowlevel.open_process`` (directly, or wrapped in + ``ssh``) plus async framed reader/writer tasks per gateway (one host + thread per ``Group``). It waits for the worker's ``b"1"`` handshake + before starting the Message protocol. +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio streams and + writes the handshake byte; ``remote_exec`` is scheduled from the Trio + nursery (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host +threads wait until the frame is written (so abrupt ``os._exit`` cannot drop +queued data). Sends from the Trio host thread (receiver callbacks) only +enqueue, to avoid deadlocking the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, +``via``, greenlet execmodels) still use the legacy thread receiver and sync +``Popen`` path. Legacy thread model ------------------- diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 42b45047..68755d50 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -551,24 +551,19 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +def _open_trio_gateway( + group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None +) -> Gateway: + """Spawn ``args``, do the worker handshake, and attach a Trio session. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Shared by the popen and ssh factories; ``args`` already encodes how the + worker is launched (direct module, uv-provisioned, or wrapped in ssh). """ import execnet - from . import _provision + from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) async def _create_and_attach() -> Gateway: process = await open_popen_process(args) @@ -577,6 +572,16 @@ async def _create_and_attach() -> Gateway: ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") + except EOFError: + with trio.move_on_after(5): + code = await process.wait() + # ssh exits 255 when it cannot reach/authenticate the host. + if remoteaddress is not None and code == 255: + raise HostNotFound(remoteaddress) from None + with trio.move_on_after(5): + process.kill() + await process.wait() + raise except BaseException: with trio.move_on_after(5): process.kill() @@ -586,7 +591,61 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) + + +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: + """Create a popen Gateway on the Trio IO path. + + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) + return _open_trio_gateway(group, spec, args) + + +def ssh_trio_args(spec: Any) -> list[str]: + """``ssh [-F cfg] ''`` for the Trio ssh path. + + The remote runs the uv-provisioned worker module; the worker command is + shell-quoted for the remote shell. + """ + import shlex + + from . import _provision + + remote_command = shlex.join(_provision.uv_worker_argv(spec)) + args = ["ssh", "-C"] + if getattr(spec, "ssh_config", None): + args += ["-F", spec.ssh_config] + assert spec.ssh is not None + args += spec.ssh.split() + args.append(remote_command) + return args + + +def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: + """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" + args = ssh_trio_args(spec) + return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + + +def should_use_trio_ssh(spec: Any) -> bool: + """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "ssh", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 687f1840..29cea870 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -156,6 +156,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: if _trio_host.should_use_trio_popen(spec): gw = _trio_host.makegateway_popen_trio(self, spec) + elif _trio_host.should_use_trio_ssh(spec): + gw = _trio_host.makegateway_ssh_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 83a50f44..f7f882ae 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -384,6 +384,8 @@ class TestSshPopenGateway: def test_sshconfig_config_parsing( self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] ) -> None: + # white-box test of the legacy Popen2IOMaster arg construction + monkeypatch.setenv("EXECNET_TRIO_HOST", "0") l = [] monkeypatch.setattr( gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) @@ -396,6 +398,17 @@ def test_sshconfig_config_parsing( i = popen_args.index("-F") assert popen_args[i + 1] == "qwe" + def test_ssh_trio_args_include_config( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from execnet import _provision + from execnet import _trio_host + + monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") + args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + assert args[args.index("-F") + 1] == "qwe" + assert "xyz" in args + def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 196c5d76..72dea348 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import os import shutil import sys import threading @@ -134,7 +133,12 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -def test_ssh_roundtrip(ssh_config: str) -> None: +@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) +def test_ssh_roundtrip( + ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str +) -> None: + # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. + monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) group = execnet.Group() try: gw = group.makegateway( From 5197b1eee591461b959fea9be07363e50f91d630 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:43:07 +0200 Subject: [PATCH 09/66] feat: ship the execnet wheel to ssh remotes for dev coordinators A dev coordinator's wheel is not on the remote filesystem, so the ssh remote command becomes a POSIX-sh prelude that reads the wheel bytes from stdin (`head -c N` into a temp dir) and execs uv against it. The coordinator streams those bytes as a preamble before the Message protocol; _open_trio_gateway grew a `preamble` argument. Released coordinators still use `uv run --with execnet==` with no shipping. Also collapse the worker CLI contract: id, execmodel and coordinator version now travel as a single JSON argument (_provision.worker_cli_arg) consumed by _trio_worker._main, instead of scattered positional args built in three launchers. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 10 ++++- src/execnet/_provision.py | 87 +++++++++++++++++++++++++++---------- src/execnet/_trio_host.py | 45 +++++++++---------- src/execnet/_trio_worker.py | 18 ++++---- testing/test_gateway.py | 10 ++++- testing/test_provision.py | 40 +++++++++++++++++ 6 files changed, 152 insertions(+), 58 deletions(-) create mode 100644 testing/test_provision.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84bcc065..29df051c 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -32,7 +32,15 @@ depends on the target: * A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via ``uv`` (``execnet._provision``): ``uv run --with `` where ```` is ``execnet==`` for a released coordinator or a locally-built, - version-cached wheel for a dev coordinator. + version-cached wheel for a dev coordinator. For an ``ssh`` remote on a + dev coordinator the wheel is not on the remote filesystem, so the remote + command is a POSIX-sh prelude that reads the wheel bytes from stdin + (``head -c N`` into a temp dir) and ``exec``s ``uv`` against it; the + coordinator streams those bytes before the Message protocol. + +The worker configuration (id, execmodel, coordinator version) is passed as a +single JSON CLI argument (``_provision.worker_cli_arg``), so every launcher +shares one contract instead of scattered positional args. Coordinator and worker roles: diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b7f6ac56..b24bfe8b 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -18,6 +18,7 @@ import json import re +import shlex import shutil import subprocess import tempfile @@ -143,36 +144,74 @@ def coordinator_requirement() -> str: return str(_build_wheel(version)) -def uv_run_argv( - *, python: str | None, requirement: str, module_args: list[str] -) -> list[str]: - """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. +def worker_cli_arg(spec: Any) -> str: + """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). - ``--no-project`` keeps the surrounding execnet checkout from being synced, so - the ephemeral env holds only the requirement (+ trio). + Passed to ``python -m execnet._trio_worker`` by every launcher (popen, uv, + ssh) so the worker config lives in one place rather than scattered positional + args. """ - argv = ["uv", "run", "--no-project"] - if python: - argv += ["--python", python] - argv += [ + import execnet + + return json.dumps( + { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "coordinator_version": execnet.__version__, + } + ) + + +def worker_module_tokens(spec: Any) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens.""" + return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + + +def _uv_prefix(spec: Any) -> list[str]: + # --no-project keeps the surrounding execnet checkout from being synced. + prefix = ["uv", "run", "--no-project"] + if spec.python: + prefix += ["--python", spec.python] + return prefix + + +def uv_worker_argv(spec: Any) -> list[str]: + """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + return [ + *_uv_prefix(spec), "--with", - requirement, - "python", - "-u", - "-m", - "execnet._trio_worker", - *module_args, + coordinator_requirement(), + *worker_module_tokens(spec), ] - return argv -def uv_worker_argv(spec: Any) -> list[str]: - """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel + bytes are returned as the preamble to stream before the Message protocol. + """ import execnet - module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] - return uv_run_argv( - python=spec.python, - requirement=coordinator_requirement(), - module_args=module_args, + version = execnet.__version__ + worker = worker_module_tokens(spec) + if _RELEASED_RE.match(version): + command = shlex.join( + [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] + ) + return command, b"" + + wheel = _build_wheel(version) + data = wheel.read_bytes() + # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. + remote_wheel = '"$d/"' + shlex.quote(wheel.name) + uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + worker_cmd = " ".join(shlex.quote(token) for token in worker) + prelude = ( + f"d=$(mktemp -d) && " + f"head -c {len(data)} > {remote_wheel} && " + f"exec {uv_run} {remote_wheel} {worker_cmd}" ) + return prelude, data diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68755d50..f9cf7a71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -500,11 +500,9 @@ def popen_module_args(spec: Any) -> list[str]: No source is sent over the wire; the worker imports the installed execnet + trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). The - coordinator version is passed so the worker can do a rough compatibility - check against its own installed version. + already has execnet (so ``sys.executable`` stays that interpreter). """ - import execnet + from . import _provision if getattr(spec, "python", None): from .gateway_io import shell_split_path @@ -516,13 +514,7 @@ def popen_module_args(spec: Any) -> list[str]: args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") - args += [ - "-m", - "execnet._trio_worker", - f"{spec.id}-worker", - spec.execmodel, - execnet.__version__, - ] + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] return args @@ -552,12 +544,19 @@ def kill(self) -> None: def _open_trio_gateway( - group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None + group: Any, + spec: Any, + args: list[str], + *, + remoteaddress: str | None = None, + preamble: bytes = b"", ) -> Gateway: """Spawn ``args``, do the worker handshake, and attach a Trio session. Shared by the popen and ssh factories; ``args`` already encodes how the worker is launched (direct module, uv-provisioned, or wrapped in ssh). + ``preamble`` is streamed to the worker's stdin before the handshake (used to + ship a wheel to a remote that receives it with ``head -c``). """ import execnet @@ -569,6 +568,8 @@ async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) + if preamble: + await async_io.write_all(preamble) ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") @@ -615,30 +616,30 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: return _open_trio_gateway(group, spec, args) -def ssh_trio_args(spec: Any) -> list[str]: - """``ssh [-F cfg] ''`` for the Trio ssh path. +def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for the Trio ssh path. - The remote runs the uv-provisioned worker module; the worker command is - shell-quoted for the remote shell. + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. """ - import shlex - from . import _provision - remote_command = shlex.join(_provision.uv_worker_argv(spec)) + remote_command, preamble = _provision.ssh_remote_command(spec) args = ["ssh", "-C"] if getattr(spec, "ssh_config", None): args += ["-F", spec.ssh_config] assert spec.ssh is not None args += spec.ssh.split() args.append(remote_command) - return args + return args, preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args = ssh_trio_args(spec) - return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + args, preamble = ssh_trio_args(spec) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.ssh, preamble=preamble + ) def should_use_trio_ssh(spec: Any) -> bool: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 2c297589..f90db607 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -290,17 +290,17 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker [ver]``. + """Entry point for ``python -m execnet._trio_worker ``. - The worker imports execnet + trio from the environment; no source is sent - over the wire to bootstrap it. + ```` is the coordinator's ``_provision.worker_cli_arg`` payload: + ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + + trio from the environment; no source is sent over the wire to bootstrap it. """ - argv = sys.argv - worker_id = argv[1] if len(argv) > 1 else "worker" - execmodel = argv[2] if len(argv) > 2 else "thread" - if len(argv) > 3: - _check_version(argv[3]) - serve_popen_trio(id=worker_id, execmodel=execmodel) + import json + + config = json.loads(sys.argv[1]) + _check_version(config["coordinator_version"]) + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/testing/test_gateway.py b/testing/test_gateway.py index f7f882ae..e123bbe2 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -404,10 +404,16 @@ def test_ssh_trio_args_include_config( from execnet import _provision from execnet import _trio_host - monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") - args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + monkeypatch.setattr( + _provision, "ssh_remote_command", lambda spec: ("worker-cmd", b"") + ) + args, preamble = _trio_host.ssh_trio_args( + execnet.XSpec("ssh=xyz//ssh_config=qwe") + ) assert args[args.index("-F") + 1] == "qwe" assert "xyz" in args + assert args[-1] == "worker-cmd" + assert preamble == b"" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_provision.py b/testing/test_provision.py new file mode 100644 index 00000000..8df8775a --- /dev/null +++ b/testing/test_provision.py @@ -0,0 +1,40 @@ +"""Unit tests for coordinator-side uv worker provisioning.""" + +from __future__ import annotations + +import json +import re + +import pytest + +import execnet +from execnet import _provision + +released = re.fullmatch(r"\d+\.\d+\.\d+", execnet.__version__) is not None + + +def test_worker_cli_arg_carries_config() -> None: + spec = execnet.XSpec("popen//id=gw5//execmodel=thread") + config = json.loads(_provision.worker_cli_arg(spec)) + assert config["id"] == "gw5-worker" + assert config["execmodel"] == "thread" + assert config["coordinator_version"] == execnet.__version__ + + +def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9") + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "execnet==9.9.9" in command + assert "head -c" not in command # no shipping + assert preamble == b"" + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_ssh_remote_command_dev_ships_wheel() -> None: + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "mktemp -d" in command + assert f"head -c {len(preamble)}" in command + assert preamble[:2] == b"PK" # a wheel is a zip archive From 21096da19f4a2be00290b81a32286502b8f8a501 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 09:48:12 +0200 Subject: [PATCH 10/66] feat: add execnet-socketserver console command Expose the socket server as a `[project.scripts]` entry point so it can be started directly, e.g. provisioned anywhere with `uvx --from execnet execnet-socketserver :8888`. Refactor the __main__ block into a main() with an argparse CLI (hostport + --once), and thread execmodel explicitly through startserver/exec_from_one_connection instead of relying on a module global (which main()'s local scope broke). Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 4 +- pyproject.toml | 3 ++ src/execnet/script/socketserver.py | 43 +++++++++++++++----- testing/test_socketserver_cli.py | 63 ++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 testing/test_socketserver_cli.py diff --git a/doc/basics.rst b/doc/basics.rst index 723de83f..a6b256b7 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -58,7 +58,9 @@ Examples for valid gateway specifications remotely sets an environment variable ``NAME`` to ``value``. * ``socket=192.168.1.4:8888`` specifies a Python Socket server - process that listens on ``192.168.1.4:8888`` + process that listens on ``192.168.1.4:8888``. Such a server can be + started with the ``execnet-socketserver`` console command, e.g. run + anywhere with ``uvx --from execnet execnet-socketserver :8888``. .. versionadded:: 1.5 diff --git a/pyproject.toml b/pyproject.toml index 853ba3d8..f419c31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Topic :: System :: Networking", ] +[project.scripts] +execnet-socketserver = "execnet.script.socketserver:main" + [project.optional-dependencies] testing = [ "pre-commit", diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index fa98743c..43d82147 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -48,7 +48,7 @@ def print_(*args) -> None: ) -def exec_from_one_connection(serversock) -> None: +def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: print_(progname, "Entering Accept loop", serversock.getsockname()) clientsock, address = serversock.accept() print_(progname, "got new connection from {} {}".format(*address)) @@ -89,12 +89,12 @@ def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): return serversock -def startserver(serversock, loop: bool = False) -> None: +def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: execute_path = os.getcwd() try: while 1: try: - exec_from_one_connection(serversock) + exec_from_one_connection(serversock, execmodel) except (KeyboardInterrupt, SystemExit): raise except BaseException as exc: @@ -112,15 +112,40 @@ def startserver(serversock, loop: bool = False) -> None: serversock.shutdown(2) -if __name__ == "__main__": - import sys +def main(argv: list[str] | None = None) -> None: + """Console entry point (``execnet-socketserver``). + + Bind a socket and serve gateway connections. Intended to be run directly, + e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + """ + import argparse - hostport = sys.argv[1] if len(sys.argv) > 1 else ":8888" from execnet.gateway_base import get_execmodel + parser = argparse.ArgumentParser( + prog="execnet-socketserver", + description="Serve execnet gateway connections over a socket.", + ) + parser.add_argument( + "hostport", + nargs="?", + default=":8888", + help="address to bind as HOST:PORT or :PORT (default: :8888)", + ) + parser.add_argument( + "--once", + action="store_true", + help="serve a single connection and exit instead of looping", + ) + args = parser.parse_args(argv) + execmodel = get_execmodel("thread") - serversock = bind_and_listen(hostport, execmodel) - startserver(serversock, loop=True) + serversock = bind_and_listen(args.hostport, execmodel) + startserver(serversock, execmodel, loop=not args.once) + + +if __name__ == "__main__": + main() elif __name__ == "__channelexec__": chan: Channel = globals()["channel"] @@ -130,4 +155,4 @@ def startserver(serversock, loop: bool = False) -> None: sock = bind_and_listen(bindname, execmodel) port = sock.getsockname() chan.send(port) - startserver(sock) + startserver(sock, execmodel) diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py new file mode 100644 index 00000000..039803fa --- /dev/null +++ b/testing/test_socketserver_cli.py @@ -0,0 +1,63 @@ +"""Test the ``execnet-socketserver`` console entry point end to end.""" + +from __future__ import annotations + +import shutil +import socket +import subprocess +import time +from collections.abc import Iterator + +import pytest + +import execnet + +SERVER = shutil.which("execnet-socketserver") + +pytestmark = pytest.mark.skipif( + SERVER is None, reason="execnet-socketserver console script not installed" +) + + +def _free_port() -> int: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@pytest.fixture +def socketserver_port() -> Iterator[int]: + assert SERVER is not None + port = _free_port() + proc = subprocess.Popen( + [SERVER, f"127.0.0.1:{port}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + # loop mode: a throwaway probe just consumes one accept iteration + for _ in range(100): + try: + socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + break + except OSError: + time.sleep(0.1) + else: + pytest.fail("execnet-socketserver did not start") + yield port + finally: + proc.kill() + proc.wait(timeout=5) + + +def test_socketserver_cli_roundtrip(socketserver_port: int) -> None: + group = execnet.Group() + try: + gw = group.makegateway(f"socket=127.0.0.1:{socketserver_port}//id=sock") + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 420693ac3d939af96929a2e8b433b5d0e2066ff1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 10:53:35 +0200 Subject: [PATCH 11/66] feat: run direct socket gateways on Trio with subprocess workers Move the direct `socket=host:port` transport onto the Trio host: - The socketserver becomes a Trio TCP listener that spawns a `python -m execnet._trio_worker --socket-fd N` subprocess per connection (passing the accepted socket by fd) instead of exec'ing sent source inline. - The worker gains a socket serve mode: it adopts an inherited socket fd into a Trio SocketStream and serves the Message protocol over it. The worker CLI now always carries its config as args, so a future popen socketpair can reuse the same path instead of hijacking stdio. - The coordinator connects a Trio TCP stream, waits for the b"1" handshake, and attaches a Trio session (no local process). A failed connect raises HostNotFound. The worker serve setup is refactored into shared _build_worker_gateway / _run_worker helpers. `installvia` still uses the legacy path for now. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 95 ++++++++++++++++++++++ src/execnet/_trio_worker.py | 124 ++++++++++++++++++++--------- src/execnet/multi.py | 2 + src/execnet/script/socketserver.py | 70 ++++++++++++++-- testing/test_socketserver_cli.py | 41 +++++----- 5 files changed, 263 insertions(+), 69 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index f9cf7a71..c58d0c34 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -137,6 +137,45 @@ async def aclose_write(self) -> None: await self._write.aclose() +class SocketStreamIO: + """Async IO over a single bidirectional Trio stream (a socket). + + ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` + bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is + idempotent, so close-read and close-write both just close the socket. + """ + + def __init__(self, stream: trio.abc.Stream) -> None: + self._stream = stream + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stream, n) + + async def write_all(self, data: bytes) -> None: + await self._stream.send_all(data) + + async def aclose_read(self) -> None: + await self._stream.aclose() + + async def aclose_write(self) -> None: + await self._stream.aclose() + + +async def adopt_socket(socket_fd: int) -> SocketStreamIO: + """Worker side: wrap an inherited socket fd and send the handshake. + + Runs on the Trio host loop. The coordinator waits for ``b"1"`` before + starting the Message protocol; the worker config comes from the CLI. + """ + import socket as _socket + + sock = _socket.socket(fileno=socket_fd) + stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) + io = SocketStreamIO(stream) + await io.write_all(b"1") + return io + + class SyncIOHandle: """Sync IO facade for Group.terminate wait/kill/close_write.""" @@ -650,3 +689,59 @@ def should_use_trio_ssh(spec: Any) -> bool: return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: + """Connect a Trio TCP stream to a running ``execnet-socketserver``. + + The server spawns the worker and synthesises its config, so the coordinator + just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio + session (no local process). + """ + import execnet + + from .gateway_bootstrap import HostNotFound + + host: TrioHost = group._ensure_trio_host() + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + + async def _create_and_attach() -> Gateway: + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(spec.socket) from exc + io = SocketStreamIO(stream) + try: + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad socket handshake: {ack!r}") + except BaseException: + with trio.move_on_after(5): + await stream.aclose() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + return gw + + return host.call(_create_and_attach) + + +def should_use_trio_socket(spec: Any) -> bool: + """Trio path for direct ``socket=host:port`` gateways. + + ``installvia`` (start a server through another gateway) still uses the legacy + path for now. + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "socket", None): + return False + if getattr(spec, "installvia", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index f90db607..5fc137fb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -204,8 +204,59 @@ def kill(self) -> None: return +def _build_worker_gateway( + host: _trio_host.TrioHost, id: str, model: ExecModel +) -> tuple[WorkerGateway, TrioWorkerExec, bool]: + """Construct the WorkerGateway + Trio exec pool (no IO yet).""" + trace(f"creating workergateway on trio id={id!r}") + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec + gateway._executetask_complete = None + if main_thread_only: + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + return gateway, trio_exec, main_thread_only + + +def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: + """Attach ``io`` as the gateway session and serve until shutdown.""" + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + + async def _start() -> _trio_host.ProtocolSession: + return await host.start_session(gateway, io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if main_thread_only: + trace("integrating as primary thread (trio worker)") + trio_exec.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) + + +async def _make_fd_io(read_fd: int, write_fd: int) -> Any: + from . import _trio_host + + return _trio_host.FdStreamsIO(read_fd, write_fd) + + def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" + """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host model = get_execmodel(execmodel) @@ -214,45 +265,27 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: # before starting the Message protocol. We are launched as a plain module # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. os.write(write_fd, b"1") - # Keep the historic trace token so tests looking for workergateway still pass. - trace(f"creating workergateway on trio id={id!r}") host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() - try: - io_stub = _WorkerIOStub(model) - gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] - gateway._trio_exec = trio_exec - gateway._executetask_complete = None - if main_thread_only: - gateway._executetask_complete = model.Event() - gateway._executetask_complete.set() + io = host.call(_make_fd_io, read_fd, write_fd) + _run_worker(host, io, id, model) - async def _start() -> _trio_host.ProtocolSession: - async_io = _trio_host.FdStreamsIO(read_fd, write_fd) - return await host.start_session(gateway, async_io) - session = host.call(_start) - gateway._attach_trio_session(session) +def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: + """Serve a WorkerGateway over an inherited socket fd. - try: - if main_thread_only: - trace("integrating as primary thread (trio worker)") - trio_exec.integrate_as_primary_thread() - gateway.join() - except KeyboardInterrupt: - # Match WorkerGateway.serve(): swallow in the worker. - trace("swallowing keyboardinterrupt, serve finished") - finally: - host.stop(timeout=5.0) - # Trio's to_thread cache uses non-daemon threads that would otherwise - # keep this disposable worker process alive after serve returns. - os._exit(0) + Used for the socketserver (an accepted TCP connection) and, in future, a + popen socketpair. The socket is adopted inside Trio and the handshake is + written on the host loop; config comes from the CLI. + """ + from . import _trio_host + + model = get_execmodel(execmodel) + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + io = host.call(_trio_host.adopt_socket, socket_fd) + _run_worker(host, io, id, model) def _rough_version(version: str) -> tuple[int, ...]: @@ -290,17 +323,30 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker ``. + """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + - trio from the environment; no source is sent over the wire to bootstrap it. + ``{"id", "execmodel", "coordinator_version"}``. With ``--socket-fd`` the + worker serves over that inherited socket (socketserver); otherwise over the + stdio pipes (popen / ssh). The worker imports execnet + trio from the + environment; no source is sent over the wire to bootstrap it. """ + import argparse import json - config = json.loads(sys.argv[1]) + parser = argparse.ArgumentParser(prog="execnet._trio_worker") + parser.add_argument("config", help="JSON worker config") + # Protocol transport: an inherited socket fd (socketserver, or a future popen + # socketpair); without it the worker serves over the stdio pipes (ssh). + parser.add_argument("--socket-fd", type=int, default=None) + ns = parser.parse_args() + + config = json.loads(ns.config) _check_version(config["coordinator_version"]) - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + if ns.socket_fd is not None: + serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + else: + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29cea870..d64722ee 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_socket(spec): + gw = _trio_host.makegateway_socket_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index 43d82147..ea580bcd 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -112,15 +112,74 @@ def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: serversock.shutdown(2) +async def _trio_serve(hostport: str, once: bool) -> None: + """Trio TCP server that spawns a worker subprocess per connection. + + No code is executed inline: each accepted socket is handed (by fd) to a + fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the + gateway over it. + """ + import itertools + import json + import subprocess + + import trio + + import execnet + + host, _, port_str = hostport.rpartition(":") + listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) + addr = listeners[0].socket.getsockname() + # Report the bound address (port may be ephemeral) for callers to read. + print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) + + counter = itertools.count() + + with trio.CancelScope() as scope: + + async def handler(stream: trio.SocketStream) -> None: + fd = stream.socket.fileno() + # Synthesise the worker config (socketserver workers default to the + # thread model, matching the legacy socket server). + config = json.dumps( + { + "id": "socketworker%d" % next(counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet._trio_worker", + config, + "--socket-fd", + str(fd), + ], + pass_fds=[fd], + ) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if once: + scope.cancel() + return + # Reap the worker without blocking other connections. + await trio.to_thread.run_sync(proc.wait) + + await trio.serve_listeners(handler, listeners) + + def main(argv: list[str] | None = None) -> None: """Console entry point (``execnet-socketserver``). - Bind a socket and serve gateway connections. Intended to be run directly, - e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + Serve execnet gateway connections over a socket, spawning a worker + subprocess per connection. Intended to be run directly, e.g. provisioned on + a host with ``uvx --from execnet execnet-socketserver``. """ import argparse - from execnet.gateway_base import get_execmodel + import trio parser = argparse.ArgumentParser( prog="execnet-socketserver", @@ -138,10 +197,7 @@ def main(argv: list[str] | None = None) -> None: help="serve a single connection and exit instead of looping", ) args = parser.parse_args(argv) - - execmodel = get_execmodel("thread") - serversock = bind_and_listen(args.hostport, execmodel) - startserver(serversock, execmodel, loop=not args.once) + trio.run(_trio_serve, args.hostport, args.once) if __name__ == "__main__": diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py index 039803fa..2b6d60dd 100644 --- a/testing/test_socketserver_cli.py +++ b/testing/test_socketserver_cli.py @@ -1,11 +1,14 @@ -"""Test the ``execnet-socketserver`` console entry point end to end.""" +"""Test the ``execnet-socketserver`` console entry point end to end. + +The Trio socketserver binds a port and spawns a ``python -m execnet._trio_worker`` +subprocess per connection (no inline code execution); the coordinator connects +over a Trio TCP stream. +""" from __future__ import annotations import shutil -import socket import subprocess -import time from collections.abc import Iterator import pytest @@ -19,33 +22,25 @@ ) -def _free_port() -> int: - s = socket.socket() - s.bind(("127.0.0.1", 0)) - port = s.getsockname()[1] - s.close() - return port - - @pytest.fixture def socketserver_port() -> Iterator[int]: assert SERVER is not None - port = _free_port() proc = subprocess.Popen( - [SERVER, f"127.0.0.1:{port}"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + [SERVER, ":0"], # ephemeral port; it prints the one it bound + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, ) try: - # loop mode: a throwaway probe just consumes one accept iteration - for _ in range(100): - try: - socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + assert proc.stdout is not None + port = None + while True: + line = proc.stdout.readline() + if not line: + pytest.fail("execnet-socketserver exited before binding") + if "listening on" in line: + port = int(line.split()[-1]) break - except OSError: - time.sleep(0.1) - else: - pytest.fail("execnet-socketserver did not start") yield port finally: proc.kill() From b83ff3be6e62bdf829a56e5a79758dd5885f9c60 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:34:48 +0200 Subject: [PATCH 12/66] feat: migrate installvia to a GATEWAY_START_SOCKET protocol message Since execnet is now always installed on both sides, replace the remote_exec source-shipping used by `socket//installvia=` with a first-class protocol message. The coordinator sends GATEWAY_START_SOCKET (with the bind host) on a request channel; the via gateway's Trio host binds an ephemeral port, replies with the (host, port) on that channel, and serves the one connection by spawning a worker subprocess. The coordinator then connects to it over the Trio socket path. This makes the inline-exec socketserver dead, so drop it: the `execnet-socketserver` script is now only the Trio server + CLI, and the Windows service wrapper launches that. The legacy `__channelexec__` mode, `bind_and_listen`, `startserver`, and the inline `exec` are gone. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 107 +++++++++++-- src/execnet/gateway_base.py | 11 ++ src/execnet/script/socketserver.py | 184 ++-------------------- src/execnet/script/socketserverservice.py | 7 +- 4 files changed, 124 insertions(+), 185 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index c58d0c34..ed78dd7d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,6 +7,8 @@ from __future__ import annotations +import itertools +import json import os import queue import subprocess @@ -25,6 +27,8 @@ from .gateway_base import ExecModel from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import dumps_internal +from .gateway_base import loads_internal from .gateway_base import trace if TYPE_CHECKING: @@ -703,15 +707,21 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - assert spec.socket is not None - host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) + if getattr(spec, "installvia", None): + realhost, realport = start_socketserver_via(group[spec.installvia]) + address = (realhost, realport) + remoteaddress = "%s:%d" % (realhost, realport) + else: + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + remoteaddress = spec.socket async def _create_and_attach() -> Gateway: try: stream = await trio.open_tcp_stream(*address) except OSError as exc: - raise HostNotFound(spec.socket) from exc + raise HostNotFound(remoteaddress) from exc io = SocketStreamIO(stream) try: ack = await io.read_exact(1) @@ -725,23 +735,96 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) def should_use_trio_socket(spec: Any) -> bool: - """Trio path for direct ``socket=host:port`` gateways. - - ``installvia`` (start a server through another gateway) still uses the legacy - path for now. - """ + """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" if not trio_host_enabled(): return False if not getattr(spec, "socket", None): return False - if getattr(spec, "installvia", None): - return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +_socket_worker_counter = itertools.count() + + +def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: + """Spawn a worker subprocess serving over the inherited socket ``fd``.""" + import execnet + + config = json.dumps( + { + "id": "socketworker%d" % next(_socket_worker_counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + return subprocess.Popen( + [sys.executable, "-m", "execnet._trio_worker", config, "--socket-fd", str(fd)], + pass_fds=[fd], + ) + + +async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> None: + """Hand an accepted socket to a fresh worker subprocess (server side). + + ``reap`` waits for the worker (loop server); when false the worker outlives + this task (one-shot / installvia). + """ + proc = _spawn_socket_worker(stream.socket.fileno()) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if reap: + await trio.to_thread.run_sync(proc.wait) + + +async def _start_socket_and_reply( + gateway: BaseGateway, channelid: int, bind_host: str +) -> None: + """Bind an ephemeral port, reply with its address, then serve one connection. + + Runs as a task on the worker's Trio host (scheduled from the message + handler). The reply travels back on ``channelid`` like a STATUS reply. + """ + listeners = await trio.open_tcp_listeners(0, host=bind_host) + addr = listeners[0].socket.getsockname() + gateway._send(Message.CHANNEL_DATA, channelid, dumps_internal((addr[0], addr[1]))) + gateway._send(Message.CHANNEL_CLOSE, channelid) + + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + await serve_socket_connection(stream, reap=True) + + +def handle_start_socket(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SOCKET`` (on the host thread).""" + bind_host = loads_internal(data) + assert isinstance(bind_host, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + # The receiver runs on the host thread, so schedule the async work directly. + host.start_soon(_start_socket_and_reply, gateway, channelid, bind_host) + + +def start_socketserver_via( + via_gateway: Any, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``via_gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = via_gateway.newchannel() + via_gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = channel.receive() + channel.waitclose() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c9933fe0..764f6a0a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -548,6 +548,17 @@ def _channel_last_message(message: Message, gateway: BaseGateway) -> None: CHANNEL_LAST_MESSAGE = 7 _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) + def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: + # Start a one-shot socketserver on this (Trio) gateway's host and reply + # with the bound (host, port) on the request channel. Handled natively + # instead of shipping source via remote_exec. + from . import _trio_host + + _trio_host.handle_start_socket(gateway, message.channelid, message.data) + + GATEWAY_START_SOCKET = 8 + _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index ea580bcd..eb2a86ae 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -1,131 +1,19 @@ #! /usr/bin/env python -""" -start socket based minimal readline exec server - -it can exeuted in 2 modes of operation - -1. as normal script, that listens for new connections - -2. via existing_gateway.remote_exec (as imported module) +"""Trio socket server for execnet gateways. +Listens on a TCP port and hands each accepted connection (by fd) to a fresh +``python -m execnet._trio_worker`` subprocess that serves the gateway over it. +No code is executed inline. Run directly, e.g. provisioned anywhere with +``uvx --from execnet execnet-socketserver``. """ -# this part of the program only executes on the server side -# from __future__ import annotations -import os -import sys -from typing import TYPE_CHECKING - -try: - import fcntl -except ImportError: - fcntl = None # type: ignore[assignment] - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - -progname = "socket_readline_exec_server-1.2" - - -debug = 0 - -if debug: # and not os.isatty(sys.stdin.fileno()) - f = open("/tmp/execnet-socket-pyout.log", "w") - old = sys.stdout, sys.stderr - sys.stdout = sys.stderr = f - - -def print_(*args) -> None: - print(" ".join(str(arg) for arg in args)) - - -exec( - """def exec_(source, locs): - exec(source, locs)""" -) - - -def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: - print_(progname, "Entering Accept loop", serversock.getsockname()) - clientsock, address = serversock.accept() - print_(progname, "got new connection from {} {}".format(*address)) - clientfile = clientsock.makefile("rb") - print_("reading line") - # rstrip so that we can use \r\n for telnet testing - source = clientfile.readline().rstrip() - clientfile.close() - g = {"clientsock": clientsock, "address": address, "execmodel": execmodel} - source = eval(source) - if source: - co = compile(source + "\n", "", "exec") - print_(progname, "compiled source, executing") - try: - exec_(co, g) # type: ignore[name-defined] # noqa: F821 - finally: - print_(progname, "finished executing code") - # background thread might hold a reference to this (!?) - # clientsock.close() - - -def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): - socket = execmodel.socket - if isinstance(hostport, str): - host, port = hostport.split(":") - hostport = (host, int(port)) - serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # set close-on-exec - if hasattr(fcntl, "FD_CLOEXEC"): - old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD) - fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) - # allow the address to be reused in a reasonable amount of time - if os.name == "posix" and sys.platform != "cygwin": - serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - serversock.bind(hostport) - serversock.listen(5) - return serversock - - -def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: - execute_path = os.getcwd() - try: - while 1: - try: - exec_from_one_connection(serversock, execmodel) - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: - if debug: - import traceback - - traceback.print_exc() - else: - print_("got exception", exc) - os.chdir(execute_path) - if not loop: - break - finally: - print_("leaving socketserver execloop") - serversock.shutdown(2) - async def _trio_serve(hostport: str, once: bool) -> None: - """Trio TCP server that spawns a worker subprocess per connection. - - No code is executed inline: each accepted socket is handed (by fd) to a - fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the - gateway over it. - """ - import itertools - import json - import subprocess - import trio - import execnet + from execnet import _trio_host host, _, port_str = hostport.rpartition(":") listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) @@ -133,50 +21,22 @@ async def _trio_serve(hostport: str, once: bool) -> None: # Report the bound address (port may be ephemeral) for callers to read. print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) - counter = itertools.count() + if once: + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + # The worker outlives this one-shot server. + await _trio_host.serve_socket_connection(stream, reap=False) + return - with trio.CancelScope() as scope: + async def handler(stream: trio.SocketStream) -> None: + await _trio_host.serve_socket_connection(stream, reap=True) - async def handler(stream: trio.SocketStream) -> None: - fd = stream.socket.fileno() - # Synthesise the worker config (socketserver workers default to the - # thread model, matching the legacy socket server). - config = json.dumps( - { - "id": "socketworker%d" % next(counter), - "execmodel": "thread", - "coordinator_version": execnet.__version__, - } - ) - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "execnet._trio_worker", - config, - "--socket-fd", - str(fd), - ], - pass_fds=[fd], - ) - # The child forked with a copy of the fd; release ours. - await stream.aclose() - if once: - scope.cancel() - return - # Reap the worker without blocking other connections. - await trio.to_thread.run_sync(proc.wait) - - await trio.serve_listeners(handler, listeners) + await trio.serve_listeners(handler, listeners) def main(argv: list[str] | None = None) -> None: - """Console entry point (``execnet-socketserver``). - - Serve execnet gateway connections over a socket, spawning a worker - subprocess per connection. Intended to be run directly, e.g. provisioned on - a host with ``uvx --from execnet execnet-socketserver``. - """ + """Console entry point (``execnet-socketserver``).""" import argparse import trio @@ -202,13 +62,3 @@ def main(argv: list[str] | None = None) -> None: if __name__ == "__main__": main() - -elif __name__ == "__channelexec__": - chan: Channel = globals()["channel"] - execmodel = chan.gateway.execmodel - bindname = chan.receive() - assert isinstance(bindname, (str, tuple)) - sock = bind_and_listen(bindname, execmodel) - port = sock.getsockname() - chan.send(port) - startserver(sock, execmodel) diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py index 18e375c4..1c991307 100644 --- a/src/execnet/script/socketserverservice.py +++ b/src/execnet/script/socketserverservice.py @@ -15,8 +15,6 @@ import win32service import win32serviceutil -from execnet.gateway_base import get_execmodel - from . import socketserver appname = "ExecNetSocketServer" @@ -65,12 +63,9 @@ def SvcDoRun(self) -> None: hostport = ":8888" print("Starting py.execnet SocketServer on %s" % hostport) - exec_model = get_execmodel("thread") - serversock = socketserver.bind_and_listen(hostport, exec_model) thread = threading.Thread( - target=socketserver.startserver, args=(serversock,), kwargs={"loop": True} + target=socketserver.main, args=([hostport],), daemon=True ) - thread.setDaemon(True) thread.start() # wait to be stopped or self.WAIT_TIME to pass From 62b746b705dece7e79484fcf9db57064aa049734 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:49:19 +0200 Subject: [PATCH 13/66] refactor: remove the legacy socket coordinator code With direct socket and installvia both on the Trio path, the legacy socket coordinator is dead. Delete gateway_socket.py (SocketIO, create_io, start_via), drop bootstrap_socket and the socket branch from gateway_bootstrap.bootstrap, and remove the now-unreachable `elif spec.socket:` arm from Group.makegateway. All socket gateways now go through _trio_host.makegateway_socket_trio. Co-Authored-By: Claude Opus 4.8 --- src/execnet/gateway_bootstrap.py | 22 ------- src/execnet/gateway_socket.py | 102 ------------------------------- src/execnet/multi.py | 5 -- 3 files changed, 129 deletions(-) delete mode 100644 src/execnet/gateway_socket.py diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py index e9d7efe1..5707dff4 100644 --- a/src/execnet/gateway_bootstrap.py +++ b/src/execnet/gateway_bootstrap.py @@ -55,26 +55,6 @@ def bootstrap_exec(io: IO, spec: XSpec) -> None: raise HostNotFound(io.remoteaddress) from None -def bootstrap_socket(io: IO, id) -> None: - # XXX: switch to spec - from execnet.gateway_socket import SocketIO - - sendexec( - io, - inspect.getsource(gateway_base), - "import socket", - inspect.getsource(SocketIO), - "try: execmodel", - "except NameError:", - " execmodel = get_execmodel('thread')", - "io = SocketIO(clientsock, execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % id, - ) - s = io.read(1) - assert s == b"1" - - def sendexec(io: IO, *sources: str) -> None: source = "\n".join(sources) io.write((repr(source) + "\n").encode("utf-8")) @@ -88,8 +68,6 @@ def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: bootstrap_import(io, spec) elif spec.ssh or spec.vagrant_ssh: bootstrap_exec(io, spec) - elif spec.socket: - bootstrap_socket(io, spec) else: raise ValueError("unknown gateway type, can't bootstrap") gw = execnet.Gateway(io, spec) diff --git a/src/execnet/gateway_socket.py b/src/execnet/gateway_socket.py deleted file mode 100644 index be42f1ab..00000000 --- a/src/execnet/gateway_socket.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import sys -from typing import cast - -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_bootstrap import HostNotFound -from execnet.multi import Group -from execnet.xspec import XSpec - - -class SocketIO: - remoteaddress: str - - def __init__(self, sock, execmodel: ExecModel) -> None: - self.sock = sock - self.execmodel = execmodel - socket = execmodel.socket - try: - # IPTOS_LOWDELAY - sock.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) - sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - except (AttributeError, OSError): - sys.stderr.write("WARNING: cannot set socketoption") - - def read(self, numbytes: int) -> bytes: - "Read exactly 'bytes' bytes from the socket." - buf = b"" - while len(buf) < numbytes: - t = self.sock.recv(numbytes - len(buf)) - if not t: - raise EOFError - buf += t - return buf - - def write(self, data: bytes) -> None: - self.sock.sendall(data) - - def close_read(self) -> None: - try: - self.sock.shutdown(0) - except self.execmodel.socket.error: - pass - - def close_write(self) -> None: - try: - self.sock.shutdown(1) - except self.execmodel.socket.error: - pass - - def wait(self) -> None: - pass - - def kill(self) -> None: - pass - - -def start_via( - gateway: Gateway, hostport: tuple[str, int] | None = None -) -> tuple[str, int]: - """Instantiate a socketserver on the given gateway. - - Returns a host, port tuple. - """ - if hostport is None: - host, port = ("localhost", 0) - else: - host, port = hostport - - from execnet.script import socketserver - - # execute the above socketserverbootstrap on the other side - channel = gateway.remote_exec(socketserver) - channel.send((host, port)) - realhost, realport = cast("tuple[str, int]", channel.receive()) - # self._trace("new_remote received" - # "port=%r, hostname = %r" %(realport, hostname)) - if not realhost or realhost == "0.0.0.0": - realhost = "localhost" - return realhost, realport - - -def create_io(spec: XSpec, group: Group, execmodel: ExecModel) -> SocketIO: - assert spec.socket is not None - assert not spec.python, "socket: specifying python executables not yet supported" - gateway_id = spec.installvia - if gateway_id: - host, port = start_via(group[gateway_id]) - else: - host, port_str = spec.socket.split(":") - port = int(port_str) - - socket = execmodel.socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - io = SocketIO(sock, execmodel) - io.remoteaddress = "%s:%d" % (host, port) - try: - sock.connect((host, port)) - except execmodel.socket.gaierror as e: - raise HostNotFound() from e - return io diff --git a/src/execnet/multi.py b/src/execnet/multi.py index d64722ee..cc85d757 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -170,11 +170,6 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: elif spec.popen or spec.ssh or spec.vagrant_ssh: io = gateway_io.create_io(spec, execmodel=self.execmodel) gw = gateway_bootstrap.bootstrap(io, spec) - elif spec.socket: - from . import gateway_socket - - sio = gateway_socket.create_io(spec, self, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(sio, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec From 19d620d976b4a677a7e2596378a0010bd0be6491 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 23:34:36 +0200 Subject: [PATCH 14/66] feat: run popen via-gateways on Trio via GATEWAY_START_POPEN Move the common `popen//via=` proxy transport onto the Trio host with a new protocol message instead of remote_exec'ing gateway_io source. The coordinator sends GATEWAY_START_POPEN (with the sub worker config) on a request channel; the master's Trio host spawns a `python -m execnet._trio_worker` sub-worker and relays its Message protocol raw over that channel (stdin<-channel, stdout->channel). The coordinator wraps the channel as ChannelByteIO and runs a normal Trio session over it. ChannelByteIO is marked an interim hack: it tunnels sub frames as CHANNEL_DATA (double-framing) and should become a proper relayed transport. ssh/foreign-python via sub-gateways still use the legacy path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 13 +++- src/execnet/_trio_host.py | 136 ++++++++++++++++++++++++++++++++++++ src/execnet/gateway_base.py | 10 +++ src/execnet/multi.py | 2 + 4 files changed, 159 insertions(+), 2 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 29df051c..ba1c8408 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -58,8 +58,17 @@ threads wait until the frame is written (so abrupt ``os._exit`` cannot drop queued data). Sends from the Trio host thread (receiver callbacks) only enqueue, to avoid deadlocking the writer task. -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, -``via``, greenlet execmodels) still use the legacy thread receiver and sync +``socket`` and ``via`` gateways run on the Trio host too. ``socket`` +connects a Trio TCP stream to an ``execnet-socketserver`` (itself a Trio +listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess +workers). Infrastructure that used to be driven by ``remote_exec``-ing +source is now expressed as native protocol messages handled on the target's +Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, +reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen +sub-worker and relay its protocol over the request channel). + +Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways +and greenlet execmodels still use the legacy thread receiver and sync ``Popen`` path. Legacy thread model diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed78dd7d..e00f3de0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -9,6 +9,7 @@ import itertools import json +import math import os import queue import subprocess @@ -165,6 +166,49 @@ async def aclose_write(self) -> None: await self._stream.aclose() +_CHANNEL_EOF = object() + + +class ChannelByteIO: + """Async byte IO tunnelled over a sync execnet ``Channel``. + + INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol + as raw bytes over a channel to the master, which double-frames it (sub frame + -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the + async protocol; it should be replaced by a proper relayed transport rather + than tunnelling bytes through the channel layer. + + The channel callback (on the host loop) feeds an unbounded memory channel + that ``read_exact`` drains; writes ``channel.send`` raw frames. + """ + + def __init__(self, channel: Any) -> None: + self._channel = channel + self._send, self._recv = trio.open_memory_channel[Any](math.inf) + self._buf = bytearray() + channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + + async def read_exact(self, n: int) -> bytes: + while len(self._buf) < n: + data = await self._recv.receive() + if data is _CHANNEL_EOF: + raise EOFError("channel closed") + assert isinstance(data, bytes) + self._buf += data + out = bytes(self._buf[:n]) + del self._buf[:n] + return out + + async def write_all(self, data: bytes) -> None: + self._channel.send(data) + + async def aclose_read(self) -> None: + return + + async def aclose_write(self) -> None: + self._channel.close() + + async def adopt_socket(socket_fd: int) -> SocketStreamIO: """Worker side: wrap an inherited socket fd and send the handshake. @@ -828,3 +872,95 @@ def start_socketserver_via( if not realhost or realhost in ("0.0.0.0", "::"): realhost = "localhost" return realhost, int(realport) + + +async def _start_popen_and_relay( + gateway: BaseGateway, channelid: int, worker_config: str +) -> None: + """Spawn a popen sub-worker and relay its Message protocol over the channel. + + Runs on the master's Trio host: bytes from the channel go to the sub's + stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """ + args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] + process = await open_popen_process(args) + channel = gateway._channelfactory.new(channelid) + send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) + channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + + async def coordinator_to_sub() -> None: + assert process.stdin is not None + async for data in recv_ch: + if data is _CHANNEL_EOF: + break + await process.stdin.send_all(data) + with trio.move_on_after(5): + await process.stdin.aclose() + + async def sub_to_coordinator() -> None: + assert process.stdout is not None + while True: + data = await process.stdout.receive_some(65536) + if not data: + break + channel.send(data) + channel.close() + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(coordinator_to_sub) + nursery.start_soon(sub_to_coordinator) + finally: + with trio.move_on_after(5): + await process.wait() + + +def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" + worker_config = loads_internal(data) + assert isinstance(worker_config, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + + +def should_use_trio_via(spec: Any) -> bool: + """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + + Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "via", None): + return False + if getattr(spec, "socket", None) or getattr(spec, "ssh", None): + return False + if getattr(spec, "python", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_via_trio(group: Any, spec: Any) -> Gateway: + """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + import execnet + + from . import _provision + + master = group[spec.via] + host: TrioHost = group._ensure_trio_host() + channel = master.newchannel() + worker_config = _provision.worker_cli_arg(spec) + master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + + async def _create_and_attach() -> Gateway: + io = ChannelByteIO(channel) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad via handshake: {ack!r}") + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 764f6a0a..679e8fd5 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,6 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: + # Spawn a popen worker subprocess on this (Trio) gateway's host and relay + # its Message protocol over the request channel (the ``via`` transport). + from . import _trio_host + + _trio_host.handle_start_popen(gateway, message.channelid, message.data) + + GATEWAY_START_POPEN = 9 + _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/multi.py b/src/execnet/multi.py index cc85d757..3a9dc602 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -160,6 +160,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_ssh_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) + elif _trio_host.should_use_trio_via(spec): + gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] From 92969c5a269a223a2d2edc8ef55e1c2abdb58b3e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:32:10 +0200 Subject: [PATCH 15/66] feat: spawn arbitrary via sub-gateways with GATEWAY_START_SUB Generalize the via relay message from popen-only to a spawn-request dict carrying the sub-spec essentials plus provisioning material: a released coordinator sends a pip requirement, a dev coordinator ships its wheel for the master to materialize into the local wheel cache. The master resolves the launch locally (direct module, uv-provisioned python=, or ssh with the wheel streamed as stdin preamble), so ssh= and python= sub specs now run on the Trio path. Also route ssh=...//via=... through the via path instead of opening a direct ssh connection, and close the request channel with an error on spawn/relay failure instead of crashing the host nursery. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 182 +++++++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 71 ++++++++------ src/execnet/gateway_base.py | 13 +-- testing/test_multi.py | 18 ++++ testing/test_ssh_local.py | 20 ++++ 5 files changed, 249 insertions(+), 55 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b24bfe8b..83fd1515 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -17,10 +17,12 @@ from __future__ import annotations import json +import os import re import shlex import shutil import subprocess +import sys import tempfile from functools import cache from pathlib import Path @@ -162,52 +164,62 @@ def worker_cli_arg(spec: Any) -> str: ) +def _worker_tokens(config: str) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens. + + The literal ``python`` token is resolved by uv (inside the provisioned + environment) or the remote shell. + """ + return ["python", "-u", "-m", "execnet._trio_worker", config] + + def worker_module_tokens(spec: Any) -> list[str]: - """``python -u -m execnet._trio_worker `` tokens.""" - return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + """``python -u -m execnet._trio_worker `` tokens for ``spec``.""" + return _worker_tokens(worker_cli_arg(spec)) -def _uv_prefix(spec: Any) -> list[str]: +def _uv_tokens(python: str | None) -> list[str]: # --no-project keeps the surrounding execnet checkout from being synced. prefix = ["uv", "run", "--no-project"] - if spec.python: - prefix += ["--python", spec.python] + if python: + prefix += ["--python", python] return prefix def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" return [ - *_uv_prefix(spec), + *_uv_tokens(spec.python), "--with", coordinator_requirement(), *worker_module_tokens(spec), ] -def ssh_remote_command(spec: Any) -> tuple[str, bytes]: - """Remote shell command + stdin preamble to launch the worker over ssh. +def _remote_shell_command( + python: str | None, + config: str, + *, + requirement: str | None = None, + wheel: Path | None = None, +) -> tuple[str, bytes]: + """Remote sh command + stdin preamble launching the worker via uv. - Released coordinator -> ``uv run --with execnet== …`` with no preamble. - Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from - stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel - bytes are returned as the preamble to stream before the Message protocol. + With ``requirement`` the remote installs from an index and no preamble is + needed. With ``wheel`` a POSIX-sh prelude receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the + wheel bytes are returned as the preamble to stream before the protocol. """ - import execnet + worker = _worker_tokens(config) + uv = _uv_tokens(python) + if wheel is None: + assert requirement is not None + return shlex.join([*uv, "--with", requirement, *worker]), b"" - version = execnet.__version__ - worker = worker_module_tokens(spec) - if _RELEASED_RE.match(version): - command = shlex.join( - [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] - ) - return command, b"" - - wheel = _build_wheel(version) data = wheel.read_bytes() # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. remote_wheel = '"$d/"' + shlex.quote(wheel.name) - uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + uv_run = " ".join(shlex.quote(token) for token in [*uv, "--with"]) worker_cmd = " ".join(shlex.quote(token) for token in worker) prelude = ( f"d=$(mktemp -d) && " @@ -215,3 +227,127 @@ def ssh_remote_command(spec: Any) -> tuple[str, bytes]: f"exec {uv_run} {remote_wheel} {worker_cmd}" ) return prelude, data + + +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> wheel-shipping prelude (see ``_remote_shell_command``). + """ + import execnet + + version = execnet.__version__ + config = worker_cli_arg(spec) + if _RELEASED_RE.match(version): + return _remote_shell_command( + spec.python, config, requirement=f"execnet=={version}" + ) + return _remote_shell_command(spec.python, config, wheel=_build_wheel(version)) + + +def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str]: + """``ssh`` client argv running ``remote_command`` on host ``ssh``.""" + args = ["ssh", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args += ssh.split() + args.append(remote_command) + return args + + +def spawn_request(spec: Any) -> dict[str, Any]: + """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. + + Carries the sub-spec essentials plus provisioning material when the sub + may need it (ssh or foreign python): a released coordinator sends a pip + requirement; a dev coordinator ships its wheel bytes for the master to + materialize into its local wheel cache. + + TODO: the wheel is shipped eagerly because only the master can tell + whether the target interpreter already has execnet; a wheel-on-demand + round-trip would avoid the transfer in the common provisioned case. + """ + import execnet + + request: dict[str, Any] = { + "config": worker_cli_arg(spec), + "python": spec.python or None, + "ssh": spec.ssh or None, + "ssh_config": spec.ssh_config or None, + } + if spec.ssh or spec.python: + version = execnet.__version__ + if _RELEASED_RE.match(version): + request["requirement"] = f"execnet=={version}" + else: + wheel = _build_wheel(version) + request["wheel"] = (wheel.name, wheel.read_bytes()) + return request + + +def materialize_wheel(name: str, data: bytes) -> Path: + """Write shipped wheel bytes into the local wheel cache (idempotent).""" + target = _wheel_cache_dir() / name + if not target.exists(): + tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") + tmp.write_bytes(data) + tmp.replace(target) + return target + + +def _requested_requirement(request: dict[str, Any]) -> tuple[str | None, Path | None]: + """(uv requirement, local wheel path) from a spawn request's material.""" + requirement = request.get("requirement") + if isinstance(requirement, str): + return requirement, None + shipped = request.get("wheel") + if shipped is not None: + name, data = shipped + path = materialize_wheel(name, data) + return str(path), path + return None, None + + +def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: + """(argv, stdin preamble) spawning a requested sub-worker on this host. + + Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs + this interpreter's worker module, a foreign ``python`` runs directly when + it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps + the remote uv command (streaming a shipped wheel as the preamble for dev + versions). + """ + from .gateway_io import shell_split_path + + config = request["config"] + assert isinstance(config, str) + python = request.get("python") + ssh = request.get("ssh") + if ssh: + assert isinstance(ssh, str) + requirement, wheel = _requested_requirement(request) + if requirement is None: + raise RuntimeError("ssh spawn request without provisioning material") + command, preamble = _remote_shell_command( + python, config, requirement=requirement, wheel=wheel + ) + return ssh_argv(ssh, request.get("ssh_config"), command), preamble + if python: + assert isinstance(python, str) + if target_has_execnet(python): + argv = [*shell_split_path(python), "-u", "-m", "execnet._trio_worker"] + return [*argv, config], b"" + requirement, _ = _requested_requirement(request) + if requirement is None or not uv_available(): + raise RuntimeError( + f"cannot provision sub-worker for python={python!r}: " + "uv and provisioning material required" + ) + return [ + *_uv_tokens(python), + "--with", + requirement, + *_worker_tokens(config), + ], b"" + return [sys.executable, "-u", "-m", "execnet._trio_worker", config], b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e00f3de0..709a8b64 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -17,6 +17,7 @@ import threading from collections.abc import Awaitable from collections.abc import Callable +from contextlib import suppress from typing import TYPE_CHECKING from typing import Any from typing import Protocol @@ -712,13 +713,8 @@ def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: from . import _provision remote_command, preamble = _provision.ssh_remote_command(spec) - args = ["ssh", "-C"] - if getattr(spec, "ssh_config", None): - args += ["-F", spec.ssh_config] assert spec.ssh is not None - args += spec.ssh.split() - args.append(remote_command) - return args, preamble + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: @@ -730,11 +726,17 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + """Trio path for ssh gateways (worker provisioned on the remote via uv). + + ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned + by the master; that goes through the via path instead. + """ if not trio_host_enabled(): return False if not getattr(spec, "ssh", None): return False + if getattr(spec, "via", None): + return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -874,22 +876,32 @@ def start_socketserver_via( return realhost, int(realport) -async def _start_popen_and_relay( - gateway: BaseGateway, channelid: int, worker_config: str +async def _start_sub_and_relay( + gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a popen sub-worker and relay its Message protocol over the channel. + """Spawn a requested sub-worker and relay its Message protocol over the channel. Runs on the master's Trio host: bytes from the channel go to the sub's stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed + before the relayed protocol bytes. """ - args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] - process = await open_popen_process(args) + from . import _provision + channel = gateway._channelfactory.new(channelid) + try: + args, preamble = _provision.sub_spawn_argv(request) + process = await open_popen_process(args) + except Exception as exc: + channel.close(f"could not spawn via sub-gateway: {exc}") + return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) async def coordinator_to_sub() -> None: assert process.stdin is not None + if preamble: + await process.stdin.send_all(preamble) async for data in recv_ch: if data is _CHANNEL_EOF: break @@ -910,38 +922,44 @@ async def sub_to_coordinator() -> None: async with trio.open_nursery() as nursery: nursery.start_soon(coordinator_to_sub) nursery.start_soon(sub_to_coordinator) + except Exception as exc: + # Do not let a relay failure crash the host nursery; surface it on + # the channel so the coordinator does not hang on the handshake. + gateway._trace("via sub relay failed:", exc) + with suppress(Exception): + channel.close(f"via sub-gateway relay failed: {exc}") finally: with trio.move_on_after(5): await process.wait() -def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: - """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" - worker_config = loads_internal(data) - assert isinstance(worker_config, str) +def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SUB`` (on the host thread).""" + request = loads_internal(data) + assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] - host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + host.start_soon(_start_sub_and_relay, gateway, channelid, request) def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). - Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and + relays its Message protocol. Socket subs go through ``installvia`` + instead; vagrant stays on the legacy path. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "ssh", None): - return False - if getattr(spec, "python", None): + if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet from . import _provision @@ -949,8 +967,9 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() channel = master.newchannel() - worker_config = _provision.worker_cli_arg(spec) - master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + request = _provision.spawn_request(spec) + master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) + remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) @@ -960,7 +979,7 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 679e8fd5..2736be96 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,15 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: - # Spawn a popen worker subprocess on this (Trio) gateway's host and relay - # its Message protocol over the request channel (the ``via`` transport). + def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: + # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this + # (Trio) gateway's host and relay its Message protocol over the request + # channel (the ``via`` transport). from . import _trio_host - _trio_host.handle_start_popen(gateway, message.channelid, message.data) + _trio_host.handle_start_sub(gateway, message.channelid, message.data) - GATEWAY_START_POPEN = 9 - _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + GATEWAY_START_SUB = 9 + _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) class GatewayReceivedTerminate(Exception): diff --git a/testing/test_multi.py b/testing/test_multi.py index f3166503..e0390298 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -231,6 +231,24 @@ def test_terminate_with_proxying(self) -> None: group.makegateway("popen//via=master//id=worker") group.terminate(1.0) + def test_via_foreign_python(self) -> None: + # A python= sub-spec through a via master: the master resolves the + # interpreter locally (this interpreter has execnet, so the sub runs + # the worker module directly, no uv provisioning). + import sys + + group = Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"popen//python={sys.executable}//via=master//id=sub" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(1.0) + @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate(execmodel: ExecModel) -> None: diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 72dea348..c21135d1 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -149,3 +149,23 @@ def test_ssh_roundtrip( assert channel.receive() == 42 finally: group.terminate(timeout=5.0) + + +def test_ssh_via_roundtrip(ssh_config: str) -> None: + """An ssh sub-gateway spawned by a popen master (GATEWAY_START_SUB relay). + + The master runs the ssh client; for a dev coordinator the wheel travels + coordinator -> master (in the spawn request) -> remote (ssh stdin preamble). + """ + group = execnet.Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}" + "//via=master//id=sshvia" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 48d328c6d61f58a342fb6c5bbfe6f860c099d4ef Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:53:29 +0200 Subject: [PATCH 16/66] feat: run vagrant_ssh gateways on the Trio path vagrant_ssh= now launches the uv-provisioned worker through `vagrant ssh -- -C ` (mirroring the ssh argv), both as a direct gateway and as a via sub-gateway through GATEWAY_START_SUB. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 31 ++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 36 +++++++++++++++++++++++++++++++----- src/execnet/multi.py | 2 ++ testing/test_provision.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 83fd1515..c55fb431 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -256,6 +256,21 @@ def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str] return args +def vagrant_ssh_argv( + machine: str, ssh_config: str | None, remote_command: str +) -> list[str]: + """``vagrant ssh`` argv running ``remote_command`` on the named VM. + + Everything after ``--`` is passed through to the underlying ssh client, + mirroring ``ssh_argv``. + """ + args = ["vagrant", "ssh", machine, "--", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args.append(remote_command) + return args + + def spawn_request(spec: Any) -> dict[str, Any]: """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. @@ -274,9 +289,10 @@ def spawn_request(spec: Any) -> dict[str, Any]: "config": worker_cli_arg(spec), "python": spec.python or None, "ssh": spec.ssh or None, + "vagrant_ssh": spec.vagrant_ssh or None, "ssh_config": spec.ssh_config or None, } - if spec.ssh or spec.python: + if spec.ssh or spec.vagrant_ssh or spec.python: version = execnet.__version__ if _RELEASED_RE.match(version): request["requirement"] = f"execnet=={version}" @@ -324,15 +340,20 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: assert isinstance(config, str) python = request.get("python") ssh = request.get("ssh") - if ssh: - assert isinstance(ssh, str) + vagrant = request.get("vagrant_ssh") + if ssh or vagrant: requirement, wheel = _requested_requirement(request) if requirement is None: - raise RuntimeError("ssh spawn request without provisioning material") + raise RuntimeError("remote spawn request without provisioning material") command, preamble = _remote_shell_command( python, config, requirement=requirement, wheel=wheel ) - return ssh_argv(ssh, request.get("ssh_config"), command), preamble + ssh_config = request.get("ssh_config") + if ssh: + assert isinstance(ssh, str) + return ssh_argv(ssh, ssh_config, command), preamble + assert isinstance(vagrant, str) + return vagrant_ssh_argv(vagrant, ssh_config, command), preamble if python: assert isinstance(python, str) if target_has_execnet(python): diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 709a8b64..2a612996 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -725,6 +725,32 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) +def should_use_trio_vagrant(spec: Any) -> bool: + """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "vagrant_ssh", None): + return False + if getattr(spec, "via", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: + """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + args = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble + ) + + def should_use_trio_ssh(spec: Any) -> bool: """Trio path for ssh gateways (worker provisioned on the remote via uv). @@ -942,17 +968,16 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). + """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` - instead; vagrant stays on the legacy path. + relays its Message protocol. Socket subs go through ``installvia`` instead. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): + if getattr(spec, "socket", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -969,7 +994,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: channel = master.newchannel() request = _provision.spawn_request(spec) master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) - remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None + remote = spec.ssh or spec.vagrant_ssh + remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 3a9dc602..29b8bd6a 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_vagrant(spec): + gw = _trio_host.makegateway_vagrant_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) elif _trio_host.should_use_trio_via(spec): diff --git a/testing/test_provision.py b/testing/test_provision.py index 8df8775a..84e660e5 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -38,3 +38,39 @@ def test_ssh_remote_command_dev_ships_wheel() -> None: assert "mktemp -d" in command assert f"head -c {len(preamble)}" in command assert preamble[:2] == b"PK" # a wheel is a zip archive + + +def test_vagrant_ssh_argv() -> None: + argv = _provision.vagrant_ssh_argv("default", None, "run-worker") + assert argv == ["vagrant", "ssh", "default", "--", "-C", "run-worker"] + argv = _provision.vagrant_ssh_argv("default", "/tmp/cfg", "run-worker") + assert argv == [ + "vagrant", + "ssh", + "default", + "--", + "-C", + "-F", + "/tmp/cfg", + "run-worker", + ] + + +def test_sub_spawn_argv_plain_popen() -> None: + import sys + + argv, preamble = _provision.sub_spawn_argv({"config": "{}"}) + assert argv == [sys.executable, "-u", "-m", "execnet._trio_worker", "{}"] + assert preamble == b"" + + +def test_sub_spawn_argv_vagrant_released() -> None: + request = { + "config": "{}", + "vagrant_ssh": "default", + "requirement": "execnet==9.9.9", + } + argv, preamble = _provision.sub_spawn_argv(request) + assert argv[:5] == ["vagrant", "ssh", "default", "--", "-C"] + assert "execnet==9.9.9" in argv[-1] + assert preamble == b"" From 3d1d31efb6be1fc56df527d2e4cae8bd4e67e50e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 09:48:52 +0200 Subject: [PATCH 17/66] refactor!: remove the legacy bootstrap stack and EXECNET_TRIO_HOST hatch The Trio host is now the only IO path. Delete gateway_io.py (ProxyIO, Popen2IOMaster, source bootstrap lines) and gateway_bootstrap.py, the Popen2IO sync pipe IO, the thread receiver, and WorkerGateway.serve(); makegateway dispatches directly on the spec. shell_split_path moves to _provision, and HostNotFound moves to gateway_base and now subclasses ConnectionError so generic OSError handling catches unreachable hosts. Co-Authored-By: Claude Fable 5 --- doc/implnotes.rst | 31 +--- src/execnet/__init__.py | 2 +- src/execnet/_provision.py | 15 +- src/execnet/_trio_host.py | 102 +------------ src/execnet/_trio_worker.py | 2 +- src/execnet/gateway.py | 23 +-- src/execnet/gateway_base.py | 190 +++-------------------- src/execnet/gateway_bootstrap.py | 74 --------- src/execnet/gateway_io.py | 255 ------------------------------- src/execnet/multi.py | 28 +--- testing/test_basics.py | 148 ++++-------------- testing/test_gateway.py | 26 +--- testing/test_ssh_local.py | 8 +- testing/test_xspec.py | 26 ++-- 14 files changed, 111 insertions(+), 819 deletions(-) delete mode 100644 src/execnet/gateway_bootstrap.py delete mode 100644 src/execnet/gateway_io.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index ba1c8408..eed92232 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -64,26 +64,11 @@ listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess workers). Infrastructure that used to be driven by ``remote_exec``-ing source is now expressed as native protocol messages handled on the target's Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, -reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen -sub-worker and relay its protocol over the request channel). - -Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways -and greenlet execmodels still use the legacy thread receiver and sync -``Popen`` path. - -Legacy thread model -------------------- - -After bootstrapping, ``BaseGateway`` opens a receiver thread which -accepts encoded messages and triggers actions to interpret them. -Sending of channel data items happens directly through -write operations to InputOutput objects so there is no -separate send thread. - -Code execution messages are scheduled on a WorkerPool. -On the worker, ``serve()`` integrates the main thread as the -primary executor when using the ``thread`` / ``main_thread_only`` -models. - -The receiver thread terminates if the remote side sends -a gateway termination message or if the IO-connection drops. +reply with its address) and ``GATEWAY_START_SUB`` (``via`` -> spawn a +sub-worker — popen, foreign python, ssh, or vagrant — and relay its protocol +over the request channel). + +The Trio host is the only IO path; the legacy thread receiver and the +source-shipping bootstrap have been removed. The receiver task terminates +when the remote side sends a gateway termination message or the +IO-connection drops. diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 1edf3254..cb91cb66 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -12,6 +12,7 @@ from .gateway_base import Channel from .gateway_base import DataFormatError from .gateway_base import DumpError +from .gateway_base import HostNotFound from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -19,7 +20,6 @@ from .gateway_base import dumps from .gateway_base import load from .gateway_base import loads -from .gateway_bootstrap import HostNotFound from .multi import Group from .multi import MultiChannel from .multi import default_group diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index c55fb431..b1c16335 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -36,6 +36,17 @@ def uv_available() -> bool: return shutil.which("uv") is not None +def shell_split_path(path: str) -> list[str]: + """Split a ``python=`` value into argv tokens with shell lexing. + + Takes care to handle Windows' ``\\`` correctly. + """ + if sys.platform.startswith("win"): + # replace \\ by / otherwise shlex will strip them out + path = path.replace("\\", "/") + return shlex.split(path) + + @cache def target_has_execnet(python: str) -> bool: """Whether interpreter ``python`` can already import execnet + trio. @@ -43,8 +54,6 @@ def target_has_execnet(python: str) -> bool: When true the worker can be launched directly on that interpreter (preserving ``sys.executable``); otherwise it must be uv-provisioned. """ - from .gateway_io import shell_split_path - argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) @@ -334,8 +343,6 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: the remote uv command (streaming a shipped wheel as the preamble for dev versions). """ - from .gateway_io import shell_split_path - config = request["config"] assert isinstance(config, str) python = request.get("python") diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 2a612996..59f6b47f 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import os import queue import subprocess import sys @@ -40,37 +39,6 @@ T = TypeVar("T") _CLOSE_WRITE = object() -_ENABLED_ENV = "EXECNET_TRIO_HOST" - - -def trio_host_enabled() -> bool: - """Return whether the Trio IO path should be used when applicable.""" - value = os.environ.get(_ENABLED_ENV, "1").strip().lower() - return value not in ("0", "false", "no", "off") - - -def should_use_trio_popen(spec: Any) -> bool: - """Trio path for local popen. - - Same-interpreter popen launches the worker module directly; a foreign - interpreter (``python=``) is provisioned via ``uv`` and only taken when - ``uv`` is available (otherwise the legacy source-copy path handles it). - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "popen", False): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - if getattr(spec, "python", None): - from . import _provision - - # Direct launch if the interpreter already has execnet; else uv-provision. - return _provision.target_has_execnet(spec.python) or _provision.uv_available() - return True class AsyncByteIO(Protocol): @@ -467,7 +435,7 @@ async def _finish(self) -> None: self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() - # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. self._done.set() @@ -593,9 +561,7 @@ def popen_module_args(spec: Any) -> list[str]: from . import _provision if getattr(spec, "python", None): - from .gateway_io import shell_split_path - - interpreter = shell_split_path(spec.python) + interpreter = _provision.shell_split_path(spec.python) else: interpreter = [sys.executable] @@ -648,7 +614,7 @@ def _open_trio_gateway( """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() @@ -677,7 +643,7 @@ async def _create_and_attach() -> Gateway: await process.wait() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -725,18 +691,6 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_vagrant(spec: Any) -> bool: - """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" - if not trio_host_enabled(): - return False - if not getattr(spec, "vagrant_ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" from . import _provision @@ -751,22 +705,6 @@ def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv). - - ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned - by the master; that goes through the via path instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """Connect a Trio TCP stream to a running ``execnet-socketserver``. @@ -776,7 +714,7 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() if getattr(spec, "installvia", None): @@ -804,7 +742,7 @@ async def _create_and_attach() -> Gateway: await stream.aclose() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -813,16 +751,6 @@ async def _create_and_attach() -> Gateway: return host.call(_create_and_attach) -def should_use_trio_socket(spec: Any) -> bool: - """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" - if not trio_host_enabled(): - return False - if not getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - _socket_worker_counter = itertools.count() @@ -967,22 +895,6 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: host.start_soon(_start_sub_and_relay, gateway, channelid, request) -def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). - - The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "via", None): - return False - if getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_via_trio(group: Any, spec: Any) -> Gateway: """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet @@ -1002,7 +914,7 @@ async def _create_and_attach() -> Gateway: ack = await io.read_exact(1) if ack != b"1": raise EOFError(f"bad via handshake: {ack!r}") - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc137fb..9e1f1be7 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -215,7 +215,7 @@ def _build_worker_gateway( main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] + gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 6e336d6f..593bc6fe 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,21 +26,14 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__( - self, - io: IO, - spec: XSpec, - *, - trio_session: object | None = None, - defer_receive: bool = False, - ) -> None: - """:private:""" + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - if trio_session is not None: - self._attach_trio_session(trio_session) - elif not defer_receive: - self._initreceive() @property def remoteaddress(self) -> str: @@ -102,9 +95,7 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session - if session is not None: - return bool(session.is_alive()) - return self._receivepool.active_count() > 0 + return session is not None and bool(session.is_alive()) def remote_status(self) -> RemoteStatus: """Obtain information about the remote execution status.""" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2736be96..0260b416 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,4 +1,4 @@ -"""Base execnet gateway code send to the other side for bootstrapping. +"""Core gateway, channel and serialization code shared by coordinator and worker. :copyright: 2004-2015 :authors: @@ -389,48 +389,6 @@ def trace(*msg: object) -> None: notrace = trace = lambda *msg: None -class Popen2IO: - error = (IOError, OSError, EOFError) - - def __init__(self, outfile, infile, execmodel: ExecModel) -> None: - # we need raw byte streams - self.outfile, self.infile = outfile, infile - if sys.platform == "win32": - import msvcrt - - try: - msvcrt.setmode(infile.fileno(), os.O_BINARY) - msvcrt.setmode(outfile.fileno(), os.O_BINARY) - except (AttributeError, OSError): - pass - self._read = getattr(infile, "buffer", infile).read - self._write = getattr(outfile, "buffer", outfile).write - self.execmodel = execmodel - - def read(self, numbytes: int) -> bytes: - """Read exactly 'numbytes' bytes from the pipe.""" - # a file in non-blocking mode may return less bytes, so we loop - buf = b"" - while numbytes > len(buf): - data = self._read(numbytes - len(buf)) - if not data: - raise EOFError("expected %d bytes, got %d" % (numbytes, len(buf))) - buf += data - return buf - - def write(self, data: bytes) -> None: - """Write out all data bytes.""" - assert isinstance(data, bytes) - self._write(data) - self.outfile.flush() - - def close_read(self) -> None: - self.infile.close() - - def close_write(self) -> None: - self.outfile.close() - - class Message: """Encapsulates Messages and their wire protocol.""" @@ -572,7 +530,11 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: class GatewayReceivedTerminate(Exception): - """Receiverthread got termination message.""" + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" def geterrortext( @@ -1043,6 +1005,8 @@ class BaseGateway: _sysex = sysex id = "" _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1054,7 +1018,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext - self._receivepool = WorkerPool(self.execmodel) self._trio_session = None def _trace(self, *msg: object) -> None: @@ -1064,43 +1027,6 @@ def _attach_trio_session(self, session: Any) -> None: """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" self._trio_session = session - def _initreceive(self) -> None: - if self._trio_session is not None: - return - self._receivepool.spawn(self._thread_receiver) - - def _thread_receiver(self) -> None: - def log(*msg: object) -> None: - self._trace("[receiver-thread]", *msg) - - log("RECEIVERTHREAD: starting to run") - io = self._io - try: - while 1: - msg = Message.from_io(io) - log("received", msg) - with self._receivelock: - msg.received(self) - del msg - except (KeyboardInterrupt, GatewayReceivedTerminate): - pass - except EOFError as exc: - log("EOF without prior gateway termination message") - self._error = exc - except Exception as exc: - log(self._geterrortext(exc)) - log("finishing receiving thread") - # wake up and terminate any execution waiting to receive - self._channelfactory._finished_receiving() - log("terminating execution") - self._terminate_execution() - log("closing read") - self._io.close_read() - log("closing write") - self._io.close_write() - log("terminating our receive pseudo pool") - self._receivepool.trigger_shutdown() - def _terminate_execution(self) -> None: pass @@ -1136,41 +1062,25 @@ def newchannel(self) -> Channel: return self._channelfactory.new() def join(self, timeout: float | None = None) -> None: - """Wait for receiverthread to terminate.""" - self._trace("waiting for receiver thread to finish") + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") session = self._trio_session if session is not None: session.wait_done(timeout) - return - self._receivepool.waitall(timeout) class WorkerGateway(BaseGateway): _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + _executetask_complete: Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - trio_exec = getattr(self, "_trio_exec", None) - if trio_exec is not None: - trio_exec.schedule(channel, sourcetask) + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") return - - if self._execpool.execmodel.backend == "main_thread_only": - assert self._executetask_complete is not None - # It's necessary to wait for a short time in order to ensure - # that we do not report a false-positive deadlock error, since - # channel close does not elicit a response that would provide - # a guarantee to remote_exec callers that the previous task - # has released the main thread. If the timeout expires then it - # should be practically impossible to report a false-positive. - if not self._executetask_complete.wait(timeout=1): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return - # It's only safe to clear here because the above wait proves - # that there is not a previous task about to set it again. - self._executetask_complete.clear() - - sourcetask_ = loads_internal(sourcetask) - self._execpool.spawn(self.executetask, (channel, sourcetask_)) + trio_exec.schedule(channel, sourcetask) def _terminate_execution(self) -> None: # called from receiverthread @@ -1192,31 +1102,6 @@ def _terminate_execution(self) -> None: ) os._exit(1) - def serve(self) -> None: - def trace(msg: str) -> None: - self._trace("[serve] " + msg) - - hasprimary = self.execmodel.backend in ("thread", "main_thread_only") - self._execpool = WorkerPool(self.execmodel, hasprimary=hasprimary) - self._executetask_complete = None - if self.execmodel.backend == "main_thread_only": - self._executetask_complete = self.execmodel.Event() - # Initialize state to indicate that there is no previous task - # executing so that we don't need a separate flag to track this. - self._executetask_complete.set() - trace("spawning receiver thread") - self._initreceive() - try: - if hasprimary: - # this will return when we are in shutdown - trace("integrating as primary thread") - self._execpool.integrate_as_primary_thread() - trace("joining receiver thread") - self.join() - except KeyboardInterrupt: - # in the worker we can't really do anything sensible - trace("swallowing keyboardinterrupt, serve finished") - def executetask( self, item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], @@ -1701,44 +1586,3 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) - - -def init_popen_io(execmodel: ExecModel) -> Popen2IO: - if not hasattr(os, "dup"): # jython - io = Popen2IO(sys.stdout, sys.stdin, execmodel) - import tempfile - - sys.stdin = tempfile.TemporaryFile("r") - sys.stdout = tempfile.TemporaryFile("w") - else: - try: - devnull = os.devnull - except AttributeError: - devnull = "NUL" if os.name == "nt" else "/dev/null" - # stdin - stdin = execmodel.fdopen(os.dup(0), "r", 1) - fd = os.open(devnull, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - - # stdout - stdout = execmodel.fdopen(os.dup(1), "w", 1) - fd = os.open(devnull, os.O_WRONLY) - os.dup2(fd, 1) - - # stderr for win32 - if os.name == "nt": - sys.stderr = execmodel.fdopen(os.dup(2), "w", 1) - os.dup2(fd, 2) - os.close(fd) - io = Popen2IO(stdout, stdin, execmodel) - # Use closefd=False since 0 and 1 are shared with - # sys.__stdin__ and sys.__stdout__. - sys.stdin = execmodel.fdopen(0, "r", 1, closefd=False) - sys.stdout = execmodel.fdopen(1, "w", 1, closefd=False) - return io - - -def serve(io: IO, id) -> None: - trace(f"creating workergateway on {io!r}") - WorkerGateway(io=io, id=id, _startcount=2).serve() diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py deleted file mode 100644 index 5707dff4..00000000 --- a/src/execnet/gateway_bootstrap.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Code to initialize the remote side of a gateway once the IO is created.""" - -from __future__ import annotations - -import inspect -import os - -import execnet - -from . import gateway_base -from .gateway_base import IO -from .xspec import XSpec - -importdir = os.path.dirname(os.path.dirname(execnet.__file__)) - - -class HostNotFound(Exception): - pass - - -def bootstrap_import(io: IO, spec: XSpec) -> None: - # Only insert the importdir into the path if we must. This prevents - # bugs where backports expect to be shadowed by the standard library on - # newer versions of python but would instead shadow the standard library. - sendexec( - io, - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet.gateway_base import serve, init_popen_io, get_execmodel", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "execmodel = get_execmodel(%r)" % spec.execmodel, - "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1", repr(s) - - -def bootstrap_exec(io: IO, spec: XSpec) -> None: - try: - sendexec( - io, - inspect.getsource(gateway_base), - "execmodel = get_execmodel(%r)" % spec.execmodel, - "io = init_popen_io(execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1" - except EOFError: - ret = io.wait() - if ret == 255 and hasattr(io, "remoteaddress"): - raise HostNotFound(io.remoteaddress) from None - - -def sendexec(io: IO, *sources: str) -> None: - source = "\n".join(sources) - io.write((repr(source) + "\n").encode("utf-8")) - - -def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: - if spec.popen: - if spec.via or spec.python: - bootstrap_exec(io, spec) - else: - bootstrap_import(io, spec) - elif spec.ssh or spec.vagrant_ssh: - bootstrap_exec(io, spec) - else: - raise ValueError("unknown gateway type, can't bootstrap") - gw = execnet.Gateway(io, spec) - return gw diff --git a/src/execnet/gateway_io.py b/src/execnet/gateway_io.py deleted file mode 100644 index 21285ab4..00000000 --- a/src/execnet/gateway_io.py +++ /dev/null @@ -1,255 +0,0 @@ -"""execnet IO initialization code. - -Creates IO instances used for gateway IO. -""" - -from __future__ import annotations - -import shlex -import sys -from typing import TYPE_CHECKING -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - from execnet.xspec import XSpec - -try: - from execnet.gateway_base import Message - from execnet.gateway_base import Popen2IO -except ImportError: - from __main__ import Message # type: ignore[no-redef] - from __main__ import Popen2IO # type: ignore[no-redef] - -from functools import partial - - -class Popen2IOMaster(Popen2IO): - # Set externally, for some specs only. - remoteaddress: str - - def __init__(self, args, execmodel: ExecModel) -> None: - PIPE = execmodel.subprocess.PIPE - self.popen = p = execmodel.subprocess.Popen(args, stdout=PIPE, stdin=PIPE) - super().__init__(p.stdin, p.stdout, execmodel=execmodel) - - def wait(self) -> int | None: - try: - return self.popen.wait() # type: ignore[no-any-return] - except OSError: - return None - - def kill(self) -> None: - try: - self.popen.kill() - except OSError as e: - sys.stderr.write("ERROR killing: %s\n" % e) - sys.stderr.flush() - - -popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" - - -def shell_split_path(path: str) -> list[str]: - """ - Use shell lexer to split the given path into a list of components, - taking care to handle Windows' '\' correctly. - """ - if sys.platform.startswith("win"): - # replace \\ by / otherwise shlex will strip them out - path = path.replace("\\", "/") - return shlex.split(path) - - -def popen_args(spec: XSpec) -> list[str]: - args = shell_split_path(spec.python) if spec.python else [sys.executable] - args.append("-u") - if spec.dont_write_bytecode: - args.append("-B") - args.extend(["-c", popen_bootstrapline]) - return args - - -def ssh_args(spec: XSpec) -> list[str]: - # NOTE: If changing this, you need to sync those changes to vagrant_args - # as well, or, take some time to further refactor the commonalities of - # ssh_args and vagrant_args. - remotepython = spec.python or "python" - args = ["ssh", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - - assert spec.ssh is not None - args.extend(spec.ssh.split()) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.append(remotecmd) - return args - - -def vagrant_ssh_args(spec: XSpec) -> list[str]: - # This is the vagrant-wrapped version of SSH. Unfortunately the - # command lines are incompatible to just channel through ssh_args - # due to ordering/templating issues. - # NOTE: This should be kept in sync with the ssh_args behaviour. - # spec.vagrant is identical to spec.ssh in that they both carry - # the remote host "address". - assert spec.vagrant_ssh is not None - remotepython = spec.python or "python" - args = ["vagrant", "ssh", spec.vagrant_ssh, "--", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.extend([remotecmd]) - return args - - -def create_io(spec: XSpec, execmodel: ExecModel) -> Popen2IOMaster: - if spec.popen: - args = popen_args(spec) - return Popen2IOMaster(args, execmodel) - if spec.ssh: - args = ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.ssh - return io - if spec.vagrant_ssh: - args = vagrant_ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.vagrant_ssh - return io - assert False - - -# -# Proxy Gateway handling code -# -# master: proxy initiator -# forwarder: forwards between master and sub -# sub: sub process that is proxied to the initiator - -RIO_KILL = 1 -RIO_WAIT = 2 -RIO_REMOTEADDRESS = 3 -RIO_CLOSE_WRITE = 4 - - -class ProxyIO: - """A Proxy IO object allows to instantiate a Gateway - through another "via" gateway. - - A master:ProxyIO object provides an IO object effectively connected to the - sub via the forwarder. To achieve this, master:ProxyIO interacts with - forwarder:serve_proxy_io() which itself instantiates and interacts with the - sub. - """ - - def __init__(self, proxy_channel: Channel, execmodel: ExecModel) -> None: - # after exchanging the control channel we use proxy_channel - # for messaging IO - self.controlchan = proxy_channel.gateway.newchannel() - proxy_channel.send(self.controlchan) - self.iochan = proxy_channel - self.iochan_file = self.iochan.makefile("r") - self.execmodel = execmodel - - def read(self, nbytes: int) -> bytes: - # TODO(typing): The IO protocol requires bytes here but ChannelFileRead - # returns str. - return self.iochan_file.read(nbytes) # type: ignore[return-value] - - def write(self, data: bytes) -> None: - self.iochan.send(data) - - def _controll(self, event: int) -> object: - self.controlchan.send(event) - return self.controlchan.receive() - - def close_write(self) -> None: - self._controll(RIO_CLOSE_WRITE) - - def close_read(self) -> None: - raise NotImplementedError() - - def kill(self) -> None: - self._controll(RIO_KILL) - - def wait(self) -> int | None: - response = self._controll(RIO_WAIT) - assert response is None or isinstance(response, int) - return response - - @property - def remoteaddress(self) -> str: - response = self._controll(RIO_REMOTEADDRESS) - assert isinstance(response, str) - return response - - def __repr__(self) -> str: - return f"" - - -class PseudoSpec: - def __init__(self, vars) -> None: - self.__dict__.update(vars) - - def __getattr__(self, name: str) -> None: - return None - - -def serve_proxy_io(proxy_channelX: Channel) -> None: - execmodel = proxy_channelX.gateway.execmodel - log = partial( - proxy_channelX.gateway._trace, "serve_proxy_io:%s" % proxy_channelX.id - ) - spec = cast("XSpec", PseudoSpec(proxy_channelX.receive())) - # create sub IO object which we will proxy back to our proxy initiator - sub_io = create_io(spec, execmodel) - control_chan = cast("Channel", proxy_channelX.receive()) - log("got control chan", control_chan) - - # read data from master, forward it to the sub - # XXX writing might block, thus blocking the receiver thread - def forward_to_sub(data: bytes) -> None: - log("forward data to sub, size %s" % len(data)) - sub_io.write(data) - - proxy_channelX.setcallback(forward_to_sub) - - def control(data: int) -> None: - if data == RIO_WAIT: - control_chan.send(sub_io.wait()) - elif data == RIO_KILL: - sub_io.kill() - control_chan.send(None) - elif data == RIO_REMOTEADDRESS: - control_chan.send(sub_io.remoteaddress) - elif data == RIO_CLOSE_WRITE: - sub_io.close_write() - control_chan.send(None) - - control_chan.setcallback(control) - - # write data to the master coming from the sub - forward_to_master_file = proxy_channelX.makefile("w") - - # read bootstrap byte from sub, send it on to master - log("reading bootstrap byte from sub", spec.id) - initial = sub_io.read(1) - assert initial == b"1", initial - log("forwarding bootstrap byte from sub", spec.id) - forward_to_master_file.write(initial) - - # enter message forwarding loop - while True: - try: - message = Message.from_io(sub_io) - except EOFError: - log("EOF from sub, terminating proxying loop", spec.id) - break - message.to_io(forward_to_master_file) - # proxy_channelX will be closed from remote_exec's finalization code - - -if __name__ == "__channelexec__": - serve_proxy_io(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29b8bd6a..c4892995 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -20,8 +20,6 @@ from typing import TypeAlias from typing import overload -from . import gateway_bootstrap -from . import gateway_io from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import WorkerPool @@ -154,26 +152,16 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if _trio_host.should_use_trio_popen(spec): - gw = _trio_host.makegateway_popen_trio(self, spec) - elif _trio_host.should_use_trio_ssh(spec): - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif _trio_host.should_use_trio_vagrant(spec): - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif _trio_host.should_use_trio_socket(spec): + if spec.socket: gw = _trio_host.makegateway_socket_trio(self, spec) - elif _trio_host.should_use_trio_via(spec): - gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: - assert not spec.socket - master = self[spec.via] - proxy_channel = master.remote_exec(gateway_io) - proxy_channel.send(vars(spec)) - proxy_io_master = gateway_io.ProxyIO(proxy_channel, self.execmodel) - gw = gateway_bootstrap.bootstrap(proxy_io_master, spec) - elif spec.popen or spec.ssh or spec.vagrant_ssh: - io = gateway_io.create_io(spec, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(io, spec) + gw = _trio_host.makegateway_via_trio(self, spec) + elif spec.ssh: + gw = _trio_host.makegateway_ssh_trio(self, spec) + elif spec.vagrant_ssh: + gw = _trio_host.makegateway_vagrant_trio(self, spec) + elif spec.popen: + gw = _trio_host.makegateway_popen_trio(self, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec diff --git a/testing/test_basics.py b/testing/test_basics.py index 1756ec34..d4ffd5d8 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -17,11 +17,9 @@ import execnet from execnet import gateway from execnet import gateway_base -from execnet import gateway_io from execnet.gateway_base import ChannelFactory from execnet.gateway_base import ExecModel from execnet.gateway_base import Message -from execnet.gateway_base import Popen2IO skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -68,81 +66,29 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def test_subprocess_interaction(anypython: str) -> None: - line = gateway_io.popen_bootstrapline - compile(line, "xyz", "exec") - args = [str(anypython), "-c", line] - popen = subprocess.Popen( - args, - bufsize=0, - universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - assert popen.stdin is not None - assert popen.stdout is not None - - def send(line: str) -> None: - assert popen.stdin is not None - popen.stdin.write(line) - popen.stdin.flush() - - def receive() -> str: - assert popen.stdout is not None - return popen.stdout.readline() - - try: - source = inspect.getsource(read_write_loop) + "read_write_loop()" - send(repr(source) + "\n") - s = receive() - assert s == "ok\n" - send("hello\n") - s = receive() - assert s == "received: hello\n" - send("world\n") - s = receive() - assert s == "received: world\n" - send("\n") # terminate loop - finally: - popen.stdin.close() - popen.stdout.close() - popen.wait() +IO_MESSAGE_EXTRA_SOURCE = """ +from io import BytesIO +class BufIO: + def __init__(self): + self.buf = BytesIO() -def read_write_loop() -> None: - sys.stdout.write("ok\n") - sys.stdout.flush() - while 1: - try: - line = sys.stdin.readline() - if not line.strip(): - break - sys.stdout.write("received: %s" % line) - sys.stdout.flush() - except (OSError, EOFError): - break + def write(self, data): + self.buf.write(data) + def read(self, numbytes): + data = self.buf.read(numbytes) + if len(data) < numbytes: + raise EOFError("expected %d bytes" % numbytes) + return data -IO_MESSAGE_EXTRA_SOURCE = """ -import sys -backend = sys.argv[1] -from io import BytesIO -import tempfile -temp_out = BytesIO() -temp_in = BytesIO() -io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) for i, handler in enumerate(Message._types): print ("checking", i, handler) for data in "hello", "hello".encode('ascii'): + io = BufIO() msg1 = Message(i, i, dumps(data)) msg1.to_io(io) - x = io.outfile.getvalue() - io.outfile.truncate(0) - io.outfile.seek(0) - io.infile.seek(0) - io.infile.write(x) - io.infile.seek(0) + io.buf.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) @@ -177,44 +123,12 @@ def checker(anypython: str, tmp_path: Path) -> Checker: return Checker(python=anypython, path=tmp_path) -def test_io_message(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend - ) +def test_io_message(checker: Checker) -> None: + out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout -def test_popen_io(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) - + f""" -io = init_popen_io(get_execmodel({execmodel.backend!r})) -io.write(b"hello") -s = io.read(1) -assert s == b"x" -""", - input="x", - ) - print(out.stderr) - assert "hello" in out.stdout - - -def test_popen_io_readloop(execmodel: ExecModel) -> None: - sio = BytesIO(b"test") - io = Popen2IO(sio, sio, execmodel) - real_read = io._read - - def newread(numbytes: int) -> bytes: - if numbytes > 1: - numbytes = numbytes - 1 - return real_read(numbytes) # type: ignore[no-any-return] - - io._read = newread - result = io.read(3) - assert result == b"tes" - - def test_rinfo_source(checker: Checker) -> None: out = checker.run_check( f""" @@ -252,19 +166,20 @@ class Arg(Exception): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull( - execmodel: ExecModel, capfd: pytest.CaptureFixture[str] -) -> None: - # Backup and restore stdin state, and rely on capfd to handle - # this for stdout and stderr. +def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: + # _prepare_protocol_fds dups the stdio fds for the Message protocol and + # points fd 0/1 at devnull; writes/reads on the original fds must go + # nowhere. Back up and restore the real fds around the call. + from execnet import _trio_worker + orig_stdin = sys.stdin - orig_stdin_fd = os.dup(0) + orig_stdout = sys.stdout + orig_fd0 = os.dup(0) + orig_fd1 = os.dup(1) try: - # The returned Popen2IO instance can be garbage collected - # prematurely since we don't hold a reference here, but we - # tolerate this because it is intended to leave behind a - # sane state afterwards. - gateway_base.init_popen_io(execmodel) + read_fd, write_fd = _trio_worker._prepare_protocol_fds() + os.close(read_fd) + os.close(write_fd) os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() @@ -272,8 +187,11 @@ def test_stdouterrin_setnull( assert not err finally: sys.stdin = orig_stdin - os.dup2(orig_stdin_fd, 0) - os.close(orig_stdin_fd) + sys.stdout = orig_stdout + os.dup2(orig_fd0, 0) + os.dup2(orig_fd1, 1) + os.close(orig_fd0) + os.close(orig_fd1) class PseudoChannel: diff --git a/testing/test_gateway.py b/testing/test_gateway.py index e123bbe2..c3c87e4a 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -16,7 +16,6 @@ import execnet from execnet import gateway_base -from execnet import gateway_io from execnet.gateway import Gateway TESTTIMEOUT = 10.0 # seconds @@ -381,23 +380,6 @@ def test_socket_gw_host_not_found(makegateway: Callable[[str], Gateway]) -> None class TestSshPopenGateway: gwtype = "ssh" - def test_sshconfig_config_parsing( - self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] - ) -> None: - # white-box test of the legacy Popen2IOMaster arg construction - monkeypatch.setenv("EXECNET_TRIO_HOST", "0") - l = [] - monkeypatch.setattr( - gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) - ) - with pytest.raises(AttributeError): - makegateway("ssh=xyz//ssh_config=qwe") - - assert len(l) == 1 - popen_args = l[0] - i = popen_args.index("-F") - assert popen_args[i + 1] == "qwe" - def test_ssh_trio_args_include_config( self, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -549,9 +531,11 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - expected_args = [*expected_args, "-u", "-c", gateway_io.popen_bootstrapline] - args = gateway_io.popen_args(execnet.XSpec(spec)) - assert args == expected_args + from execnet import _trio_host + + args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + assert args[: len(expected_args)] == expected_args + assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] def test_assert_main_thread_only( diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index c21135d1..086ce877 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -133,12 +133,8 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) -def test_ssh_roundtrip( - ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str -) -> None: - # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. - monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) +def test_ssh_roundtrip(ssh_config: str) -> None: + # The worker is provisioned over ssh with uv. group = execnet.Group() try: gw = group.makegateway( diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 5240d72d..f9681f55 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -11,10 +11,8 @@ import execnet from execnet import XSpec +from execnet import _provision from execnet.gateway import Gateway -from execnet.gateway_io import popen_args -from execnet.gateway_io import ssh_args -from execnet.gateway_io import vagrant_ssh_args skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -69,22 +67,20 @@ def test_execmodel(self) -> None: def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") - spec.ssh_config = "/home/user/ssh_config" - assert ssh_args(spec)[:6] == ["ssh", "-C", "-F", spec.ssh_config, "-p", "22100"] + args = _provision.ssh_argv("-p 22100 user@host", "/home/user/ssh_config", "cmd") + assert args[:6] == ["ssh", "-C", "-F", "/home/user/ssh_config", "-p", "22100"] + assert spec.ssh is not None def test_vagrant_options(self) -> None: - spec = XSpec("vagrant_ssh=default//python=python3") - assert vagrant_ssh_args(spec)[:-1] == ["vagrant", "ssh", "default", "--", "-C"] + args = _provision.vagrant_ssh_argv("default", None, "cmd") + assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - spec = XSpec("popen//python=sudo python3") - assert popen_args(spec) == [ - "sudo", - "python3", - "-u", - "-c", - "import sys;exec(eval(sys.stdin.readline()))", - ] + from execnet import _trio_host + + spec = XSpec("popen//python=sudo python3//id=gw0") + args = _trio_host.popen_module_args(spec) + assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") From 0a24039959a02ea96679fe863bc9b9437d0d7abf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 11:11:22 +0200 Subject: [PATCH 18/66] refactor: unify transports on StapledStream with sans-IO frame decoding Phase B.1+B.2 of the Trio port: - add gateway_base.FrameDecoder, an incremental sans-IO decoder for the 9-byte-header Message framing (feed arbitrary chunks, complete messages come out; close() flags mid-frame EOF) - replace the hand-rolled AsyncByteIO wrappers (ProcessStreamsIO, FdStreamsIO, SocketStreamIO) with trio.StapledStream / plain trio.SocketStream behind a neutral ByteStream protocol (send_all/receive_some/send_eof/aclose) that a future anyio backend can satisfy structurally; the via tunnel keeps its interim bridge as ChannelByteStream - ProtocolSession's reader becomes the uniform receive_some+feed loop; exact reads survive only as the one-byte handshake ack - route worker exec requests through a single FIFO pump task: batched frame decoding removed the per-message awaits that had accidentally serialized main_thread_only exec admission, so admission now happens explicitly in message-arrival order instead of racing tasks - give pre-commit's mypy the trio dependency so trio types are real; drop now-redundant casts Co-Authored-By: Claude Fable 5 --- .pre-commit-config.yaml | 1 + src/execnet/_trio_host.py | 211 +++++++++++++----------------------- src/execnet/_trio_worker.py | 45 ++++---- src/execnet/gateway_base.py | 36 ++++++ testing/test_basics.py | 84 ++++++++++++++ 5 files changed, 223 insertions(+), 154 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37ccd44d..f7a9f86c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,4 +31,5 @@ repos: - id: mypy additional_dependencies: - pytest + - trio - types-pywin32 diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 59f6b47f..1a8dd197 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -21,11 +21,11 @@ from typing import Any from typing import Protocol from typing import TypeVar -from typing import cast import trio from .gateway_base import ExecModel +from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import dumps_internal @@ -41,144 +41,95 @@ _CLOSE_WRITE = object() -class AsyncByteIO(Protocol): - async def read_exact(self, n: int) -> bytes: ... +RECEIVE_CHUNK = 65536 - async def write_all(self, data: bytes) -> None: ... - async def aclose_read(self) -> None: ... +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. - async def aclose_write(self) -> None: ... - - -async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: - buf = bytearray() - while len(buf) < n: - chunk = await stream.receive_some(n - len(buf)) - if not chunk: - raise EOFError("expected %d bytes, got %d" % (n, len(buf))) - buf += chunk - return bytes(buf) - - -async def read_message(io: AsyncByteIO) -> Message: - header = await io.read_exact(9) - msgtype, channel, payload = Message.from_header(header) - data = await io.read_exact(payload) if payload else b"" - return Message.from_parts(msgtype, channel, data) - - -class ProcessStreamsIO: - """Async IO over a Trio Process stdin/stdout pair.""" - - def __init__(self, process: trio.Process) -> None: - assert process.stdin is not None - assert process.stdout is not None - self.process = process - self._stdin = process.stdin - self._stdout = process.stdout - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stdout, n) - - async def write_all(self, data: bytes) -> None: - await self._stdin.send_all(data) - - async def aclose_read(self) -> None: - await self._stdout.aclose() - - async def aclose_write(self) -> None: - await self._stdin.aclose() - - -class FdStreamsIO: - """Async IO over OS file descriptors (worker stdio pipes).""" - - def __init__(self, read_fd: int, write_fd: int) -> None: - self._read = trio.lowlevel.FdStream(read_fd) - self._write = trio.lowlevel.FdStream(write_fd) - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._read, n) - - async def write_all(self, data: bytes) -> None: - await self._write.send_all(data) + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ - async def aclose_read(self) -> None: - await self._read.aclose() + async def send_all(self, data: bytes) -> None: ... - async def aclose_write(self) -> None: - await self._write.aclose() + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + async def send_eof(self) -> None: ... -class SocketStreamIO: - """Async IO over a single bidirectional Trio stream (a socket). + async def aclose(self) -> None: ... - ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` - bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is - idempotent, so close-read and close-write both just close the socket. - """ - def __init__(self, stream: trio.abc.Stream) -> None: - self._stream = stream +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stream, n) - async def write_all(self, data: bytes) -> None: - await self._stream.send_all(data) +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) - async def aclose_read(self) -> None: - await self._stream.aclose() - async def aclose_write(self) -> None: - await self._stream.aclose() +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") _CHANNEL_EOF = object() -class ChannelByteIO: - """Async byte IO tunnelled over a sync execnet ``Channel``. +class ChannelByteStream: + """``ByteStream`` tunnelled over a sync execnet ``Channel``. INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol as raw bytes over a channel to the master, which double-frames it (sub frame -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it should be replaced by a proper relayed transport rather - than tunnelling bytes through the channel layer. + async protocol; it dissolves once low-level raw channels exist (Phase B.4). The channel callback (on the host loop) feeds an unbounded memory channel - that ``read_exact`` drains; writes ``channel.send`` raw frames. + that ``receive_some`` drains; writes ``channel.send`` raw bytes. """ def __init__(self, channel: Any) -> None: self._channel = channel self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() + self._eof = False channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) - async def read_exact(self, n: int) -> bytes: - while len(self._buf) < n: + async def send_all(self, data: bytes) -> None: + self._channel.send(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: data = await self._recv.receive() if data is _CHANNEL_EOF: - raise EOFError("channel closed") - assert isinstance(data, bytes) - self._buf += data - out = bytes(self._buf[:n]) - del self._buf[:n] + self._eof = True + else: + assert isinstance(data, bytes) + self._buf += data + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] return out - async def write_all(self, data: bytes) -> None: - self._channel.send(data) - - async def aclose_read(self) -> None: - return + async def send_eof(self) -> None: + self._channel.close() - async def aclose_write(self) -> None: + async def aclose(self) -> None: self._channel.close() -async def adopt_socket(socket_fd: int) -> SocketStreamIO: +async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. Runs on the Trio host loop. The coordinator waits for ``b"1"`` before @@ -188,9 +139,8 @@ async def adopt_socket(socket_fd: int) -> SocketStreamIO: sock = _socket.socket(fileno=socket_fd) stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) - io = SocketStreamIO(stream) - await io.write_all(b"1") - return io + await stream.send_all(b"1") + return stream class SyncIOHandle: @@ -235,7 +185,7 @@ class ProtocolSession: def __init__( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, host: TrioHost, @@ -361,13 +311,17 @@ def log(*msg: object) -> None: gateway._trace("[trio-receiver]", *msg) log("RECEIVER: starting") + decoder = FrameDecoder() try: while True: - msg = await read_message(self.io) - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - del msg + data = await self.io.receive_some(RECEIVE_CHUNK) + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("clean EOF") + for msg in decoder.feed(data): + log("received", msg) + with gateway._receivelock: + msg.received(gateway) except GatewayReceivedTerminate: log("GATEWAY_TERMINATE") except EOFError as exc: @@ -401,15 +355,15 @@ async def _writer_handle_item(self, item: object) -> bool: """Handle one outbound queue item. Return False when writer should stop.""" if item is _CLOSE_WRITE: try: - await self.io.aclose_write() + await self.io.send_eof() except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + self.gateway._trace("send_eof failed", exc) return False assert isinstance(item, tuple) blob, done, errors = item assert isinstance(blob, bytes) try: - await self.io.write_all(blob) + await self.io.send_all(blob) except Exception as exc: self.gateway._trace("write failed", exc) errors.append(exc) @@ -445,11 +399,7 @@ async def _finish(self) -> None: gateway._terminate_execution, abandon_on_cancel=True ) try: - await self.io.aclose_read() - except Exception: - pass - try: - await self.io.aclose_write() + await self.io.aclose() except Exception: pass @@ -496,14 +446,12 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) + return trio.from_thread.run(async_fn, *args, trio_token=self._token) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast( - "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) - ) + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -516,7 +464,7 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: async def start_session( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, ) -> ProtocolSession: @@ -621,12 +569,10 @@ def _open_trio_gateway( async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: - async_io = ProcessStreamsIO(process) + async_io = staple_process_stream(process) if preamble: - await async_io.write_all(preamble) - ack = await async_io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") + await async_io.send_all(preamble) + await read_handshake_ack(async_io, "bootstrap") except EOFError: with trio.move_on_after(5): code = await process.wait() @@ -732,18 +678,15 @@ async def _create_and_attach() -> Gateway: stream = await trio.open_tcp_stream(*address) except OSError as exc: raise HostNotFound(remoteaddress) from exc - io = SocketStreamIO(stream) try: - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad socket handshake: {ack!r}") + await read_handshake_ack(stream, "socket") except BaseException: with trio.move_on_after(5): await stream.aclose() raise gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) + session = await host.start_session(gw, stream) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw @@ -910,10 +853,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteIO(channel) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad via handshake: {ack!r}") + io = ChannelByteStream(channel) + await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 9e1f1be7..5fc9bceb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import os import queue import sys @@ -51,9 +52,14 @@ def __init__( tuple[Channel, ExecItem, threading.Event] | None ] = queue.SimpleQueue() self._primary_wake = threading.Event() - # Serialize main_thread_only admission (wait+clear) so two tasks cannot - # both observe the idle Event before either clears it. - self._admit_lock = trio.Lock() + # Exec requests flow through a single pump task so admission happens + # strictly in message-arrival order (trio task scheduling order is + # deliberately unordered, so per-request tasks would race for the + # main_thread_only slot). + self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] + self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] + self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) + self._pump_started = False def active_count(self) -> int: with self._lock: @@ -82,24 +88,25 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") return # Already on the Trio host thread (Message handler). - self.host.start_soon(self._run_exec, channel, item) - - async def _run_exec(self, channel: Channel, item: ExecItem) -> None: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - - def _wait_slot() -> bool: - return complete.wait(timeout=1) - - async with self._admit_lock: - if not await trio.to_thread.run_sync( - _wait_slot, abandon_on_cancel=True - ): + if not self._pump_started: + self._pump_started = True + self.host.start_soon(self._pump) + self._pending_send.send_nowait((channel, item)) + + async def _pump(self) -> None: + """Admit queued exec requests in FIFO order, then run each as a task.""" + async for channel, item in self._pending_recv: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return + continue complete.clear() + self.host.start_soon(self._run_exec, channel, item) + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: if self.main_thread_only: @@ -252,7 +259,7 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: from . import _trio_host - return _trio_host.FdStreamsIO(read_fd, write_fd) + return _trio_host.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0260b416..d36c7542 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -529,6 +529,42 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) + + class GatewayReceivedTerminate(Exception): """Receiver got a gateway termination message.""" diff --git a/testing/test_basics.py b/testing/test_basics.py index d4ffd5d8..473cd741 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -236,6 +236,90 @@ def test_wire_protocol(self) -> None: assert isinstance(repr(msg), str) +class TestFrameDecoder: + def _messages(self) -> list[Message]: + return [ + Message(Message.CHANNEL_DATA, 1, b"x" * 20), + Message(Message.STATUS, 42, b""), + Message(Message.CHANNEL_DATA, 7, b"y"), + ] + + def test_single_feed_yields_all(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got = list(decoder.feed(blob)) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) + def test_adversarial_chunk_splits(self, chunksize: int) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got: list[Message] = [] + for start in range(0, len(blob), chunksize): + got.extend(decoder.feed(blob[start : start + chunksize])) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + def test_close_mid_frame_raises(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() + assert list(decoder.feed(blob[:-2])) == [] + with pytest.raises(EOFError, match="mid-frame"): + decoder.close() + + def test_feed_buffers_even_when_not_iterated(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() + decoder.feed(blob[:4]) # result deliberately not iterated + (msg,) = decoder.feed(blob[4:]) + assert (msg.msgcode, msg.channelid, msg.data) == ( + Message.CHANNEL_DATA, + 5, + b"data", + ) + + def test_memory_stream_roundtrip(self) -> None: + """Protocol-level: frames sent over a trio memory stream pair arrive + intact through the receive_some + FrameDecoder loop.""" + import trio + import trio.testing + + messages = self._messages() + + async def main() -> list[Message]: + ours, theirs = trio.testing.memory_stream_pair() + received: list[Message] = [] + + async def sender() -> None: + for m in messages: + await theirs.send_all(m.pack()) + await theirs.send_eof() + + async def receiver() -> None: + decoder = gateway_base.FrameDecoder() + while True: + data = await ours.receive_some(4096) + if not data: + decoder.close() + break + received.extend(decoder.feed(data)) + + async with trio.open_nursery() as nursery: + nursery.start_soon(sender) + nursery.start_soon(receiver) + return received + + received = trio.run(main) + assert [(m.msgcode, m.channelid, m.data) for m in received] == [ + (m.msgcode, m.channelid, m.data) for m in messages + ] + + class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: From f271da5e379dd23392b39ddcdcc0792bc6fc43c1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 12:19:03 +0200 Subject: [PATCH 19/66] refactor: extract the LoopPortal cross-thread primitive Phase B.3 of the Trio port: one portal module replaces the three ad-hoc cross-thread bridges. - new execnet/portal.py: LoopPortal (trio token holder with run/run_sync/post/is_loop_thread; post = run_sync_soon, strict FIFO) and SyncReceiver (loop-to-thread queue whose get() stays KeyboardInterrupt-interruptible on the main thread) - TrioHost owns a LoopPortal; call/call_sync/is_host_thread delegate - ProtocolSession's outbound queue becomes an unbounded trio memory channel; every send is posted through the portal so loop callbacks and foreign threads share one FIFO, and the writer task is a plain async-for -- the recreated-Event wake dance is gone. Blocking and close semantics are unchanged: non-loop threads still wait for the OS write (120s timeout), closed sends still raise OSError, and a finished loop maps trio.RunFinishedError to the same OSError. - TrioWorkerExec's main-thread exec handoff uses SyncReceiver Because each end only needs the other loop's token, the same primitive will serve two-loop setups (facade host loop + user loop) in B.5/B.6. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 85 ++++++++++++++------------------ src/execnet/_trio_worker.py | 23 +++------ src/execnet/portal.py | 96 +++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 src/execnet/portal.py diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1a8dd197..cd20a0f2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import queue import subprocess import sys import threading @@ -31,6 +30,7 @@ from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace +from .portal import LoopPortal if TYPE_CHECKING: from .gateway import Gateway @@ -194,14 +194,24 @@ def __init__( self.io = io self.process = process self.host = host - self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() - self._wake: trio.Event | None = None + self._outbound_send: trio.MemorySendChannel[object] + self._outbound_recv: trio.MemoryReceiveChannel[object] + self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() self._send_closed = False self._lock = threading.Lock() + def _post_outbound(self, item: object) -> None: + """Schedule ``item`` onto the outbound channel. + + Goes through the portal even from the host thread so every send — + loop callbacks and foreign threads alike — lands in one global FIFO + order. Raises ``trio.RunFinishedError`` after loop shutdown. + """ + self.host.portal.post(self._outbound_send.send_nowait, item) + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -211,14 +221,16 @@ def enqueue_message(self, message: Message) -> None: The Trio host thread (receiver callbacks) must not wait — that would deadlock the writer task on the same event loop. """ - wait = not self.host.is_host_thread() + wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] with self._lock: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") - self._outbound.put((message.pack(), done, errors)) - self._wake_writer() + try: + self._post_outbound((message.pack(), done, errors)) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None if done is None: return if not done.wait(timeout=120.0): @@ -226,25 +238,13 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] - def _wake_writer(self) -> None: - wake = self._wake - if wake is None: - return - if self.host.is_host_thread(): - wake.set() - else: - try: - self.host.call_sync(wake.set) - except Exception: - pass - def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -332,22 +332,7 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: - self._wake = trio.Event() - while True: - while True: - try: - item = self._outbound.get_nowait() - except queue.Empty: - break - if not await self._writer_handle_item(item): - return - # Reset wake before re-check to avoid losing a notification. - self._wake = trio.Event() - try: - item = self._outbound.get_nowait() - except queue.Empty: - await self._wake.wait() - continue + async for item in self._outbound_recv: if not await self._writer_handle_item(item): return @@ -385,8 +370,8 @@ async def _finish(self) -> None: gateway = self.gateway with self._lock: self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock the worker's join() before heavy exec-pool shutdown @@ -410,7 +395,7 @@ class TrioHost: def __init__(self, name: str = "execnet-trio-host") -> None: self._name = name self._thread: threading.Thread | None = None - self._token: trio.lowlevel.TrioToken | None = None + self._portal: LoopPortal | None = None self._nursery: trio.Nursery | None = None self._ready = threading.Event() self._shutdown: trio.Event | None = None @@ -425,14 +410,20 @@ def start(self) -> None: raise RuntimeError("TrioHost failed to start") self._started = True + @property + def portal(self) -> LoopPortal: + if self._portal is None: + raise RuntimeError("TrioHost is not running") + return self._portal + def is_host_thread(self) -> bool: - return self._thread is not None and threading.current_thread() is self._thread + return self._portal is not None and self._portal.is_loop_thread() def _run(self) -> None: trio.run(self._main) async def _main(self) -> None: - self._token = trio.lowlevel.current_trio_token() + self._portal = LoopPortal() self._shutdown = trio.Event() try: async with trio.open_nursery() as nursery: @@ -444,14 +435,10 @@ async def _main(self) -> None: self._nursery = None def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return self.portal.run(async_fn, *args) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return self.portal.run_sync(sync_fn, *args) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -475,7 +462,7 @@ async def start_session( return session def stop(self, timeout: float | None = 5.0) -> None: - if not self._started or self._token is None or self._shutdown is None: + if not self._started or self._portal is None or self._shutdown is None: return def _set() -> None: @@ -483,7 +470,7 @@ def _set() -> None: self._shutdown.set() try: - trio.from_thread.run_sync(_set, trio_token=self._token) + self._portal.run_sync(_set) except Exception: pass if self._thread is not None: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc9bceb..bb71b027 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -4,7 +4,6 @@ import functools import os -import queue import sys import threading from typing import TYPE_CHECKING @@ -17,6 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace +from .portal import SyncReceiver if TYPE_CHECKING: from . import _trio_host @@ -48,10 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary_q: queue.SimpleQueue[ + self._primary: SyncReceiver[ tuple[Channel, ExecItem, threading.Event] | None - ] = queue.SimpleQueue() - self._primary_wake = threading.Event() + ] = SyncReceiver() # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the @@ -111,8 +110,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: try: if self.main_thread_only: done = threading.Event() - self._primary_q.put((channel, item, done)) - self._primary_wake.set() + self._primary.put((channel, item, done)) await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) else: await trio.to_thread.run_sync( @@ -126,15 +124,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: def integrate_as_primary_thread(self) -> None: """Block the main thread running main_thread_only exec tasks.""" while True: - self._primary_wake.wait() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - self._primary_wake.clear() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - continue + task = self._primary.get() if task is None: break channel, item, done = task @@ -146,8 +136,7 @@ def integrate_as_primary_thread(self) -> None: def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary_q.put(None) - self._primary_wake.set() + self._primary.put(None) def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) diff --git a/src/execnet/portal.py b/src/execnet/portal.py new file mode 100644 index 00000000..9804ca45 --- /dev/null +++ b/src/execnet/portal.py @@ -0,0 +1,96 @@ +"""Cross-thread / cross-loop communication primitives. + +A :class:`LoopPortal` is a handle to a running trio loop that foreign +threads use to run functions on the loop or push work into it. Because +each direction only needs the *receiving* loop's token, two trio loops in +two threads can communicate by holding each other's portal (the sync +facade's host loop and a user loop; the worker loop and the process main +thread). + +:class:`SyncReceiver` covers the opposite direction: a plain thread +(typically the process main thread) receiving items produced on a loop, +in a way that stays interruptible by KeyboardInterrupt. +""" + +from __future__ import annotations + +import queue +import threading +from collections.abc import Awaitable +from collections.abc import Callable +from typing import Any +from typing import Generic +from typing import TypeVar + +import trio + +T = TypeVar("T") + + +class LoopPortal: + """Handle to a running trio loop, usable from foreign threads. + + Must be constructed on the loop's own thread (it captures the current + trio token). + """ + + def __init__(self) -> None: + self._token = trio.lowlevel.current_trio_token() + + def is_loop_thread(self) -> bool: + """Whether the calling thread is running this portal's loop.""" + try: + return trio.lowlevel.current_trio_token() is self._token + except RuntimeError: + return False + + def run(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Run ``await async_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def run_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + """Run ``sync_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + def post(self, sync_fn: Callable[..., object], *args: Any) -> None: + """Schedule ``sync_fn(*args)`` on the loop without waiting. + + Thread-safe and callable from the loop thread itself; all posts run + in strict FIFO order (``TrioToken.run_sync_soon``). Raises + ``trio.RunFinishedError`` once the loop has shut down. + """ + self._token.run_sync_soon(sync_fn, *args) + + +class SyncReceiver(Generic[T]): + """Receive items on a plain thread, KeyboardInterrupt-friendly. + + ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields + KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. + So producers put into an unbounded queue and set a wake event, and + :meth:`get` waits on the event and drains the queue. + """ + + def __init__(self) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wake = threading.Event() + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wake.set() + + def get(self) -> T: + """Block until an item is available (interruptible on main thread).""" + while True: + self._wake.wait() + try: + return self._items.get_nowait() + except queue.Empty: + # Re-check after clearing so a put between get_nowait and + # clear cannot be lost. + self._wake.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue From 0eb6cc531774148003de81ef1e84a6174d3eb804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 14:45:43 +0200 Subject: [PATCH 20/66] feat: add the trio-native AsyncGateway core with low-level RawChannels Phase B.4 starts the async-native inversion: a new _trio_gateway module hosts AsyncGateway, whose single serve task reads the framed Message protocol off a ByteStream (receive_some + sans-IO FrameDecoder) and dispatches inline -- no receiver thread, no receive lock -- plus a writer task draining an unbounded outbound queue. RawChannel is the low-level half of the two-level channel model: id-routed raw byte payloads with the sync Channel close semantics (CHANNEL_CLOSE both ways, CHANNEL_LAST_MESSAGE as write-EOF leaving the peer sendonly, CHANNEL_CLOSE_ERROR surfacing as RemoteError). Errors keep the execnet contract -- OSError on closed sends, EOFError/RemoteError on receive -- so no trio exception types leak into the API. The ByteStream protocol and RECEIVE_CHUNK move here from _trio_host so the async core sits at the bottom of the dependency stack. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 398 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 24 +-- testing/test_trio_gateway.py | 202 ++++++++++++++++++ 3 files changed, 602 insertions(+), 22 deletions(-) create mode 100644 src/execnet/_trio_gateway.py create mode 100644 testing/test_trio_gateway.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py new file mode 100644 index 00000000..67e103ed --- /dev/null +++ b/src/execnet/_trio_gateway.py @@ -0,0 +1,398 @@ +"""Trio-native gateway core: async dispatch loop and low-level raw channels. + +Async-first counterpart of the sync machinery in ``gateway_base``: an +:class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch +task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline +on that task, so there is no receiver thread and no receive lock. + +Two-level channel model: + +* :class:`RawChannel` (this module) -- id-routed raw byte payload streams + over the gateway: no serialization, no strconfig, no callbacks. + ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top + decides what the bytes mean. +* ``AsyncChannel`` -- the serialized object API layered on a RawChannel. + +The code deliberately sticks to idioms an anyio backend can mirror later: +a neutral ``ByteStream`` protocol, the sans-IO ``FrameDecoder``, and +unbounded memory channels. +""" + +from __future__ import annotations + +import math +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import Protocol + +import trio + +from .gateway_base import FrameDecoder +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import RemoteError +from .gateway_base import Unserializer +from .gateway_base import dumps_internal +from .gateway_base import loads_internal +from .gateway_base import trace + +RECEIVE_CHUNK = 65536 + + +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. + + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ + + async def send_all(self, data: bytes) -> None: ... + + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + + async def send_eof(self) -> None: ... + + async def aclose(self) -> None: ... + + +class RawChannel: + """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. + + Payload boundaries are preserved: every :meth:`send_bytes` arrives as + one :meth:`receive_bytes` result on the peer. No serialization and no + flow control beyond the gateway's outbound queue. + + Close semantics mirror the sync ``Channel`` state machine: + + * :meth:`aclose` closes both directions (``CHANNEL_CLOSE`` / + ``CHANNEL_CLOSE_ERROR`` to the peer). + * :meth:`send_eof` only ends our payload stream (``CHANNEL_LAST_MESSAGE``); + the peer drains, hits EOF, and may keep sending to us. + """ + + _strconfig: tuple[bool, bool] | None = None + + def __init__(self, gateway: AsyncGateway, id: int) -> None: + self.gateway = gateway + self.id = id + self._closed = False # no more sends (local aclose or remote close) + self._sent_eof = False + self._remote_closed = False + self._remote_error: RemoteError | None = None + self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + async def send_bytes(self, data: bytes) -> None: + """Send one payload; the peer receives it as a single item. + + OSError is raised when the channel or gateway is closed, matching + the sync ``Channel.send`` contract. + """ + if self._closed or self._sent_eof: + raise OSError(f"cannot send to {self!r}") + await self.gateway._send(Message.CHANNEL_DATA, self.id, data) + + async def receive_bytes(self) -> bytes: + """Receive the next payload. + + Raises EOFError once the peer closed or sent EOF and all payloads + are drained; a peer close-with-error raises that ``RemoteError``. + """ + try: + return await self._payloads.receive() + except (trio.EndOfChannel, trio.ClosedResourceError): + raise self._pending_error() from None + + async def send_eof(self) -> None: + """Signal that no more payloads follow (peer keeps its send side).""" + if self._closed or self._sent_eof: + raise OSError(f"cannot send EOF to {self!r}") + self._sent_eof = True + await self.gateway._send(Message.CHANNEL_LAST_MESSAGE, self.id) + + async def aclose(self, error: str | None = None) -> None: + """Close both directions; ``error`` reaches the peer as a RemoteError.""" + if self._closed: + await trio.lowlevel.checkpoint() + return + self._closed = True + self._payload_send.close() + self.gateway._forget_channel(self.id) + if not self._remote_closed: + # A peer-initiated close needs no reply; a dead gateway is + # already as closed as it gets. + with suppress(OSError): + if error is not None: + await self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error) + ) + else: + await self.gateway._send(Message.CHANNEL_CLOSE, self.id) + + def __aiter__(self) -> RawChannel: + return self + + async def __anext__(self) -> bytes: + try: + return await self.receive_bytes() + except EOFError: + raise StopAsyncIteration from None + + def _pending_error(self) -> BaseException: + return ( + self._remote_error + or self.gateway._error + or EOFError(f"raw channel {self.id} closed") + ) + + # dispatch-loop internals (inline on the gateway's serve task) + + def _feed(self, data: bytes) -> None: + try: + self._payload_send.send_nowait(data) + except (trio.BrokenResourceError, trio.ClosedResourceError): + pass # locally closed: drop, like the sync channel + + def _close_from_remote( + self, error: RemoteError | None, *, sendonly: bool + ) -> None: + if error is not None: + self._remote_error = error + self._remote_closed = True + if not sendonly: + self._closed = True + self.gateway._forget_channel(self.id) + self._payload_send.close() + + +class AsyncGateway: + """Async-native gateway: the framed Message protocol over a ByteStream. + + Serving (``serve_gateway`` or ``nursery.start(gateway._serve)``) runs a + reader task that dispatches messages inline and a writer task draining + an unbounded outbound queue -- sends never block on the peer. + + ``_startcount`` follows the sync convention: locally allocated channel + ids step by two, coordinators from 1 (odd) and workers from 2 (even), + so the two peers never collide. + """ + + _error: BaseException | None = None + + def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: + self._stream = stream + self.id = id + self._channels: dict[int, RawChannel] = {} + self._count = _startcount + self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) + self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._closed = False + self._serve_started = False + self._writer_done = trio.Event() + self._done = trio.Event() + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + @property + def closed(self) -> bool: + return self._closed + + async def wait_closed(self) -> None: + """Wait until serving has fully shut down.""" + await self._done.wait() + + def _trace(self, *msg: object) -> None: + trace(self.id, *msg) + + def open_raw_channel(self, id: int | None = None) -> RawChannel: + """Return the raw channel for ``id``, allocating a fresh id if None. + + An explicit id attaches to a channel the peer references (e.g. an id + received in a request payload); the same object is returned if the + dispatch loop already routed data to it. + """ + if self._closed: + raise OSError(f"connection already closed: {self!r}") + if id is None: + id = self._count + self._count += 2 + return self._channel_for(id) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + if not self._closed: + with suppress(OSError): + await self._send(Message.GATEWAY_TERMINATE) + await self.aclose() + + async def aclose(self) -> None: + """Flush queued frames, close the stream, and wait for shutdown.""" + if not self._closed: + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + if self._serve_started: + await self._done.wait() + else: + self._finish_channels() + + async def _serve( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: + """Run the reader and writer until EOF, termination, or ``aclose``.""" + self._serve_started = True + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + task_status.started() + await self._reader() + # Reader is done: let the writer flush queued frames + # (close replies), then stop it. + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + nursery.cancel_scope.cancel() + finally: + self._closed = True + self._outbound_send.close() + self._finish_channels() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _reader(self) -> None: + decoder = FrameDecoder() + try: + while True: + try: + data = await self._stream.receive_some(RECEIVE_CHUNK) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + if not self._closed: + self._error = exc + return + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("connection closed (no gateway termination)") + for message in decoder.feed(data): + self._trace("received", message) + self._dispatch(message) + except GatewayReceivedTerminate: + self._trace("received GATEWAY_TERMINATE") + except EOFError as exc: + self._trace("EOF without prior gateway termination message") + self._error = exc + + async def _writer(self) -> None: + try: + async for frame in self._outbound: + await self._stream.send_all(frame) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + self._trace("writer failed", exc) + if self._error is None: + self._error = exc + self._outbound_send.close() # fail future sends fast + else: + # Queue closed and drained: signal write-EOF to the peer. + with suppress(Exception): + await self._stream.send_eof() + finally: + self._writer_done.set() + + def _dispatch(self, message: Message) -> None: + """Route one message; runs inline on the serve task.""" + code = message.msgcode + channelid = message.channelid + if code == Message.CHANNEL_DATA: + self._channel_for(channelid)._feed(message.data) + elif code == Message.CHANNEL_CLOSE: + self._channel_for(channelid)._close_from_remote(None, sendonly=False) + elif code == Message.CHANNEL_CLOSE_ERROR: + error_message = loads_internal(message.data) + assert isinstance(error_message, str) + self._channel_for(channelid)._close_from_remote( + RemoteError(error_message), sendonly=False + ) + elif code == Message.CHANNEL_LAST_MESSAGE: + self._channel_for(channelid)._close_from_remote(None, sendonly=True) + elif code == Message.GATEWAY_TERMINATE: + raise GatewayReceivedTerminate(self) + elif code == Message.STATUS: + status = { + "numchannels": len(self._channels), + "numexecuting": 0, + "execmodel": "trio", + } + self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data) + assert isinstance(data, tuple) + if channelid == 0: + self._strconfig = data + else: + # picked up by the serialized channel layer + self._channel_for(channelid)._strconfig = data + else: + # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core + self._trace("rejecting unsupported message", message) + self._send_nowait( + Message.CHANNEL_CLOSE_ERROR, + channelid, + dumps_internal(f"unsupported message on async gateway: {message!r}"), + ) + + def _channel_for(self, id: int) -> RawChannel: + try: + return self._channels[id] + except KeyError: + channel = self._channels[id] = RawChannel(self, id) + return channel + + def _forget_channel(self, id: int) -> None: + self._channels.pop(id, None) + + async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + # The queue is unbounded, so this never waits on the peer -- but it + # is a real checkpoint and raises once the gateway is closed. + try: + await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + + def _finish_channels(self) -> None: + for channel in list(self._channels.values()): + channel._close_from_remote(None, sendonly=True) + self._channels.clear() + + +@asynccontextmanager +async def serve_gateway( + stream: ByteStream, *, id: str, _startcount: int = 1 +) -> AsyncIterator[AsyncGateway]: + """Serve an :class:`AsyncGateway` over ``stream`` for the ``with`` body.""" + gateway = AsyncGateway(stream, id=id, _startcount=_startcount) + async with trio.open_nursery() as nursery: + await nursery.start(gateway._serve) + try: + yield gateway + finally: + await gateway.aclose() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index cd20a0f2..b09dca15 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -18,11 +18,12 @@ from contextlib import suppress from typing import TYPE_CHECKING from typing import Any -from typing import Protocol from typing import TypeVar import trio +from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import ByteStream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -41,27 +42,6 @@ _CLOSE_WRITE = object() -RECEIVE_CHUNK = 65536 - - -class ByteStream(Protocol): - """Neutral bidirectional byte-stream protocol for gateway transports. - - ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` - satisfy this structurally; a future anyio backend's byte streams use the - same four names. ``send_eof`` signals write-EOF to the peer (half-close - for sockets; for pipe pairs trio falls back to closing the send half). - """ - - async def send_all(self, data: bytes) -> None: ... - - async def receive_some(self, max_bytes: int | None = None) -> bytes: ... - - async def send_eof(self) -> None: ... - - async def aclose(self) -> None: ... - - def staple_process_stream(process: trio.Process) -> ByteStream: """One bidirectional stream over a Trio Process stdin/stdout pair.""" assert process.stdin is not None diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py new file mode 100644 index 00000000..872c35a5 --- /dev/null +++ b/testing/test_trio_gateway.py @@ -0,0 +1,202 @@ +"""Protocol tests for the trio-native async gateway core (RawChannel level). + +Two AsyncGateways are wired together over an in-memory stream pair (the +transport harness, as in ``TestFrameDecoder``); the assertions cover execnet +semantics: payload routing by channel id, the close/EOF/sendonly state +machine, RemoteError propagation, and gateway termination. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest +import trio +import trio.testing + +from execnet._trio_gateway import AsyncGateway +from execnet.gateway_base import Message +from execnet.gateway_base import RemoteError +from execnet.gateway_base import dumps_internal +from execnet.gateway_base import loads_internal + + +@asynccontextmanager +async def gateway_pair() -> AsyncIterator[tuple[AsyncGateway, AsyncGateway]]: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + left = AsyncGateway(left_stream, id="left", _startcount=1) + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(left._serve) + await nursery.start(right._serve) + try: + yield left, right + finally: + await left.aclose() + await right.aclose() + + +def test_payload_boundaries_are_preserved() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"first payload") + await sender.send_bytes(b"second") + assert await receiver.receive_bytes() == b"first payload" + assert await receiver.receive_bytes() == b"second" + + trio.run(main) + + +def test_send_eof_makes_peer_sendonly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"data") + await sender.send_eof() + assert await receiver.receive_bytes() == b"data" + with pytest.raises(EOFError): + await receiver.receive_bytes() + # the receiver of an EOF may still send back + await receiver.send_bytes(b"reply") + assert await sender.receive_bytes() == b"reply" + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"after eof") + + trio.run(main) + + +def test_close_drains_then_blocks_both_directions() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"x") + await sender.aclose() + # payloads sent before the close still drain + assert await receiver.receive_bytes() == b"x" + with pytest.raises(EOFError): + await receiver.receive_bytes() + with pytest.raises(OSError, match="cannot send"): + await receiver.send_bytes(b"y") + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"z") + + trio.run(main) + + +def test_close_with_error_raises_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.aclose(error="boom happened") + with pytest.raises(RemoteError, match="boom happened"): + await receiver.receive_bytes() + + trio.run(main) + + +def test_async_iteration_yields_payloads_until_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + payloads = [b"a", b"bb", b"ccc"] + for payload in payloads: + await sender.send_bytes(payload) + await sender.send_eof() + assert [data async for data in receiver] == payloads + + trio.run(main) + + +def test_terminate_closes_peer_cleanly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = right.open_raw_channel() + await left.terminate() + await right.wait_closed() + assert right._error is None + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_status_reply_travels_on_raw_channel() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send(Message.STATUS, channel.id) + status = loads_internal(await channel.receive_bytes()) + assert status["execmodel"] == "trio" + assert status["numexecuting"] == 0 + # the peer closes the status channel after the reply + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_unsupported_message_is_rejected_with_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal(("code", None, None, {})), + ) + with pytest.raises(RemoteError, match="unsupported message"): + await channel.receive_bytes() + + trio.run(main) + + +def test_send_after_gateway_close_raises() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left.aclose() + with pytest.raises(OSError, match="cannot send"): + await channel.send_bytes(b"x") + with pytest.raises(OSError, match="already closed"): + left.open_raw_channel() + + trio.run(main) + + +def test_peer_disappearing_surfaces_eof_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + channel = right.open_raw_channel() + # peer vanishes without a termination message + await left_stream.aclose() + await right.wait_closed() + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_mid_frame_eof_is_an_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + frame = Message(Message.CHANNEL_DATA, 1, b"payload").pack() + await left_stream.send_all(frame[:5]) + await left_stream.aclose() + await right.wait_closed() + assert isinstance(right._error, EOFError) + assert "mid-frame" in str(right._error) + + trio.run(main) From ed5f99ffa66d942b16a969e0d2e2c87c2fb85b9d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:06:32 +0200 Subject: [PATCH 21/66] feat: layer the serialized AsyncChannel API on RawChannel The high level of the two-level channel model: AsyncChannel wraps a RawChannel with dumps/loads per item, per-channel strconfig (RECONFIGURE travels the wire, both ends coerce on load), async iteration, receive timeouts via trio.fail_after surfacing execnet's TimeoutError, and wait_closed mirroring the sync waitclose contract (reraise RemoteError). Channel objects serialize over the wire: a duck-typed save_AsyncChannel emits the existing CHANNEL opcode and AsyncGateway grows a factory adapter the Unserializer resolves ids through, so channels received inside items attach to the local gateway like sync channels do. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 144 ++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 6 ++ testing/test_trio_gateway.py | 89 +++++++++++++++++++++- 3 files changed, 227 insertions(+), 12 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 67e103ed..aefb0162 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -24,6 +24,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from contextlib import suppress +from typing import Any from typing import Protocol import trio @@ -32,6 +33,7 @@ from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import RemoteError +from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import loads_internal @@ -81,6 +83,7 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._closed = False # no more sends (local aclose or remote close) self._sent_eof = False self._remote_closed = False + self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) @@ -123,6 +126,7 @@ async def aclose(self, error: str | None = None) -> None: return self._closed = True self._payload_send.close() + self._receive_closed.set() self.gateway._forget_channel(self.id) if not self._remote_closed: # A peer-initiated close needs no reply; a dead gateway is @@ -159,18 +163,129 @@ def _feed(self, data: bytes) -> None: except (trio.BrokenResourceError, trio.ClosedResourceError): pass # locally closed: drop, like the sync channel - def _close_from_remote( - self, error: RemoteError | None, *, sendonly: bool - ) -> None: + def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> None: if error is not None: self._remote_error = error self._remote_closed = True + self._receive_closed.set() if not sendonly: self._closed = True self.gateway._forget_channel(self.id) self._payload_send.close() +class AsyncChannel: + """Serialized object API over a :class:`RawChannel`. + + Every payload is one dumps/loads-serialized item; close/EOF semantics + and error propagation come from the raw layer. Channels are + async-iterable, and :meth:`receive` supports the familiar execnet + timeout (raising ``TimeoutError``). + + Channel objects themselves serialize: sending an AsyncChannel inside an + item transfers a reference the peer receives as its own AsyncChannel + for the same id (the wire CHANNEL opcode, as with sync channels). + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self.gateway = raw.gateway + self.id = raw.id + + def __repr__(self) -> str: + flag = "closed" if self.isclosed() else "open" + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._raw._closed + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side. + + The item must be a simple Python type; OSError is raised when the + channel or gateway is closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + await self._raw.send_bytes(dumps_internal(item)) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + Raises EOFError once the peer closed or sent EOF, a RemoteError for + a peer close-with-error, and TimeoutError if no item arrived within + ``timeout`` seconds. + """ + if timeout is None: + data = await self._raw.receive_bytes() + else: + try: + with trio.fail_after(timeout): + data = await self._raw.receive_bytes() + except trio.TooSlowError: + raise TimeoutError("no item after %r seconds" % timeout) from None + return loads_internal(data, self) + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._raw.send_eof() + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._raw.aclose(error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._raw._receive_closed.wait() + error = self._raw._remote_error or self.gateway._error + if error is not None: + raise error + + async def reconfigure( + self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False + ) -> None: + """Set the string coercion for both ends of this channel.""" + strconfig = (py2str_as_py3str, py3str_as_py2str) + self._raw._strconfig = strconfig + await self.gateway._send( + Message.RECONFIGURE, self.id, dumps_internal(strconfig) + ) + + def __aiter__(self) -> AsyncChannel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + # Unserializer duck-type: loads_internal(data, self) reads _strconfig + # and _channelfactory off the object to resolve CHANNEL opcodes. + + @property + def _strconfig(self) -> tuple[bool, bool]: + return self._raw._strconfig or self.gateway._strconfig + + @property + def _channelfactory(self) -> _AsyncChannelFactory: + return self.gateway._channelfactory + + +class _AsyncChannelFactory: + """Duck-typed factory for the Unserializer CHANNEL opcode (``.new(id)``).""" + + def __init__(self, gateway: AsyncGateway) -> None: + self.gateway = gateway + + def new(self, id: int) -> AsyncChannel: + return self.gateway.open_channel(id) + + class AsyncGateway: """Async-native gateway: the framed Message protocol over a ByteStream. @@ -189,6 +304,8 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._stream = stream self.id = id self._channels: dict[int, RawChannel] = {} + self._async_channels: dict[int, AsyncChannel] = {} + self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) @@ -226,6 +343,15 @@ def open_raw_channel(self, id: int | None = None) -> RawChannel: self._count += 2 return self._channel_for(id) + def open_channel(self, id: int | None = None) -> AsyncChannel: + """Return the serialized channel for ``id``, allocating one if None.""" + raw = self.open_raw_channel(id) + try: + return self._async_channels[raw.id] + except KeyError: + channel = self._async_channels[raw.id] = AsyncChannel(raw) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -240,9 +366,8 @@ async def aclose(self) -> None: self._outbound_send.close() with trio.move_on_after(5): await self._writer_done.wait() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() if self._serve_started: await self._done.wait() else: @@ -269,9 +394,8 @@ async def _serve( self._closed = True self._outbound_send.close() self._finish_channels() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() self._done.set() async def _reader(self) -> None: @@ -364,6 +488,7 @@ def _channel_for(self, id: int) -> RawChannel: def _forget_channel(self, id: int) -> None: self._channels.pop(id, None) + self._async_channels.pop(id, None) async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # The queue is unbounded, so this never waits on the peer -- but it @@ -382,6 +507,7 @@ def _finish_channels(self) -> None: for channel in list(self._channels.values()): channel._close_from_remote(None, sendonly=True) self._channels.clear() + self._async_channels.clear() @asynccontextmanager diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index d36c7542..9d39cce0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1622,3 +1622,9 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 872c35a5..d95622af 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -15,6 +15,7 @@ import trio import trio.testing +from execnet import gateway_base from execnet._trio_gateway import AsyncGateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -129,7 +130,7 @@ async def main() -> None: def test_status_reply_travels_on_raw_channel() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send(Message.STATUS, channel.id) status = loads_internal(await channel.receive_bytes()) @@ -144,7 +145,7 @@ async def main() -> None: def test_unsupported_message_is_rejected_with_remote_error() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send( Message.CHANNEL_EXEC, @@ -159,7 +160,7 @@ async def main() -> None: def test_send_after_gateway_close_raises() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left.aclose() with pytest.raises(OSError, match="cannot send"): @@ -200,3 +201,85 @@ async def main() -> None: assert "mid-frame" in str(right._error) trio.run(main) + + +def test_channel_serializes_builtin_items() -> None: + items = [42, "text", b"bytes", [1, 2], ("a", 1), {"key": [True, None]}, {1, 2}] + + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + for item in items: + await sender.send(item) + await sender.send_eof() + assert [item async for item in receiver] == items + + trio.run(main) + + +def test_channel_receive_timeout() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_channel() + with pytest.raises(gateway_base.TimeoutError): + await channel.receive(timeout=0.05) + + trio.run(main) + + +def test_channel_close_with_error_and_wait_closed() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.aclose(error="exec exploded") + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.receive() + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.wait_closed() + + trio.run(main) + + +def test_channel_wait_closed_on_clean_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.send(1) + await sender.send_eof() + await receiver.wait_closed() + # items sent before the EOF still drain after waitclose + assert await receiver.receive() == 1 + + trio.run(main) + + +def test_channel_objects_travel_over_the_wire() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + carrier = left.open_channel() + right_carrier = right.open_channel(carrier.id) + extra = left.open_channel() + await carrier.send({"reply-to": extra}) + received = await right_carrier.receive() + remote_extra = received["reply-to"] + assert remote_extra.id == extra.id + await remote_extra.send("over the transferred channel") + assert await extra.receive() == "over the transferred channel" + + trio.run(main) + + +def test_channel_reconfigure_string_coercion() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.reconfigure(py3str_as_py2str=True) + await receiver.send("text") + # our side now loads py3 strings as bytes + assert await sender.receive() == b"text" + + trio.run(main) From 80ad5dfade4a14bedcece250af1530fcaacafa8e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:35:21 +0200 Subject: [PATCH 22/66] feat: run async popen gateways with remote_exec inside the user's trio run open_popen_gateway spawns a _trio_worker subprocess, does the handshake, and serves an AsyncGateway directly in the caller's nursery -- the first end-to-end trio-native path with no host thread. AsyncGateway grows remote_exec accepting the same source kinds (string / pure function / module) as the sync API; exec-finish close, RemoteError propagation, and concurrent execs all flow through the raw/serialized channel layers. Source normalization moves to _exec_source (shared by both coordinators; gateway.py re-exports the old names for its tests), and the transport helpers (staple_*, handshake, popen argv) move down into _trio_gateway so the async core has no dependency on the sync host machinery. Co-Authored-By: Claude Fable 5 --- src/execnet/_exec_source.py | 87 ++++++++++++++++++++++++ src/execnet/_trio_gateway.py | 128 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 64 ++---------------- src/execnet/_trio_worker.py | 4 +- src/execnet/gateway.py | 81 ++++------------------ testing/test_gateway.py | 4 +- testing/test_trio_gateway.py | 68 +++++++++++++++++++ testing/test_xspec.py | 4 +- 8 files changed, 307 insertions(+), 133 deletions(-) create mode 100644 src/execnet/_exec_source.py diff --git a/src/execnet/_exec_source.py b/src/execnet/_exec_source.py new file mode 100644 index 00000000..10c55795 --- /dev/null +++ b/src/execnet/_exec_source.py @@ -0,0 +1,87 @@ +"""Normalize ``remote_exec`` sources (string / function / module) to code. + +Shared by the sync coordinator ``Gateway`` and the trio-native +``AsyncGateway`` so both accept the same source kinds with identical +restrictions (pure functions taking ``channel`` first, no closures, no +non-builtin globals). +""" + +from __future__ import annotations + +import inspect +import linecache +import textwrap +import types +from collections.abc import Callable + + +def normalize_exec_source( + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + kwargs: dict[str, object], +) -> tuple[str, str | None, str | None]: + """Return ``(source, file_name, call_name)`` for a CHANNEL_EXEC payload.""" + call_name = None + file_name = None + if isinstance(source, types.ModuleType): + file_name = inspect.getsourcefile(source) + linecache.updatecache(file_name) # type: ignore[arg-type] + source = inspect.getsource(source) + elif isinstance(source, types.FunctionType): + call_name = source.__name__ + file_name = inspect.getsourcefile(source) + source = _source_of_function(source) + else: + source = textwrap.dedent(str(source)) + + if not call_name and kwargs: + raise TypeError("can't pass kwargs to non-function remote_exec") + return source, file_name, call_name + + +def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: + import ast + import builtins + + vars = dict.fromkeys(codeobj.co_varnames) + return [ + node.id + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Name) + and node.id not in vars + and node.id not in builtins.__dict__ + ] + + +def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: + if function.__name__ == "": + raise ValueError("can't evaluate lambda functions'") + # XXX: we dont check before remote instantiation + # if arguments are used properly + try: + sig = inspect.getfullargspec(function) + except AttributeError: + args = inspect.getargspec(function)[0] + else: + args = sig.args + if not args or args[0] != "channel": + raise ValueError("expected first function argument to be `channel`") + + closure = function.__closure__ + codeobj = function.__code__ + + if closure is not None: + raise ValueError("functions with closures can't be passed") + + try: + source = inspect.getsource(function) + except OSError as e: + raise ValueError("can't find source file for %s" % function) from e + + source = textwrap.dedent(source) # just for inner functions + + used_globals = _find_non_builtin_globals(source, codeobj) + if used_globals: + raise ValueError("the use of non-builtin globals isn't supported", used_globals) + + leading_ws = "\n" * (codeobj.co_firstlineno - 1) + return leading_ws + source diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index aefb0162..f51eec9d 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -21,7 +21,11 @@ from __future__ import annotations import math +import subprocess +import sys +import types from collections.abc import AsyncIterator +from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress from typing import Any @@ -29,6 +33,7 @@ import trio +from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message @@ -60,6 +65,69 @@ async def send_eof(self) -> None: ... async def aclose(self) -> None: ... +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) + + +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) + + +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). + """ + from . import _provision + + if getattr(spec, "python", None): + interpreter = _provision.shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] + return args + + +def popen_worker_argv(spec: Any) -> list[str]: + """Argv for a popen worker: direct module launch, or uv-provisioned. + + A bare ``python=`` interpreter without execnet gets execnet + trio + provisioned via ``uv``; otherwise the worker module is launched directly. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + return _provision.uv_worker_argv(spec) + return popen_module_args(spec) + + class RawChannel: """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. @@ -352,6 +420,27 @@ def open_channel(self, id: int | None = None) -> AsyncChannel: channel = self._async_channels[raw.id] = AsyncChannel(raw) return channel + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> AsyncChannel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as the sync ``Gateway.remote_exec``: + a source string, a pure function called with ``channel`` and + ``**kwargs``, or a module. The remote end closes the channel when + execution finishes. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.open_channel() + await self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -522,3 +611,42 @@ async def serve_gateway( yield gateway finally: await gateway.aclose() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn a popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. On + exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed + if it has not exited within a bounded grace period. + """ + from .xspec import XSpec + + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + process = await open_popen_process(popen_worker_argv(spec)) + try: + stream = staple_process_stream(process) + await read_handshake_ack(stream, "popen") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + try: + async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: + try: + yield gateway + finally: + await gateway.terminate() + finally: + with trio.CancelScope(shield=True): + with trio.move_on_after(5): + await process.wait() + if process.returncode is None: + process.kill() + await process.wait() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b09dca15..78be2c71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -24,6 +24,10 @@ from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import ByteStream +from ._trio_gateway import open_popen_process +from ._trio_gateway import popen_worker_argv +from ._trio_gateway import read_handshake_ack +from ._trio_gateway import staple_process_stream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -42,27 +46,6 @@ _CLOSE_WRITE = object() -def staple_process_stream(process: trio.Process) -> ByteStream: - """One bidirectional stream over a Trio Process stdin/stdout pair.""" - assert process.stdin is not None - assert process.stdout is not None - return trio.StapledStream(process.stdin, process.stdout) - - -def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: - """One bidirectional stream over OS pipe fds (worker stdio pipes).""" - return trio.StapledStream( - trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) - ) - - -async def read_handshake_ack(stream: ByteStream, what: str) -> None: - """Wait for the worker's single ``b"1"`` ready byte.""" - ack = await stream.receive_some(1) - if ack != b"1": - raise EOFError(f"bad {what} handshake: {ack!r}") - - _CHANNEL_EOF = object() @@ -458,35 +441,6 @@ def _set() -> None: self._started = False -async def open_popen_process(args: list[str]) -> trio.Process: - return await trio.lowlevel.open_process( - args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - -def popen_module_args(spec: Any) -> list[str]: - """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. - - No source is sent over the wire; the worker imports the installed execnet + - trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). - """ - from . import _provision - - if getattr(spec, "python", None): - interpreter = _provision.shell_split_path(spec.python) - else: - interpreter = [sys.executable] - - args = [*interpreter, "-u"] - if getattr(spec, "dont_write_bytecode", False): - args.append("-B") - args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] - return args - - class _TempIO: """Placeholder IO used only while constructing a Trio-backed Gateway.""" @@ -572,15 +526,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ - from . import _provision - - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) - return _open_trio_gateway(group, spec, args) + return _open_trio_gateway(group, spec, popen_worker_argv(spec)) def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bb71b027..01b0186c 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -246,9 +246,9 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: - from . import _trio_host + from . import _trio_gateway - return _trio_host.staple_fd_stream(read_fd, write_fd) + return _trio_gateway.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 593bc6fe..b1d3ed46 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -5,21 +5,30 @@ from __future__ import annotations -import inspect -import linecache -import textwrap import types from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from . import gateway_base +from ._exec_source import _find_non_builtin_globals +from ._exec_source import _source_of_function +from ._exec_source import normalize_exec_source from .gateway_base import IO from .gateway_base import Channel from .gateway_base import Message from .multi import Group from .xspec import XSpec +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", + "_find_non_builtin_globals", + "_source_of_function", + "rinfo_source", +] + class Gateway(gateway_base.BaseGateway): """Gateway to a local or remote Python Interpreter.""" @@ -127,22 +136,7 @@ def remote_exec( will be available in the global namespace of the remotely executing code. """ - call_name = None - file_name = None - if isinstance(source, types.ModuleType): - file_name = inspect.getsourcefile(source) - linecache.updatecache(file_name) # type: ignore[arg-type] - source = inspect.getsource(source) - elif isinstance(source, types.FunctionType): - call_name = source.__name__ - file_name = inspect.getsourcefile(source) - source = _source_of_function(source) - else: - source = textwrap.dedent(str(source)) - - if not call_name and kwargs: - raise TypeError("can't pass kwargs to non-function remote_exec") - + source, file_name, call_name = normalize_exec_source(source, kwargs) channel = self.newchannel() self._send( Message.CHANNEL_EXEC, @@ -185,52 +179,3 @@ def rinfo_source(channel) -> None: pid=os.getpid(), ) ) - - -def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: - import ast - import builtins - - vars = dict.fromkeys(codeobj.co_varnames) - return [ - node.id - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.Name) - and node.id not in vars - and node.id not in builtins.__dict__ - ] - - -def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: - if function.__name__ == "": - raise ValueError("can't evaluate lambda functions'") - # XXX: we dont check before remote instantiation - # if arguments are used properly - try: - sig = inspect.getfullargspec(function) - except AttributeError: - args = inspect.getargspec(function)[0] - else: - args = sig.args - if not args or args[0] != "channel": - raise ValueError("expected first function argument to be `channel`") - - closure = function.__closure__ - codeobj = function.__code__ - - if closure is not None: - raise ValueError("functions with closures can't be passed") - - try: - source = inspect.getsource(function) - except OSError as e: - raise ValueError("can't find source file for %s" % function) from e - - source = textwrap.dedent(source) # just for inner functions - - used_globals = _find_non_builtin_globals(source, codeobj) - if used_globals: - raise ValueError("the use of non-builtin globals isn't supported", used_globals) - - leading_ws = "\n" * (codeobj.co_firstlineno - 1) - return leading_ws + source diff --git a/testing/test_gateway.py b/testing/test_gateway.py index c3c87e4a..b7098773 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -531,9 +531,9 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - from execnet import _trio_host + from execnet import _trio_gateway - args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + args = _trio_gateway.popen_module_args(execnet.XSpec(spec + "//id=gw0")) assert args[: len(expected_args)] == expected_args assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index d95622af..4825beec 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError from execnet.gateway_base import dumps_internal @@ -272,6 +273,73 @@ async def main() -> None: trio.run(main) +def _remote_add(channel, a, b) -> None: # type: ignore[no-untyped-def] + channel.send(a + b) + + +class TestPopenAsyncGateway: + """Integration: an AsyncGateway serving a real popen worker inside + the user's own trio run (no host thread).""" + + def test_remote_exec_roundtrip(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() + 1)" + ) + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + trio.run(main) + + def test_remote_exec_function_with_kwargs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec(_remote_add, a=40, b=2) + assert await channel.receive() == 42 + + trio.run(main) + + def test_remote_error_propagates(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError('kaboom')") + with pytest.raises(RemoteError, match="kaboom"): + await channel.receive() + + trio.run(main) + + def test_exec_finish_closes_channel_ending_iteration(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(3): channel.send(i)" + ) + assert [item async for item in channel] == [0, 1, 2] + + trio.run(main) + + def test_concurrent_remote_execs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + results = [] + + async def run_one(value: int) -> None: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 10)" + ) + await channel.send(value) + results.append(await channel.receive()) + + async with trio.open_nursery() as nursery: + for value in range(5): + nursery.start_soon(run_one, value) + assert sorted(results) == [0, 10, 20, 30, 40] + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f9681f55..661e50af 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -76,10 +76,10 @@ def test_vagrant_options(self) -> None: assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - from execnet import _trio_host + from execnet import _trio_gateway spec = XSpec("popen//python=sudo python3//id=gw0") - args = _trio_host.popen_module_args(spec) + args = _trio_gateway.popen_module_args(spec) assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: From b3b1ab871c496db9a49730516512fdfe254bd0ca Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:51:33 +0200 Subject: [PATCH 23/66] feat: add AsyncGroup owning gateways as nursery child scopes AsyncGroup is the trio-native group: an async context manager whose nursery serves every makegateway() as a child task. Leaving the block terminates all gateways concurrently with the safe_terminate contract -- GATEWAY_TERMINATE plus a timeout grace, then kill, bounded at roughly twice the timeout even when a kill sticks (issues #43/#221) -- with the cleanup shielded so external cancellation cannot leak workers. open_popen_gateway becomes a thin single-gateway AsyncGroup wrapper, so the popen integration tests now exercise the group termination path too. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 142 ++++++++++++++++++++++++++++------- testing/test_trio_gateway.py | 44 +++++++++++ 2 files changed, 159 insertions(+), 27 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index f51eec9d..37be39c8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -28,11 +28,15 @@ from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress +from typing import TYPE_CHECKING from typing import Any from typing import Protocol import trio +if TYPE_CHECKING: + from typing_extensions import Self + from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -613,21 +617,8 @@ async def serve_gateway( await gateway.aclose() -@asynccontextmanager -async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: - """Spawn a popen worker and serve an AsyncGateway over its stdio. - - Runs inside the caller's own trio run -- no host thread involved. On - exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed - if it has not exited within a bounded grace period. - """ - from .xspec import XSpec - - if not isinstance(spec, XSpec): - spec = XSpec(spec) - if spec.execmodel is None: - # the sync Group normally stamps this before spawning - spec.execmodel = "thread" +async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn a popen worker for ``spec`` and complete the ready handshake.""" process = await open_popen_process(popen_worker_argv(spec)) try: stream = staple_process_stream(process) @@ -637,16 +628,113 @@ async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGa process.kill() await process.wait() raise - try: - async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: - try: - yield gateway - finally: - await gateway.terminate() - finally: - with trio.CancelScope(shield=True): - with trio.move_on_after(5): - await process.wait() - if process.returncode is None: - process.kill() + return stream, process + + +class AsyncGroup: + """Trio-native group: an async context manager owning the gateway nursery. + + Gateways created with :meth:`makegateway` are served as child tasks of + the group's nursery. Leaving the ``async with`` block terminates every + gateway with the safe_terminate contract: GATEWAY_TERMINATE plus a + ``timeout`` grace, then kill -- bounded at roughly twice the timeout + even when a kill gets stuck (see issues #43 / #221). + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._nursery: trio.Nursery | None = None + self._gateways: list[AsyncGateway] = [] + self._processes: dict[AsyncGateway, trio.Process] = {} + + def __repr__(self) -> str: + ids = [gateway.id for gateway in self._gateways] + return f"" + + async def __aenter__(self) -> Self: + self._nursery_manager = trio.open_nursery() + self._nursery = await self._nursery_manager.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> bool | None: + # The serve tasks only end once their gateways shut down, so + # terminate before letting the nursery join its children. + # Shielded: cleanup stays bounded even under cancellation. + terminate_error: BaseException | None = None + try: + with trio.CancelScope(shield=True): + await self.terminate(self._termination_timeout) + except BaseException as error: + terminate_error = error + self._nursery = None + suppress_body_exc = await self._nursery_manager.__aexit__( + exc_type, exc_value, traceback + ) + if terminate_error is not None: + raise terminate_error + return suppress_body_exc + + async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: + """Create a gateway for ``spec`` served on the group's nursery. + + Only popen-style specs (including uv-provisioned ``python=``) are + supported on the async path for now. + """ + from .xspec import XSpec + + if self._nursery is None: + raise RuntimeError(f"{self!r} is not entered") + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if not (spec.popen or spec.python): + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + if spec.id is None: + spec.id = "gw%d" % len(self._gateways) + stream, process = await _connect_popen_worker(spec) + gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + await self._nursery.start(gateway._serve) + self._gateways.append(gateway) + self._processes[gateway] = process + return gateway + + async def terminate(self, timeout: float | None = None) -> None: + """Terminate all gateways; never hangs (kill after ``timeout``).""" + gateways = list(self._gateways) + self._gateways.clear() + async with trio.open_nursery() as nursery: + for gateway in gateways: + nursery.start_soon(self._terminate_one, gateway, timeout) + + async def _terminate_one( + self, gateway: AsyncGateway, timeout: float | None + ) -> None: + grace = math.inf if timeout is None else timeout + await gateway.terminate() + process = self._processes.pop(gateway, None) + if process is None: + return + with trio.move_on_after(grace): + await process.wait() + if process.returncode is None: + process.kill() + with trio.move_on_after(grace): await process.wait() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. + Convenience for a single-gateway :class:`AsyncGroup`. + """ + async with AsyncGroup() as group: + yield await group.makegateway(spec) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 4825beec..88e52983 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import AsyncGroup from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -340,6 +341,49 @@ async def run_one(value: int) -> None: trio.run(main) +class TestAsyncGroup: + def test_multiple_gateways_with_auto_ids(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + first = await group.makegateway() + second = await group.makegateway() + assert {first.id, second.id} == {"gw0", "gw1"} + for gateway in (first, second): + channel = await gateway.remote_exec("channel.send(42)") + assert await channel.receive() == 42 + + trio.run(main) + + def test_group_exit_terminates_and_reaps_workers(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + await group.makegateway() + await group.makegateway() + processes = list(group._processes.values()) + # workers exited on GATEWAY_TERMINATE, nobody had to kill them + assert [process.returncode for process in processes] == [0, 0] + + trio.run(main) + + def test_terminate_kills_hung_worker_within_bound(self) -> None: + async def main() -> None: + async with AsyncGroup(termination_timeout=1.0) as group: + gateway = await group.makegateway() + await gateway.remote_exec("import time\nwhile True: time.sleep(1)") + processes = list(group._processes.values()) + assert all(process.returncode is not None for process in processes) + + trio.run(main) + + def test_unsupported_spec_is_rejected(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + with pytest.raises(ValueError, match="unsupported spec"): + await group.makegateway("ssh=nowhere.example.invalid") + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): From 8286b778bef0fe068b61cdbd20bd1b91ac5173bc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:44:50 +0200 Subject: [PATCH 24/66] feat: make the via tunnel frame-native over raw channel routing The via transport no longer double-frames. ChannelFactory grows a raw-receiver registry (CHANNEL_DATA payloads for registered ids route verbatim, no serialization) plus allocate_id, giving the sync gateways the low-level half of the two-level channel model. The master relay forwards the sub-worker's ready byte alone and then runs its stdout through a FrameDecoder, sending exactly one whole sub-protocol frame per CHANNEL_DATA -- and writes coordinator payloads (one frame each, by writer construction) straight to the sub's stdin. Coordinator ends of the tunnel match: RawTunnelStream (sync master, replacing the interim ChannelByteStream hack) and RawChannelStream (async master, wrapping a RawChannel as a ByteStream). AsyncGroup accepts via= specs relayed through a group member, and terminates tunneled gateways before their masters. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 92 ++++++++++++++++++++++++++--- src/execnet/_trio_host.py | 111 +++++++++++++++++++++++------------ src/execnet/gateway_base.py | 45 ++++++++++++++ testing/test_trio_gateway.py | 21 +++++++ 4 files changed, 223 insertions(+), 46 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 37be39c8..94f33838 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -246,6 +246,46 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._payload_send.close() +class RawChannelStream: + """``ByteStream`` over a :class:`RawChannel` -- the frame-native via tunnel. + + The gateway writer performs one ``send_all`` per frame, so every raw + payload carries exactly one whole sub-protocol frame (the master relay + keeps that invariant in the other direction). ``receive_some`` buffers + payloads and honours ``max_bytes`` for the handshake read. + """ + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self._buf = bytearray() + self._eof = False + + async def send_all(self, data: bytes) -> None: + await self._raw.send_bytes(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: + try: + self._buf += await self._raw.receive_bytes() + except EOFError: + self._eof = True + except RemoteError as exc: + self._eof = True + raise EOFError(f"via tunnel closed: {exc}") from None + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] + return out + + async def send_eof(self) -> None: + with suppress(OSError): + await self._raw.send_eof() + + async def aclose(self) -> None: + await self._raw.aclose() + + class AsyncChannel: """Serialized object API over a :class:`RawChannel`. @@ -517,7 +557,7 @@ async def _writer(self) -> None: try: async for frame in self._outbound: await self._stream.send_all(frame) - except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc @@ -682,8 +722,9 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Only popen-style specs (including uv-provisioned ``python=``) are - supported on the async path for now. + Popen-style specs (including uv-provisioned ``python=``) and + ``via=`` sub-gateways relayed through a group member are supported + on the async path for now. """ from .xspec import XSpec @@ -698,20 +739,53 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) - stream, process = await _connect_popen_worker(spec) + process: trio.Process | None = None + if spec.via: + stream: ByteStream = await self._open_via_stream(spec) + else: + stream, process = await _connect_popen_worker(spec) gateway = AsyncGateway(stream, id=spec.id, _startcount=1) await self._nursery.start(gateway._serve) self._gateways.append(gateway) - self._processes[gateway] = process + if process is not None: + self._processes[gateway] = process return gateway + async def _open_via_stream(self, spec: Any) -> ByteStream: + """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a + raw channel (each payload one whole sub-protocol frame).""" + from . import _provision + + master = self._gateway_by_id(spec.via) + raw = master.open_raw_channel() + request = _provision.spawn_request(spec) + await master._send(Message.GATEWAY_START_SUB, raw.id, dumps_internal(request)) + stream = RawChannelStream(raw) + await read_handshake_ack(stream, "via") + return stream + + def _gateway_by_id(self, id: str) -> AsyncGateway: + for gateway in self._gateways: + if gateway.id == id: + return gateway + raise KeyError(f"no gateway {id!r} in {self!r}") + async def terminate(self, timeout: float | None = None) -> None: - """Terminate all gateways; never hangs (kill after ``timeout``).""" + """Terminate all gateways; never hangs (kill after ``timeout``). + + Tunneled (``via``) gateways go first so their termination frames + still travel through a live master. + """ gateways = list(self._gateways) self._gateways.clear() - async with trio.open_nursery() as nursery: - for gateway in gateways: - nursery.start_soon(self._terminate_one, gateway, timeout) + tunneled = [gw for gw in gateways if gw not in self._processes] + spawned = [gw for gw in gateways if gw in self._processes] + for batch in (tunneled, spawned): + if not batch: + continue + async with trio.open_nursery() as nursery: + for gateway in batch: + nursery.start_soon(self._terminate_one, gateway, timeout) async def _terminate_one( self, gateway: AsyncGateway, timeout: float | None diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 78be2c71..1bccc30e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -49,36 +49,46 @@ _CHANNEL_EOF = object() -class ChannelByteStream: - """``ByteStream`` tunnelled over a sync execnet ``Channel``. +class RawTunnelStream: + """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol - as raw bytes over a channel to the master, which double-frames it (sub frame - -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it dissolves once low-level raw channels exist (Phase B.4). - - The channel callback (on the host loop) feeds an unbounded memory channel - that ``receive_some`` drains; writes ``channel.send`` raw bytes. + The frame-native via tunnel: the master relays whole sub-protocol + frames as verbatim CHANNEL_DATA payloads (no serialization, no + double-framing), so this stream only buffers payloads; the + sub-session's FrameDecoder sees exact frame boundaries. """ - def __init__(self, channel: Any) -> None: - self._channel = channel + def __init__(self, gateway: BaseGateway, channelid: int) -> None: + self._gateway = gateway + self.channelid = channelid self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() self._eof = False - channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + self._closed = False + gateway._channelfactory.register_raw_receiver( + channelid, self._on_data, self._on_close + ) + + def _on_data(self, data: bytes) -> None: + self._send.send_nowait(data) + + def _on_close(self, error: Any) -> None: + self._send.send_nowait(_CHANNEL_EOF if error is None else error) async def send_all(self, data: bytes) -> None: - self._channel.send(data) + self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) async def receive_some(self, max_bytes: int | None = None) -> bytes: if not self._buf and not self._eof: - data = await self._recv.receive() - if data is _CHANNEL_EOF: + item = await self._recv.receive() + if item is _CHANNEL_EOF: + self._eof = True + elif isinstance(item, Exception): self._eof = True + raise EOFError(f"via tunnel closed: {item}") from None else: - assert isinstance(data, bytes) - self._buf += data + assert isinstance(item, bytes) + self._buf += item if max_bytes is None: max_bytes = len(self._buf) out = bytes(self._buf[:max_bytes]) @@ -86,10 +96,18 @@ async def receive_some(self, max_bytes: int | None = None) -> bytes: return out async def send_eof(self) -> None: - self._channel.close() + self._close_tunnel() async def aclose(self) -> None: - self._channel.close() + self._close_tunnel() + + def _close_tunnel(self) -> None: + if self._closed: + return + self._closed = True + self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + with suppress(OSError): + self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) async def adopt_socket(socket_fd: int) -> trio.SocketStream: @@ -689,24 +707,35 @@ def start_socketserver_via( async def _start_sub_and_relay( gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a requested sub-worker and relay its Message protocol over the channel. - - Runs on the master's Trio host: bytes from the channel go to the sub's - stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """Spawn a requested sub-worker and relay its Message protocol frames. + + Runs on the master's Trio host (the ``via`` transport). The tunnel is + frame-native both ways: coordinator payloads arrive verbatim through the + raw-receiver registry and go to the sub's stdin unchanged (each payload + one whole frame), while the sub's stdout runs through a FrameDecoder so + every CHANNEL_DATA sent back carries exactly one frame -- except the + initial ready byte, which is forwarded on its own for the handshake. A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed before the relayed protocol bytes. """ from . import _provision - channel = gateway._channelfactory.new(channelid) + def send_close_error(text: str) -> None: + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE_ERROR, channelid, dumps_internal(text)) + try: args, preamble = _provision.sub_spawn_argv(request) process = await open_popen_process(args) except Exception as exc: - channel.close(f"could not spawn via sub-gateway: {exc}") + send_close_error(f"could not spawn via sub-gateway: {exc}") return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + gateway._channelfactory.register_raw_receiver( + channelid, + send_ch.send_nowait, + lambda error: send_ch.send_nowait(_CHANNEL_EOF), + ) async def coordinator_to_sub() -> None: assert process.stdin is not None @@ -721,12 +750,18 @@ async def coordinator_to_sub() -> None: async def sub_to_coordinator() -> None: assert process.stdout is not None - while True: - data = await process.stdout.receive_some(65536) - if not data: - break - channel.send(data) - channel.close() + ack = bytes(await process.stdout.receive_some(1)) + if ack: + gateway._send(Message.CHANNEL_DATA, channelid, ack) + decoder = FrameDecoder() + while True: + data = bytes(await process.stdout.receive_some(RECEIVE_CHUNK)) + if not data: + break + for message in decoder.feed(data): + gateway._send(Message.CHANNEL_DATA, channelid, message.pack()) + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE, channelid) try: async with trio.open_nursery() as nursery: @@ -736,9 +771,9 @@ async def sub_to_coordinator() -> None: # Do not let a relay failure crash the host nursery; surface it on # the channel so the coordinator does not hang on the handshake. gateway._trace("via sub relay failed:", exc) - with suppress(Exception): - channel.close(f"via sub-gateway relay failed: {exc}") + send_close_error(f"via sub-gateway relay failed: {exc}") finally: + gateway._channelfactory.unregister_raw_receiver(channelid) with trio.move_on_after(5): await process.wait() @@ -759,14 +794,16 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() - channel = master.newchannel() + channelid = master._channelfactory.allocate_id() request = _provision.spawn_request(spec) - master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) remote = spec.ssh or spec.vagrant_ssh remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteStream(channel) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 9d39cce0..22afa46e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -890,6 +890,12 @@ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._callbacks: dict[ int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] ] = {} + # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim + # (no serialization) -- the low-level raw channel routing used by + # byte-shaped consumers such as the via tunnel. + self._raw_receivers: dict[ + int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] + ] = {} self._writelock = gateway.execmodel.Lock() self.gateway = gateway self.count = startcount @@ -910,6 +916,31 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] = Channel(self.gateway, id) return channel + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def register_raw_receiver( + self, + id: int, + feed: Callable[[bytes], None], + on_close: Callable[[RemoteError | None], None], + ) -> None: + """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. + + ``on_close`` fires once when the channel closes (with the peer's + RemoteError, if any) or when receiving finishes. + """ + self._raw_receivers[id] = (feed, on_close) + + def unregister_raw_receiver(self, id: int) -> None: + self._raw_receivers.pop(id, None) + def channels(self) -> list[Channel]: return self._list(self._channels.values()) @@ -925,6 +956,11 @@ def _no_longer_opened(self, id: int) -> None: callback(endmarker) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + raw = self._raw_receivers.pop(id, None) + if raw is not None: + _feed, on_close = raw + on_close(remoteerror) + return channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -945,6 +981,11 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non def _local_receive(self, id: int, data) -> None: # executes in receiver thread + raw = self._raw_receivers.get(id) + if raw is not None: + feed, _on_close = raw + feed(data) + return channel = self._channels.get(id) try: callback, _endmarker, strconfig = self._callbacks[id] @@ -974,6 +1015,10 @@ def _finished_receiving(self) -> None: self._local_close(id, sendonly=True) for id in self._list(self._callbacks): self._no_longer_opened(id) + for id in self._list(self._raw_receivers): + item = self._raw_receivers.pop(id, None) + if item is not None: + item[1](None) class ChannelFile: diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 88e52983..630d3e7d 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -375,6 +375,27 @@ async def main() -> None: trio.run(main) + def test_via_gateway_relays_through_master(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + master = await group.makegateway("popen//id=master") + sub = await group.makegateway("popen//via=master") + master_channel = await master.remote_exec( + "import os; channel.send(os.getpid())" + ) + sub_channel = await sub.remote_exec( + "import os; channel.send(os.getpid())" + ) + master_pid = await master_channel.receive() + sub_pid = await sub_channel.receive() + # a real second process, reached through the master's relay + assert sub_pid != master_pid + echo = await sub.remote_exec("channel.send(channel.receive() * 2)") + await echo.send(21) + assert await echo.receive() == 42 + + trio.run(main) + def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: From 57f899f78cc9b20e733c77cbb138deb5ce593f36 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:51:16 +0200 Subject: [PATCH 25/66] docs: record B.4 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 145 ++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 handoff-phase-b-async-core.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md new file mode 100644 index 00000000..8ba250ec --- /dev/null +++ b/handoff-phase-b-async-core.md @@ -0,0 +1,145 @@ +# Handoff: Phase B — invert execnet onto an async-native Trio core + +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; +work continues at **B.5**. Plan context lives in session memory +(`trio-port-plan`), but everything needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). ssh paths have a real local harness +in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-24, after commit `8286b77`) + +Trio is the only IO path; no source is shipped over the wire (workers run +`python -m execnet._trio_worker `, foreign/remote interpreters +are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All +transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` +sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). +The architecture is still sync-first: Trio hides behind the sync +`Channel`/`Gateway`/`Group`. Phase B inverts this. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | +| `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | +| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | + +Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair +protocol tests + popen/via integration inside `trio.run`). + +Semantics that MUST survive (xdist depends on them): + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; sends from + the loop thread only enqueue. After close: `OSError("cannot send (already + closed?)")`; `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (see `TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now. + +## Phase B goal + +Three public namespaces, all pure Trio (**no anyio/asyncio** — an +`execnet.anyio` backend is deferred Phase E; keep idioms portable: neutral +stream protocol, sans-IO protocol logic, no gratuitous trio-only constructs): + +``` +execnet.sync – today's blocking API, rebuilt as a facade (top-level + execnet.Group etc. stay as compatibility aliases) +execnet.trio – trio-native AsyncGroup/AsyncGateway/AsyncChannel, awaited + directly inside the user's own trio.run +execnet.portal – the communicating API (exists as portal.py; public + exposure happens in B.6) +``` + +## Remaining work + +### B.4 Async-native core — DONE (commits `0eb6cc5`..`8286b77`) + +Everything lives in `_trio_gateway.py` (see file map). Landed: +RawChannel + AsyncChannel two-level model, AsyncGateway single-task +dispatch (no `_receivelock` on the async path), AsyncGroup with bounded +nursery-scoped termination, async popen/`python=`/`via=` gateways with +`remote_exec` inside the user's own `trio.run`, and the frame-native via +tunnel (raw-receiver registry in the sync `ChannelFactory`; +`ChannelByteStream` is gone). + +Leftovers deliberately deferred: + +- **ChannelFile / makefile over RawChannel** — do together with the B.5 + facade (the current `ChannelFileRead` is str-based; decide there whether + a text wrapper stays for backward compat). +- rsync data plane / wheel shipping over raw channels — later. +- async ssh/socket gateways — the async `makegateway` only does + popen-style and `via=`; other transports stay on the sync host path + until B.5/B.6 need them. +- worker-side CHANNEL_EXEC on an AsyncGateway is rejected with a + RemoteError ("unsupported message") — async exec is Phase C + (`exec=task`). + +### B.5 Rebuild the sync API as a facade + +Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, +`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` +send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, +`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, +`RemoteError`, `DataFormatError`. Existing tests keep running against the +facade unchanged — that is the acceptance bar. + +Known tricky spots: + +- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go + through the portal as a non-waiting post, and tolerate interpreter + shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` + guard). +- KeyboardInterrupt lands on the main thread while blocked inside a portal + call — decide propagation (today the sync side blocks in + `threading.Event.wait`, which KI can interrupt). +- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated + shims. `main_thread_only` must keep working throughout (pytest-xdist runs + GUI-bound code on the worker main thread via + `TrioWorkerExec.integrate_as_primary_thread`). +- Retire `ExecModel` internals and `WorkerPool` at the END of the phase + (WorkerPool is still the worker exec duck-type target for STATUS and is + used by `multi.safe_terminate` + `testing/test_threadpool.py` + + `testing/conftest.py`'s `pool` fixture). + +### B.6 Namespace split + +Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level +`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the +facade; add trio-native tests (memory_stream_pair + FrameDecoder for +protocol-level, real popen gateways inside `trio.run` for integration). + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; rough + major/minor version check (`_trio_worker._check_version`) warns. +- Wheel-on-demand for `GATEWAY_START_SUB` is a recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the sub-spec + has `python=`/`ssh=`). +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `Group.terminate(timeout)` must never hang (bounded even when kill is + stuck). + +## After Phase B (context, don't build now) + +- **C**: worker config axes `loop=thread|main` × `exec=thread|main|task` + (compat: `execmodel=thread` → loop=main+exec=thread; `main_thread_only` → + loop=thread+exec=main); `exec=task` later enables async remote_exec. +- **D**: docs + trio-surface tests, pytest-xdist verification. +- **E** (deferred): `execnet.anyio` — coordinator core on anyio, + `backend="asyncio"` knob for the facade. `FrameDecoder` and the B.4 + channel layers are IO-free by construction, so they port as-is; anyio's + `StapledByteStream` satisfies the `ByteStream` protocol structurally. From f932084db8e0dadd362078a36b74b3c66649096e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:34:23 +0200 Subject: [PATCH 26/66] feat: rebuild the sync API as a facade over the async gateway engine The sync coordinator and worker now run their protocol IO on the B.4 async core instead of the parallel ProtocolSession implementation: - AsyncGateway grows per-frame write acknowledgements (outbound queue items carry an optional on_written callback; the writer fails pending frames on shutdown instead of stranding their senders), plus a _finalize hook for subclass shutdown work. - AsyncGroup learns every transport (ssh=, vagrant_ssh=, socket= with installvia=, alongside popen/python=/via=), an overridable gateway factory, per-process reaper tasks, and remoteaddress stamping. - SyncBridgeGateway subclasses AsyncGateway to dispatch messages into the classic sync Message handlers under the receive lock; it keeps the xdist invariant that non-loop-thread sends block until the OS write (120s -> OSError) while loop-thread sends only enqueue, all in one portal-posted FIFO. ProtocolSession is gone; the worker serves on the same bridge. - multi.Group owns a FacadeAsyncGroup task on its TrioHost: makegateway delegates to AsyncGroup.makegateway, and terminate keeps the exit()/join() member contract while the bounded GATEWAY_TERMINATE + grace + kill shutdown runs through AsyncGroup.terminate. - Channel.__del__ posts its close message through the portal without waiting, so GC never blocks on a dying loop. - RawTunnelStream.aclose now feeds EOF to its own reader; previously a terminated via bridge could wait forever on a reader that no longer had a registered raw receiver. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 235 +++++++++++++-- src/execnet/_trio_host.py | 539 +++++++++++++---------------------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 19 +- src/execnet/multi.py | 61 ++-- testing/test_trio_gateway.py | 2 +- 6 files changed, 460 insertions(+), 398 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 94f33838..fa4f2ba9 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -40,6 +40,7 @@ from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate +from .gateway_base import HostNotFound from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -411,6 +412,8 @@ class AsyncGateway: """ _error: BaseException | None = None + #: where the peer lives, when the transport knows (ssh host, socket addr) + remoteaddress: str | None = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -420,7 +423,9 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) - self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._outbound_send, self._outbound = trio.open_memory_channel[ + tuple[bytes, Callable[[BaseException | None], None] | None] + ](math.inf) self._closed = False self._serve_started = False self._writer_done = trio.Event() @@ -526,10 +531,20 @@ async def _serve( finally: self._closed = True self._outbound_send.close() - self._finish_channels() - with trio.CancelScope(shield=True), suppress(Exception): - await self._stream.aclose() - self._done.set() + try: + await self._finalize() + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _finalize(self) -> None: + """Serve-shutdown hook: release the channel layer. + + The sync facade overrides this to close its sync channels and shut + down execution instead. + """ + self._finish_channels() async def _reader(self) -> None: decoder = FrameDecoder() @@ -554,21 +569,43 @@ async def _reader(self) -> None: self._error = exc async def _writer(self) -> None: + error: BaseException | None = None try: - async for frame in self._outbound: - await self._stream.send_all(frame) + async for frame, on_written in self._outbound: + try: + await self._stream.send_all(frame) + except BaseException as exc: + error = exc + if on_written is not None: + on_written(exc) + raise + if on_written is not None: + on_written(None) except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc - self._outbound_send.close() # fail future sends fast else: # Queue closed and drained: signal write-EOF to the peer. with suppress(Exception): await self._stream.send_eof() finally: + # No more writes will happen: fail queued frames instead of + # leaving their senders waiting on acknowledgements. + self._outbound_send.close() + self._fail_pending_writes(error) self._writer_done.set() + def _fail_pending_writes(self, error: BaseException | None) -> None: + exc = error if error is not None else OSError("cannot send (already closed?)") + while True: + try: + _frame, on_written = self._outbound.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + return + if on_written is not None: + on_written(exc) + def _dispatch(self, message: Message) -> None: """Route one message; runs inline on the serve task.""" code = message.msgcode @@ -627,14 +664,32 @@ async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> No # The queue is unbounded, so this never waits on the peer -- but it # is a real checkpoint and raises once the gateway is closed. try: - await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + await self._outbound_send.send( + (Message(msgcode, channelid, data).pack(), None) + ) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def enqueue_frame( + self, + frame: bytes, + on_written: Callable[[BaseException | None], None] | None = None, + ) -> None: + """Queue one wire frame (sync, loop thread only). + + ``on_written`` fires exactly once: after the frame reached the OS + write, or with the failure when it never will. Raises OSError when + the outbound side is already closed. + """ + try: + self._outbound_send.send_nowait((frame, on_written)) except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: raise OSError("cannot send (already closed?)") from exc def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" - with suppress(trio.BrokenResourceError, trio.ClosedResourceError): - self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + with suppress(OSError): + self.enqueue_frame(Message(msgcode, channelid, data).pack()) def _finish_channels(self) -> None: for channel in list(self._channels.values()): @@ -657,18 +712,99 @@ async def serve_gateway( await gateway.aclose() -async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: - """Spawn a popen worker for ``spec`` and complete the ready handshake.""" - process = await open_popen_process(popen_worker_argv(spec)) +def ssh_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for an ssh worker. + + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. + """ + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.ssh is not None + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble + + +def vagrant_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(vagrant ssh argv, stdin preamble)`` for a vagrant_ssh worker.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + argv = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return argv, preamble + + +async def connect_command_worker( + args: list[str], + *, + preamble: bytes = b"", + remoteaddress: str | None = None, +) -> tuple[ByteStream, trio.Process]: + """Spawn ``args`` and complete the worker ready handshake. + + ``preamble`` is streamed to the worker's stdin before the handshake + (a shipped wheel that the remote receives with ``head -c``). With a + ``remoteaddress``, a handshake EOF plus exit code 255 (ssh could not + reach or authenticate the host) becomes :class:`HostNotFound`. + """ + process = await open_popen_process(args) try: stream = staple_process_stream(process) - await read_handshake_ack(stream, "popen") + if preamble: + await stream.send_all(preamble) + await read_handshake_ack(stream, "bootstrap") + except BaseException as exc: + host_not_found = False + with trio.CancelScope(shield=True): + if isinstance(exc, EOFError) and remoteaddress is not None: + with trio.move_on_after(5): + host_not_found = await process.wait() == 255 + with trio.move_on_after(5): + process.kill() + await process.wait() + if host_not_found: + assert remoteaddress is not None + raise HostNotFound(remoteaddress) from None + raise + return stream, process + + +async def connect_socket_worker( + address: tuple[str, int], remoteaddress: str +) -> ByteStream: + """Connect to a running socketserver and complete the ready handshake.""" + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(remoteaddress) from exc + try: + await read_handshake_ack(stream, "socket") except BaseException: with trio.CancelScope(shield=True), trio.move_on_after(5): - process.kill() - await process.wait() + await stream.aclose() raise - return stream, process + return stream + + +async def start_socketserver_via( + gateway: AsyncGateway, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = gateway.open_channel() + await gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = await channel.receive() + await channel.wait_closed() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) class AsyncGroup: @@ -722,9 +858,10 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Popen-style specs (including uv-provisioned ``python=``) and - ``via=`` sub-gateways relayed through a group member are supported - on the async path for now. + All transport types are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways relayed through a group + member. """ from .xspec import XSpec @@ -732,25 +869,73 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: raise RuntimeError(f"{self!r} is not entered") if not isinstance(spec, XSpec): spec = XSpec(spec) - if not (spec.popen or spec.python): - raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") if spec.execmodel is None: # the sync Group normally stamps this before spawning spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None + remoteaddress: str | None = None if spec.via: stream: ByteStream = await self._open_via_stream(spec) + remote = spec.ssh or spec.vagrant_ssh + if remote: + remoteaddress = f"{remote}[via {spec.via}]" + elif spec.socket: + address, remoteaddress = await self._resolve_socket_address(spec) + stream = await connect_socket_worker(address, remoteaddress) + elif spec.ssh: + args, preamble = ssh_transport_args(spec) + remoteaddress = spec.ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.vagrant_ssh: + args, preamble = vagrant_transport_args(spec) + remoteaddress = spec.vagrant_ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.popen or spec.python: + stream, process = await connect_command_worker(popen_worker_argv(spec)) else: - stream, process = await _connect_popen_worker(spec) - gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + gateway = self._make_gateway(stream, spec) + gateway.remoteaddress = remoteaddress await self._nursery.start(gateway._serve) self._gateways.append(gateway) if process is not None: self._processes[gateway] = process + self._nursery.start_soon(self._reap_process, process) return gateway + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + """Construct the gateway object for a freshly connected stream. + + Overridden by the sync facade to build bridge gateways instead. + """ + return AsyncGateway(stream, id=spec.id, _startcount=1) + + async def _reap_process(self, process: trio.Process) -> None: + # Prompt reaping for workers that exit on their own (no zombies). + with suppress(Exception): + await process.wait() + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + """``((host, port), remoteaddress)`` for a ``socket=`` spec. + + ``installvia=`` asks that group member to start a one-shot + socketserver first. The sync facade overrides this to talk to its + sync master gateway. + """ + if getattr(spec, "installvia", None): + master = self._gateway_by_id(spec.installvia) + realhost, realport = await start_socketserver_via(master) + return (realhost, realport), "%s:%d" % (realhost, realport) + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + return (host_str, int(port_str)), spec.socket + async def _open_via_stream(self, spec: Any) -> ByteStream: """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a raw channel (each payload one whole sub-protocol frame).""" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1bccc30e..bf0e3a7e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -23,11 +23,12 @@ import trio from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream from ._trio_gateway import open_popen_process -from ._trio_gateway import popen_worker_argv from ._trio_gateway import read_handshake_ack -from ._trio_gateway import staple_process_stream +from ._trio_gateway import ssh_transport_args from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -40,10 +41,13 @@ if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway + from .multi import Group T = TypeVar("T") -_CLOSE_WRITE = object() + +# Kept name: the ssh argv/preamble builder moved to the async core. +ssh_trio_args = ssh_transport_args _CHANNEL_EOF = object() @@ -106,6 +110,8 @@ def _close_tunnel(self) -> None: return self._closed = True self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + # Unblock our own reader: no more payloads will be routed here. + self._send.send_nowait(_CHANNEL_EOF) with suppress(OSError): self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) @@ -125,19 +131,21 @@ async def adopt_socket(socket_fd: int) -> trio.SocketStream: class SyncIOHandle: - """Sync IO facade for Group.terminate wait/kill/close_write.""" + """Sync IO facade for Gateway.exit close_write and terminate wait/kill.""" remoteaddress: str def __init__( self, execmodel: ExecModel, - session: ProtocolSession, + session: SyncBridgeGateway, *, + process: trio.Process | None = None, remoteaddress: str | None = None, ) -> None: self.execmodel = execmodel self._session = session + self._process = process if remoteaddress is not None: self.remoteaddress = remoteaddress @@ -148,68 +156,134 @@ def write(self, data: bytes) -> None: raise RuntimeError("sync write not supported on Trio IO handle") def close_read(self) -> None: - self._session.request_close_read() + return def close_write(self) -> None: self._session.request_close_write() def wait(self) -> int | None: - return self._session.wait_process() + process = self._process + if process is None: + return None + + async def _wait() -> int | None: + # Always await wait() so the child is reaped (no zombies). + code: int | None = await process.wait() + return code + + try: + return self._session.host.call(_wait) + except Exception: + return process.returncode def kill(self) -> None: - self._session.kill_process() + process = self._process + if process is None: + return + + async def _kill() -> None: + with trio.move_on_after(5): + process.kill() + await process.wait() + + try: + self._session.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + +class SyncBridgeGateway(AsyncGateway): + """Async engine serving a sync ``BaseGateway``. + The reader/writer/framing machinery is inherited from + :class:`AsyncGateway`; dispatch is overridden to run the classic sync + ``Message`` handlers (channel queues, callbacks, exec scheduling) under + the gateway's receive lock instead of routing to raw channels. -class ProtocolSession: - """Reader/writer tasks for one gateway connection.""" + Foreign threads send through :meth:`enqueue_message`: every enqueue goes + through the portal so loop callbacks and threads land in one global FIFO, + and non-loop threads wait until the frame hit the OS write (120s -> + OSError) so an abrupt ``os._exit`` cannot drop already-"sent" data. + """ def __init__( self, - gateway: BaseGateway, - io: ByteStream, + stream: ByteStream, *, - process: trio.Process | None = None, + id: str, + sync_gateway: BaseGateway, host: TrioHost, ) -> None: - self.gateway = gateway - self.io = io - self.process = process + super().__init__(stream, id=id) + self.sync_gateway = sync_gateway self.host = host - self._outbound_send: trio.MemorySendChannel[object] - self._outbound_recv: trio.MemoryReceiveChannel[object] - self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) - self._done = threading.Event() - self._process_exitcode: int | None = None - self._process_done = threading.Event() + self._done_sync = threading.Event() self._send_closed = False - self._lock = threading.Lock() + self._send_lock = threading.Lock() - def _post_outbound(self, item: object) -> None: - """Schedule ``item`` onto the outbound channel. + # -- engine hooks (run on the host loop) -- - Goes through the portal even from the host thread so every send — - loop callbacks and foreign threads alike — lands in one global FIFO - order. Raises ``trio.RunFinishedError`` after loop shutdown. - """ - self.host.portal.post(self._outbound_send.send_nowait, item) + def _dispatch(self, message: Message) -> None: + gateway = self.sync_gateway + try: + with gateway._receivelock: + message.received(gateway) + except (GatewayReceivedTerminate, EOFError): + raise + except Exception as exc: + gateway._trace("dispatch failed:", gateway._geterrortext(exc)) + raise EOFError("error dispatching message") from exc + + async def _finalize(self) -> None: + gateway = self.sync_gateway + with self._send_lock: + self._send_closed = True + if gateway._error is None: + gateway._error = self._error + gateway._trace("[trio-bridge] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock the worker's join() before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done_sync.set() + gateway._trace("[trio-bridge] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + + # -- sync session interface (any thread) -- def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. - Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist - crash tests) cannot drop already-"sent" data still in the queue. - - The Trio host thread (receiver callbacks) must not wait — that would - deadlock the writer task on the same event loop. + The Trio host thread (receiver callbacks) must not wait — that + would deadlock the writer task on the same event loop. """ + frame = message.pack() wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] - with self._lock: - if self._send_closed or self._done.is_set(): + + def on_written(error: BaseException | None) -> None: + if error is not None: + errors.append(error) + if done is not None: + done.set() + + def post() -> None: + try: + self.enqueue_frame(frame, on_written) + except OSError as exc: + on_written(exc) + + with self._send_lock: + if self._send_closed: raise OSError("cannot send (already closed?)") try: - self._post_outbound((message.pack(), done, errors)) + # Through the portal even from the host thread so every + # send lands in one global FIFO order. + self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None if done is None: @@ -219,155 +293,33 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def post_message(self, message: Message) -> None: + """Best-effort non-waiting send (Channel.__del__ during GC).""" + frame = message.pack() + + def post() -> None: + with suppress(OSError): + self.enqueue_frame(frame) + + try: + self.host.portal.post(post) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None + def request_close_write(self) -> None: - with self._lock: + with self._send_lock: if self._send_closed: return self._send_closed = True with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - - def request_close_read(self) -> None: - # Reader observes EOF / cancel; nothing required from callers. - return + # The writer drains queued frames, then signals write-EOF. + self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done.wait(timeout) - - def wait_process(self) -> int | None: - if self.process is None: - return None - - async def _wait() -> int | None: - assert self.process is not None - # Always await wait() so the child is reaped (no zombies). - code: int | None = await self.process.wait() - self._process_exitcode = code - self._process_done.set() - return code - - try: - return self.host.call(_wait) - except Exception: - self._process_done.wait() - return self._process_exitcode - - def kill_process(self) -> None: - if self.process is None: - return - - async def _kill() -> None: - assert self.process is not None - with trio.move_on_after(5): - self.process.kill() - self._process_exitcode = await self.process.wait() - self._process_done.set() - - try: - self.host.call(_kill) - except Exception as exc: - trace("ERROR killing trio process:", exc) + return self._done_sync.wait(timeout) def is_alive(self) -> bool: - return not self._done.is_set() - - async def task( - self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED - ) -> None: - try: - async with trio.open_nursery() as nursery: - nursery.start_soon(self._writer) - if self.process is not None: - nursery.start_soon(self._supervisor) - task_status.started() - await self._reader() - nursery.cancel_scope.cancel() - finally: - await self._finish() - - async def _reader(self) -> None: - gateway = self.gateway - - def log(*msg: object) -> None: - gateway._trace("[trio-receiver]", *msg) - - log("RECEIVER: starting") - decoder = FrameDecoder() - try: - while True: - data = await self.io.receive_some(RECEIVE_CHUNK) - if not data: - decoder.close() # raises EOFError on a mid-frame EOF - raise EOFError("clean EOF") - for msg in decoder.feed(data): - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - except GatewayReceivedTerminate: - log("GATEWAY_TERMINATE") - except EOFError as exc: - log("EOF without prior gateway termination message") - gateway._error = exc - except Exception as exc: - log(gateway._geterrortext(exc)) - log("finishing receiver") - - async def _writer(self) -> None: - async for item in self._outbound_recv: - if not await self._writer_handle_item(item): - return - - async def _writer_handle_item(self, item: object) -> bool: - """Handle one outbound queue item. Return False when writer should stop.""" - if item is _CLOSE_WRITE: - try: - await self.io.send_eof() - except Exception as exc: - self.gateway._trace("send_eof failed", exc) - return False - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) - try: - await self.io.send_all(blob) - except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() - return True - - async def _supervisor(self) -> None: - assert self.process is not None - try: - self._process_exitcode = await self.process.wait() - finally: - self._process_done.set() - - async def _finish(self) -> None: - gateway = self.gateway - with self._lock: - self._send_closed = True - with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - gateway._trace("[trio-receiver] finishing channels") - gateway._channelfactory._finished_receiving() - # Unblock the worker's join() before heavy exec-pool shutdown - # so the primary thread is not waiting on _done while terminate waits - # on the primary thread draining work. - self._done.set() - gateway._trace("[trio-receiver] terminating execution") - # May sleep/SIGINT; keep it off the Trio scheduling thread. - await trio.to_thread.run_sync( - gateway._terminate_execution, abandon_on_cancel=True - ) - try: - await self.io.aclose() - except Exception: - pass + return not self._done_sync.is_set() class TrioHost: @@ -430,16 +382,15 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: self._nursery.start_soon(async_fn, *args) async def start_session( - self, - gateway: BaseGateway, - io: ByteStream, - *, - process: trio.Process | None = None, - ) -> ProtocolSession: + self, gateway: BaseGateway, io: ByteStream + ) -> SyncBridgeGateway: + """Serve ``gateway`` over ``io`` as a task on the root nursery.""" if self._nursery is None: raise RuntimeError("TrioHost nursery is not available") - session = ProtocolSession(gateway, io, process=process, host=self) - await self._nursery.start(session.task) + session = SyncBridgeGateway( + io, id=str(gateway.id), sync_gateway=gateway, host=self + ) + await self._nursery.start(session._serve) return session def stop(self, timeout: float | None = 5.0) -> None: @@ -484,145 +435,79 @@ def kill(self) -> None: return -def _open_trio_gateway( - group: Any, - spec: Any, - args: list[str], - *, - remoteaddress: str | None = None, - preamble: bytes = b"", -) -> Gateway: - """Spawn ``args``, do the worker handshake, and attach a Trio session. - - Shared by the popen and ssh factories; ``args`` already encodes how the - worker is launched (direct module, uv-provisioned, or wrapped in ssh). - ``preamble`` is streamed to the worker's stdin before the handshake (used to - ship a wheel to a remote that receives it with ``head -c``). - """ - import execnet - - from .gateway_base import HostNotFound - - host: TrioHost = group._ensure_trio_host() - - async def _create_and_attach() -> Gateway: - process = await open_popen_process(args) - try: - async_io = staple_process_stream(process) - if preamble: - await async_io.send_all(preamble) - await read_handshake_ack(async_io, "bootstrap") - except EOFError: - with trio.move_on_after(5): - code = await process.wait() - # ssh exits 255 when it cannot reach/authenticate the host. - if remoteaddress is not None and code == 255: - raise HostNotFound(remoteaddress) from None - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - except BaseException: - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, async_io, process=process) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) - - -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +class FacadeAsyncGroup(AsyncGroup): + """AsyncGroup owning the async side of a sync ``Group``. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Runs on the group's :class:`TrioHost`. Gateways come out as + :class:`SyncBridgeGateway` objects bound to freshly built sync + ``Gateway`` facades, and the via / installvia flows go through the sync + master gateway (its dispatch is sync, so async channels cannot be used + on it). """ - return _open_trio_gateway(group, spec, popen_worker_argv(spec)) - - -def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: - """``(ssh argv, stdin preamble)`` for the Trio ssh path. - - The remote runs the uv-provisioned worker; for a dev coordinator the - preamble carries the wheel bytes that the remote command receives. - """ - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.ssh is not None - return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble - - -def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: - """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args, preamble = ssh_trio_args(spec) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.ssh, preamble=preamble - ) - - -def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: - """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.vagrant_ssh is not None - args = _provision.vagrant_ssh_argv( - spec.vagrant_ssh, spec.ssh_config, remote_command - ) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble - ) + def __init__(self, group: Group, host: TrioHost) -> None: + super().__init__() + self.group = group + self.host = host + self.shutdown = trio.Event() -def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: - """Connect a Trio TCP stream to a running ``execnet-socketserver``. + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + import execnet - The server spawns the worker and synthesises its config, so the coordinator - just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio - session (no local process). - """ - import execnet + sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + bridge = SyncBridgeGateway( + stream, id=spec.id, sync_gateway=sync_gw, host=self.host + ) + sync_gw._attach_trio_session(bridge) + return bridge - from .gateway_base import HostNotFound + async def _open_via_stream(self, spec: Any) -> ByteStream: + from . import _provision - host: TrioHost = group._ensure_trio_host() - if getattr(spec, "installvia", None): - realhost, realport = start_socketserver_via(group[spec.installvia]) - address = (realhost, realport) - remoteaddress = "%s:%d" % (realhost, realport) - else: + master = self.group[spec.via] + channelid = master._channelfactory.allocate_id() + request = _provision.spawn_request(spec) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) + await read_handshake_ack(io, "via") + return io + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + if getattr(spec, "installvia", None): + master = self.group[spec.installvia] + # Blocking sync channel receive on the master: run in a thread + # while this loop keeps dispatching the master's messages. + realhost, realport = await trio.to_thread.run_sync( + start_socketserver_via, master, abandon_on_cancel=True + ) + return (realhost, realport), "%s:%d" % (realhost, realport) assert spec.socket is not None host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) - remoteaddress = spec.socket + return (host_str, int(port_str)), spec.socket - async def _create_and_attach() -> Gateway: - try: - stream = await trio.open_tcp_stream(*address) - except OSError as exc: - raise HostNotFound(remoteaddress) from exc - try: - await read_handshake_ack(stream, "socket") - except BaseException: - with trio.move_on_after(5): - await stream.aclose() - raise + async def run(self, task_status: trio.TaskStatus[FacadeAsyncGroup]) -> None: + """Own the group nursery as a host task until :attr:`shutdown`.""" + async with self: + task_status.started(self) + await self.shutdown.wait() - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, stream) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - return host.call(_create_and_attach) +def makegateway_trio(group: Group, spec: Any) -> Gateway: + """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" + host: TrioHost = group._ensure_trio_host() + async_group: FacadeAsyncGroup = group._ensure_async_group() + bridge = host.call(async_group.makegateway, spec) + assert isinstance(bridge, SyncBridgeGateway) + gw: Gateway = bridge.sync_gateway # type: ignore[assignment] + gw._io = SyncIOHandle( + group.execmodel, + bridge, + process=async_group._processes.get(bridge), + remoteaddress=bridge.remoteaddress, + ) + return gw _socket_worker_counter = itertools.count() @@ -784,31 +669,3 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] host.start_soon(_start_sub_and_relay, gateway, channelid, request) - - -def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" - import execnet - - from . import _provision - - master = group[spec.via] - host: TrioHost = group._ensure_trio_host() - channelid = master._channelfactory.allocate_id() - request = _provision.spawn_request(spec) - remote = spec.ssh or spec.vagrant_ssh - remoteaddress = f"{remote}[via {spec.via}]" if remote else None - - async def _create_and_attach() -> Gateway: - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) - master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) - await read_handshake_ack(io, "via") - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 01b0186c..6a2d04f1 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -224,7 +224,7 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - """Attach ``io`` as the gateway session and serve until shutdown.""" gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) - async def _start() -> _trio_host.ProtocolSession: + async def _start() -> _trio_host.SyncBridgeGateway: return await host.start_session(gateway, io) session = host.call(_start) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 22afa46e..2923eaa3 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -706,7 +706,11 @@ def __del__(self) -> None: else: msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending - self.gateway._send(msgcode, self.id) + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(msgcode, self.id) def _getremoteerror(self): try: @@ -1130,6 +1134,19 @@ def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # ValueError might be because the IO is already closed raise OSError("cannot send (already closed?)") from e + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index c4892995..ca3b69dd 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -12,7 +12,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Sequence -from functools import partial +from contextlib import suppress from threading import Lock from typing import TYPE_CHECKING from typing import Any @@ -51,6 +51,7 @@ def __init__( self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] self._trio_host: Any = None + self._async_group: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -69,6 +70,20 @@ def _ensure_trio_host(self) -> Any: self._trio_host.start() return self._trio_host + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -152,18 +167,9 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if spec.socket: - gw = _trio_host.makegateway_socket_trio(self, spec) - elif spec.via: - gw = _trio_host.makegateway_via_trio(self, spec) - elif spec.ssh: - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif spec.vagrant_ssh: - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif spec.popen: - gw = _trio_host.makegateway_popen_trio(self, spec) - else: + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) if spec.chdir or spec.nice or spec.env: @@ -211,6 +217,10 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._trio_host.call_sync(self._async_group.shutdown.set) + self._async_group = None if self._trio_host is not None: self._trio_host.stop(timeout=1.0) self._trio_host = None @@ -225,7 +235,7 @@ def terminate(self, timeout: float | None = None) -> None: Timeout defaults to None meaning open-ended waiting and no kill attempts. """ - while self: + while self or self._gateways_to_join: vias: set[str] = set() for gw in self: if gw.spec.via: @@ -233,23 +243,16 @@ def terminate(self, timeout: float | None = None) -> None: for gw in self: if gw.id not in vias: gw.exit() - - def join_wait(gw: Gateway) -> None: + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._trio_host.call(self._async_group.terminate, timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: gw.join() - gw._io.wait() - - def kill(gw: Gateway) -> None: - trace("Gateways did not come down after timeout: %r" % gw) - gw._io.kill() - - safe_terminate( - self.execmodel, - timeout, - [ - (partial(join_wait, gw), partial(kill, gw)) - for gw in self._gateways_to_join - ], - ) self._gateways_to_join[:] = [] def remote_exec( diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 630d3e7d..a6f27981 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -400,7 +400,7 @@ def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: with pytest.raises(ValueError, match="unsupported spec"): - await group.makegateway("ssh=nowhere.example.invalid") + await group.makegateway("id=notype") trio.run(main) From 80e237ada130a0379dc35681bb661fd67ff9c322 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:51:32 +0200 Subject: [PATCH 27/66] docs: record B.5 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 88 +++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 8ba250ec..9379ca6d 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,7 +1,7 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; -work continues at **B.5**. Plan context lives in session memory +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; +work continues at **B.6**. Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` @@ -9,28 +9,38 @@ Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). Known flake: `test_socket_installvia` EOFs rarely under load. -## Where the repo stands (2026-07-24, after commit `8286b77`) +## Where the repo stands (2026-07-24, after B.5) Trio is the only IO path; no source is shipped over the wire (workers run `python -m execnet._trio_worker `, foreign/remote interpreters are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). -The architecture is still sync-first: Trio hides behind the sync -`Channel`/`Gateway`/`Group`. Phase B inverts this. + +**The inversion landed (B.5)**: there is one protocol engine — +`AsyncGateway` — and the sync API is a facade over it. `ProtocolSession` +is gone. Coordinator and worker sessions are `SyncBridgeGateway` +(an `AsyncGateway` subclass) whose `_dispatch` runs the classic sync +`Message.received` handlers under `_receivelock`; the sync +`Channel`/`ChannelFactory` state machine in `gateway_base` is unchanged +and remains the semantic layer for blocking users (and the worker's +exec'd code). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; `makegateway` and the bounded process shutdown delegate to +`AsyncGroup`, while `terminate` still calls each member's +`exit()`/`join()` (pinned by `test_basic_group`). File map (src/execnet/): | file | role | |---|---| -| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | -| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (raw-receiver registry: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway` (`_send` via bridge session, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core**: `ByteStream` Protocol, `RawChannel`, `AsyncChannel`, `AsyncGateway` (single serve task; outbound queue items are `(frame, on_written)` so sync senders can wait for the OS write; writer fails pending frames on shutdown; `_finalize` hook for subclass shutdown), `AsyncGroup` (async CM owning the nursery; **all transports**: popen/`python=`/ssh/vagrant_ssh/socket(+installvia)/`via=`; overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`; per-process reaper tasks; terminate = tunneled first, GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream`, `open_popen_gateway`, transport helpers (`connect_command_worker` incl. ssh-255→HostNotFound, `connect_socket_worker`, `start_socketserver_via` (async), `ssh_transport_args`, `popen_worker_argv`) | | `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | -| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | -| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread; `start_session` returns a bridge), **`SyncBridgeGateway`** (sync dispatch + portal-FIFO `enqueue_message`/`post_message`/`request_close_write`, threading-`Event` `wait_done`), **`FacadeAsyncGroup`** (bridge-building AsyncGroup; via/installvia through the sync master), `makegateway_trio`, `SyncIOHandle` (remoteaddress/wait/kill/close_write), `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers (`_start_sub_and_relay` frame-native) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds`; serves on a `SyncBridgeGateway` | | `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | | `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | -| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `multi.py` | sync `Group` facade (`_ensure_async_group`, terminate via `AsyncGroup`), `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | | `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair @@ -87,32 +97,38 @@ Leftovers deliberately deferred: RemoteError ("unsupported message") — async exec is Phase C (`exec=task`). -### B.5 Rebuild the sync API as a facade - -Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, -`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` -send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, -`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, -`RemoteError`, `DataFormatError`. Existing tests keep running against the -facade unchanged — that is the acceptance bar. - -Known tricky spots: - -- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go - through the portal as a non-waiting post, and tolerate interpreter - shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` - guard). -- KeyboardInterrupt lands on the main thread while blocked inside a portal - call — decide propagation (today the sync side blocks in - `threading.Event.wait`, which KI can interrupt). -- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated - shims. `main_thread_only` must keep working throughout (pytest-xdist runs - GUI-bound code on the worker main thread via - `TrioWorkerExec.integrate_as_primary_thread`). -- Retire `ExecModel` internals and `WorkerPool` at the END of the phase - (WorkerPool is still the worker exec duck-type target for STATUS and is - used by `multi.safe_terminate` + `testing/test_threadpool.py` + - `testing/conftest.py`'s `pool` fixture). +### B.5 Rebuild the sync API as a facade — DONE (commit `f932084`) + +Public surface stayed byte-for-byte; the whole sync suite passes +unchanged (508 passed). What landed, and the decisions taken: + +- One engine: `SyncBridgeGateway(AsyncGateway)` serves both coordinator + and worker; `ProtocolSession` deleted. Sync dispatch still runs the + `gateway_base` Message handlers, so the sync `Channel`/`ChannelFactory` + semantics (queues, callbacks, ENDMARKER, weakref drop) are untouched — + and remain what the worker's exec'd code sees. +- Send invariant kept: `enqueue_message` posts every frame through the + portal (one global FIFO); non-loop threads wait on the frame's + `on_written` ack (120s → OSError), the loop thread only enqueues. +- `Channel.__del__` now goes through `_send_nonblocking` → + `SyncBridgeGateway.post_message` (portal post, never waits; falls back + to `_send` for duck-typed test gateways). +- KI decision: blocking data paths (send-ack, receive, waitclose, join) + wait on `threading.Event`/`queue.Queue` — KI-interruptible as before. + Management ops (makegateway, terminate) run inside `portal.run` + (`trio.from_thread.run`) where KI is deferred; accepted for now. +- `Group.terminate` still calls member `exit()`/`join()` (contract pinned + by `test_basic_group`), with process wait/kill delegated to + `AsyncGroup.terminate` (tunneled-first, ~2×timeout bound). +- Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel + (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's + serve task never finishes and terminate hangs. +- `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` + classes as-is for backward compat (they only use channel send/receive). +- NOT yet retired: `ExecModel` internals and `WorkerPool` (still the + worker STATUS duck-type shape and used by `multi.safe_terminate`, + `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at + the end of the phase (B.6 or later) if the tests pinning them move. ### B.6 Namespace split From 4f84335d5a465c8971376663ef47e0cc3f6ba648 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:53:36 +0200 Subject: [PATCH 28/66] feat: split the public surface into execnet.sync / execnet.trio / execnet.portal execnet.sync re-exports the blocking facade; the top-level execnet.* names now alias into it. execnet.trio exposes the async-native core (AsyncGroup/AsyncGateway/AsyncChannel, raw channels, stream helpers, shared serialization + errors) for use inside the caller's own trio.run. execnet.portal (LoopPortal/SyncReceiver) is the communicating layer both build on. The trio and portal modules load lazily via module __getattr__ so plain `import execnet` still does not import the trio event loop. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 59 +++++++++++++++++--------- src/execnet/portal.py | 2 + src/execnet/sync.py | 49 ++++++++++++++++++++++ src/execnet/trio.py | 61 +++++++++++++++++++++++++++ testing/test_namespaces.py | 85 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 237 insertions(+), 19 deletions(-) create mode 100644 src/execnet/sync.py create mode 100644 src/execnet/trio.py create mode 100644 testing/test_namespaces.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index cb91cb66..2108f4b2 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,29 +4,40 @@ pure python lib for connecting to local and remote Python Interpreters. +Three public namespaces: + +* :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` + names below are aliases into it. +* :mod:`execnet.trio` — the trio-native API, awaited inside your own + ``trio.run``. +* :mod:`execnet.portal` — cross-thread / cross-loop communication + primitives shared by both. + (c) 2012, Holger Krekel and others """ +from typing import Any + from ._version import version as __version__ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from .sync import Channel +from .sync import DataFormatError +from .sync import DumpError +from .sync import Gateway +from .sync import Group +from .sync import HostNotFound +from .sync import LoadError +from .sync import MultiChannel +from .sync import RemoteError +from .sync import RSync +from .sync import TimeoutError +from .sync import XSpec +from .sync import default_group +from .sync import dump +from .sync import dumps +from .sync import load +from .sync import loads +from .sync import makegateway +from .sync import set_execmodel __all__ = [ "Channel", @@ -50,3 +61,13 @@ "makegateway", "set_execmodel", ] + + +def __getattr__(name: str) -> Any: + # Lazy namespace modules: keep ``import execnet`` from loading the + # trio event loop machinery until it is actually used. + if name in ("trio", "portal"): + import importlib + + return importlib.import_module(f".{name}", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 9804ca45..a204cc5a 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -24,6 +24,8 @@ import trio +__all__ = ["LoopPortal", "SyncReceiver"] + T = TypeVar("T") diff --git a/src/execnet/sync.py b/src/execnet/sync.py new file mode 100644 index 00000000..0d3c835f --- /dev/null +++ b/src/execnet/sync.py @@ -0,0 +1,49 @@ +"""The blocking execnet API. + +A facade over the trio-native core in :mod:`execnet.trio`: gateways run +their protocol IO on a dedicated Trio host thread while this surface +blocks the calling thread. The top-level ``execnet.*`` names are aliases +into this module. +""" + +from .gateway import Gateway +from .gateway_base import Channel +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .multi import Group +from .multi import MultiChannel +from .multi import default_group +from .multi import makegateway +from .multi import set_execmodel +from .rsync import RSync +from .xspec import XSpec + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "dump", + "dumps", + "load", + "loads", + "makegateway", + "set_execmodel", +] diff --git a/src/execnet/trio.py b/src/execnet/trio.py new file mode 100644 index 00000000..df0ab493 --- /dev/null +++ b/src/execnet/trio.py @@ -0,0 +1,61 @@ +"""The trio-native execnet API. + +Everything here is awaited directly inside your own ``trio.run`` — no +host thread, no blocking calls:: + + import trio + import execnet.trio + + async def main(): + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + trio.run(main) + +The serialization helpers and error types are shared with the blocking +API in :mod:`execnet.sync`. +""" + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannel +from ._trio_gateway import RawChannelStream +from ._trio_gateway import open_popen_gateway +from ._trio_gateway import serve_gateway +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +__all__ = [ + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", + "ByteStream", + "DataFormatError", + "DumpError", + "HostNotFound", + "LoadError", + "RawChannel", + "RawChannelStream", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", + "serve_gateway", +] diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py new file mode 100644 index 00000000..dd9b3777 --- /dev/null +++ b/testing/test_namespaces.py @@ -0,0 +1,85 @@ +"""The three public namespaces: execnet.sync / execnet.trio / execnet.portal. + +Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the +trio namespace exposes the async-native core; the portal namespace the +cross-thread primitives. +""" + +from __future__ import annotations + +import subprocess +import sys + +import execnet +import execnet.portal +import execnet.sync +import execnet.trio + + +def test_top_level_names_alias_execnet_sync() -> None: + for name in execnet.sync.__all__: + assert getattr(execnet, name) is getattr(execnet.sync, name), name + + +def test_top_level_all_matches_sync_surface() -> None: + assert set(execnet.__all__) == set(execnet.sync.__all__) | {"__version__"} + + +def test_trio_namespace_exposes_async_core() -> None: + from execnet import _trio_gateway + + assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup + assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway + assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel + assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway + # serialization + errors are shared with the sync surface + assert execnet.trio.RemoteError is execnet.RemoteError + assert execnet.trio.dumps is execnet.dumps + + +def test_portal_namespace() -> None: + assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.LoopPortal is not None + + +def test_lazy_submodule_attribute_access() -> None: + # After ``import execnet`` alone, execnet.trio / execnet.portal are + # reachable as attributes (PEP 562) without having been imported. + out = subprocess.run( + [ + sys.executable, + "-c", + "import execnet; print(execnet.trio.AsyncGroup.__name__);" + " print(execnet.portal.LoopPortal.__name__)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.split() == ["AsyncGroup", "LoopPortal"] + + +def test_import_execnet_does_not_import_trio() -> None: + # The blocking surface must stay importable without loading the trio + # event loop machinery (it loads lazily on first gateway / namespace + # use). + out = subprocess.run( + [ + sys.executable, + "-c", + "import sys, execnet; print('trio' in sys.modules)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "False" + + +def test_unknown_attribute_raises() -> None: + try: + execnet.does_not_exist + except AttributeError as exc: + assert "does_not_exist" in str(exc) + else: + raise AssertionError("expected AttributeError") From 33decae92ad469da1e6568aa9b20c2c9f002590c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:55:44 +0200 Subject: [PATCH 29/66] docs: record phase B completion in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 9379ca6d..ab078740 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; -work continues at **B.6**. Plan context lives in session memory -(`trio-port-plan`), but everything needed is restated here. +For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is +complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in +session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -130,12 +130,28 @@ unchanged (508 passed). What landed, and the decisions taken: `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at the end of the phase (B.6 or later) if the tests pinning them move. -### B.6 Namespace split - -Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level -`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the -facade; add trio-native tests (memory_stream_pair + FrameDecoder for -protocol-level, real popen gateways inside `trio.run` for integration). +### B.6 Namespace split — DONE (commit `4f84335`) + +`execnet.sync` (blocking facade re-exports; top-level `execnet.*` aliases +into it), `execnet.trio` (AsyncGroup/AsyncGateway/AsyncChannel, raw +channels, stream helpers, shared serialization + errors), and +`execnet.portal` (LoopPortal/SyncReceiver). `trio`/`portal` load lazily +via module `__getattr__`, so `import execnet` still does not import the +trio event loop (pinned by `testing/test_namespaces.py`). Trio-native +protocol/integration tests already live in `testing/test_trio_gateway.py` +(memory_stream_pair + real popen gateways inside `trio.run`). + +### End-of-phase leftovers (deferred into C/D) + +- Retire `ExecModel` internals and `WorkerPool`: still pinned by + `multi.safe_terminate`, `testing/test_threadpool.py`, and conftest's + `pool` fixture, and `get_execmodel`/backend names feed Phase C's + `execmodel=` compat mapping — retire once C lands the new worker + config axes. +- ChannelFile / makefile over RawChannel (async makefile) — untouched; + the sync str-based wrappers stay for backward compat. +- rsync data plane / wheel shipping over raw channels — later. +- Async remote_exec on the worker (`exec=task`) — Phase C. ## Invariants (do not regress) From 51b9053a7663e088532e7b791017a24a7ec92666 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:38:41 +0200 Subject: [PATCH 30/66] fix: attach the bridge session before it can serve messages The worker created its SyncBridgeGateway inside host.call and only attached it to the sync gateway afterwards on the main thread. A coordinator message arriving in that window (STATUS, CHANNEL_EXEC as the first action on a fresh gateway) dispatched into a gateway whose _trio_session was still None, so the reply fell through to the sync IO stub and killed the session -- the whole test suite under pytest -n 12 showed this as freshly created gateways being dead on arrival. The race predates the B.5 facade (ProtocolSession had the same window). SyncBridgeGateway now attaches itself in __init__, before its serve task can dispatch anything, and the redundant attach calls at the two construction sites are gone. Also drop the apipkg-era unknown-attribute test from the namespace tests -- unknown attributes are plain Python module behaviour now. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 8 +++++--- src/execnet/_trio_worker.py | 5 +++-- testing/test_namespaces.py | 9 --------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index bf0e3a7e..4d05690d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -220,6 +220,10 @@ def __init__( self._done_sync = threading.Event() self._send_closed = False self._send_lock = threading.Lock() + # Attach before any serving can happen: the first inbound message + # may need to reply through gateway._send, which must already + # route to this session (not the sync IO stub). + sync_gateway._attach_trio_session(self) # -- engine hooks (run on the host loop) -- @@ -455,11 +459,9 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) - bridge = SyncBridgeGateway( + return SyncBridgeGateway( stream, id=spec.id, sync_gateway=sync_gw, host=self.host ) - sync_gw._attach_trio_session(bridge) - return bridge async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6a2d04f1..bfaeea43 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -225,10 +225,11 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) async def _start() -> _trio_host.SyncBridgeGateway: + # The bridge attaches itself to the gateway before serving starts, + # so inbound messages can reply through gateway._send right away. return await host.start_session(gateway, io) - session = host.call(_start) - gateway._attach_trio_session(session) + host.call(_start) try: if main_thread_only: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index dd9b3777..5f44f8a9 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -74,12 +74,3 @@ def test_import_execnet_does_not_import_trio() -> None: check=True, ) assert out.stdout.strip() == "False" - - -def test_unknown_attribute_raises() -> None: - try: - execnet.does_not_exist - except AttributeError as exc: - assert "does_not_exist" in str(exc) - else: - raise AssertionError("expected AttributeError") From a69b844d9e5f0d80546147218f18da56d59bb7cf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:40:39 +0200 Subject: [PATCH 31/66] chore: add pytest-xdist to the dev group, ignore claude local settings Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + pyproject.toml | 3 +++ uv.lock | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d734b32f..9f676ac3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ include/ .vagrant.d/ .config/ .local/ +.claude/settings.local.json diff --git a/pyproject.toml b/pyproject.toml index f419c31d..4e94c938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ testing = [ ] [dependency-groups] +dev = [ + "pytest-xdist>=3.8.0", +] testing = [ "execnet[testing]", ] diff --git a/uv.lock b/uv.lock index e4657f7b..15185f2a 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -398,6 +398,9 @@ testing = [ ] [package.dev-dependencies] +dev = [ + { name = "pytest-xdist" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -416,6 +419,7 @@ requires-dist = [ provides-extras = ["testing"] [package.metadata.requires-dev] +dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -819,6 +823,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.5.0" From 72b0eefd392f7f97f4693751091c574bf10b918e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:46:41 +0200 Subject: [PATCH 32/66] =?UTF-8?q?docs:=20hand=20off=20phase=20C/D=20?= =?UTF-8?q?=E2=80=94=20worker=20axes=20gap=20analysis,=20supersede=20the?= =?UTF-8?q?=20B=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 11 ++- handoff-phase-c-worker-axes.md | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 handoff-phase-c-worker-axes.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index ab078740..d3a69ff6 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is -complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in -session memory (`trio-port-plan`), but everything needed is restated here. +**SUPERSEDED by `handoff-phase-c-worker-axes.md`** — kept as the Phase B +record. **Phase B is complete (B.1–B.6)** on branch +`feat/trio-host-thread-io`; work continues at **Phase C**. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -123,6 +123,11 @@ unchanged (508 passed). What landed, and the decisions taken: - Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's serve task never finishes and terminate hangs. +- Post-B fix (`51b9053`): the session-attach startup race — the bridge + must attach itself to the sync gateway in `__init__`, before serving + starts, or a first-message reply goes through the IO stub and kills + the fresh gateway (predated B.5; surfaced by `pytest -n 12`). The + suite now runs xdist-clean. - `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` classes as-is for backward compat (they only use channel send/receive). - NOT yet retired: `ExecModel` internals and `WorkerPool` (still the diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md new file mode 100644 index 00000000..ced2d895 --- /dev/null +++ b/handoff-phase-c-worker-axes.md @@ -0,0 +1,139 @@ +# Handoff: Phase C — worker config axes (`loop=` × `exec=`) + Phase D + +For a fresh session on branch `feat/trio-host-thread-io`. Phase B (the +inversion onto the async-native Trio core) is **complete**; this doc +supersedes `handoff-phase-b-async-core.md` (kept for the B record). +Plan context lives in session memory (`trio-port-plan`), but everything +needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). The suite is xdist-clean: `uv run +pytest testing/ -n 12` passes (~7s) since the session-attach race fix +(`51b9053`) — keep it that way. ssh paths have a real local harness in +`testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-25, after `a69b844`) + +One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API +is a facade over it — coordinator and worker sessions are +`SyncBridgeGateway` (an `AsyncGateway` subclass in `_trio_host.py`) whose +`_dispatch` runs the classic sync `Message.received` handlers under +`_receivelock`; `SyncBridgeGateway.__init__` attaches itself to the sync +gateway *before* serving starts (the fix for the startup race where a +STATUS/CHANNEL_EXEC arriving first replied through the IO stub and killed +the session). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; makegateway and bounded process shutdown delegate to +`AsyncGroup`, while `terminate` keeps the member `exit()`/`join()` +contract (pinned by `test_basic_group`). `AsyncGroup` supports every +transport (popen, `python=`, ssh, vagrant_ssh, socket with installvia, +`via=`). Public namespaces: `execnet.sync` (top-level `execnet.*` +aliases into it), `execnet.trio`, `execnet.portal` (trio/portal lazy via +module `__getattr__`; `import execnet` must not import trio — pinned by +`testing/test_namespaces.py`). + +No source is shipped over the wire: workers run +`python -m execnet._trio_worker `; foreign/remote +interpreters are uv-provisioned via `_provision.py`; dev coordinators +ship a wheel. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + sans-IO `FrameDecoder`, serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`), sync `Channel`/`ChannelFactory` (raw-receiver registry), `BaseGateway`/`WorkerGateway` (`_send` via bridge, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | +| `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | +| `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | +| `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | +| `sync.py` / `trio.py` / `portal.py` | the three public namespaces | +| `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | + +## Semantics that MUST survive (xdist depends on them) + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; loop + thread sends only enqueue. All sends go through one portal-posted FIFO. +- After close: `OSError("cannot send (already closed?)")`; + `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (`TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now + (open decision: revisit in D). +- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues + #43/#221). +- Sync blocking waits (send-ack, receive, waitclose, join) stay on + `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt + them; `portal.run` (KI-deferred) is only for management ops. + +## Phase C — what is missing (nothing of C exists yet) + +Target axes: `loop=thread|main` × `exec=thread|main|task`. +Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; +`execmodel=main_thread_only` → `loop=thread` + `exec=main`. + +1. **Spec surface**: `xspec.py` only knows `execmodel=`. Add `loop=` / + `exec=` keys, combination validation, and the compat mapping. Same + for the worker CLI config JSON (`_provision.worker_cli_arg` still + ships `{"id", "execmodel", "coordinator_version"}`). +2. **`loop=main` does not exist**: the worker always starts `TrioHost` + on a dedicated thread (`serve_popen_trio`), main thread parked in + `gateway.join()` or integrated as exec primary. The compat mapping + means plain `execmodel=thread` workers should end up with `trio.run` + on the main thread — a structural change to `_run_worker`, not a + flag. +3. **`exec=main`** is today's `main_thread_only` machinery + (`integrate_as_primary_thread` + `_executetask_complete` deadlock + guard) — keep behavior, re-home under the new naming. +4. **`exec=task` (async remote_exec)**: `CHANNEL_EXEC` on a pure + `AsyncGateway` is still rejected ("unsupported message" in + `_trio_gateway._dispatch`). Needs a task-based exec scheduler + handing exec'd code an `AsyncChannel`, preserving FIFO admission, + plus explicit cancellation semantics. +5. **STATUS / `remote_status()`** reports `execmodel`; must speak the + new axes compatibly. +6. **Retire `ExecModel`/`WorkerPool`** (deferred from B): replace + internals with plain `threading`/`queue`; deprecated shims for + `group.execmodel`/`set_execmodel`/`spec.execmodel`. Pinned today by + `multi.safe_terminate`, `testing/test_threadpool.py`, conftest's + `pool` fixture, `Channel._items` (`execmodel.queue`), and the worker + STATUS duck-type — gated on the compat mapping landing first. +7. **Wheel-on-demand for `GATEWAY_START_SUB`**: recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the + sub-spec has `python=`/`ssh=`). + +## Phase D — what is missing + +1. **Docs are entirely stale**: `doc/` still describes source + bootstrapping and execmodels; nothing on uv provisioning, the + no-source-shipping/version-compat policy, the three namespaces, or + `execnet.trio`/`execnet.portal` APIs. `doc/implnotes.rst` references + pre-inversion internals. Changelog entries for the whole branch. +2. **Trio-surface test gaps**: async ssh/socket/vagrant transport tests + (the asyncssh harness only exercises the sync facade; both share + `connect_command_worker`, so a direct `AsyncGroup` ssh test is + cheap), cancellation mid-`remote_exec`/`receive`, `send_eof` and + reconfigure against real workers, B.5 engine paths (write-ack + failure, `enqueue_message` after close). +3. **pytest-xdist verification**: run pytest-xdist's own suite against + this branch plus real `-n` smoke runs (crash/endmarker tests, + `main_thread_only` GUI case). Our own suite running `-n 12` green is + necessary but not sufficient. +4. **Close the open decision**: channel callbacks on the loop thread — + keep or move. + +Recommended order: C.1+C.2 first (spec axes + loop placement) since the +compat mapping unblocks the ExecModel retirement, then `exec=task`; run +the xdist verification (D.3) mid-C as an early canary rather than only +at the end. + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; + rough major/minor version check (`_trio_worker._check_version`) warns. +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `import execnet` must not import the trio event loop. +- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO + `FrameDecoder`) — `execnet.anyio` is deferred Phase E. From 2742eb0aee4eb45fe5f39cfd27847146681ab6ed Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 21:05:43 +0200 Subject: [PATCH 33/66] fix: drain asyncssh redirect cleanup in the local ssh harness asyncssh stores its redirect cleanup coroutines (Queue.join et al) un-awaited and only runs them inside process.wait_closed(); the server handler never called it, so every ssh test leaked coroutines that pytest's unraisable hook surfaced as RuntimeWarnings during GC -- visible as stray noise in xdist runs. Await wait_closed() in the handler and drop the filterwarnings mark, which never covered the GC-time warning anyway. Co-Authored-By: Claude Fable 5 --- testing/test_ssh_local.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 086ce877..443b5da2 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -20,14 +20,9 @@ import execnet -pytestmark = [ - pytest.mark.skipif( - shutil.which("ssh") is None, reason="system ssh client required" - ), - # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, - # surfaced by pytest's unraisable-exception hook during GC; harmless here. - pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), -] +pytestmark = pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" +) # Committed, intentionally-insecure test keys (see sshkeys/README.md). SSHKEYS = Path(__file__).parent / "sshkeys" @@ -58,6 +53,10 @@ async def _handle(self, process: asyncssh.SSHServerProcess) -> None: ) await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) process.exit(await proc.wait()) + # Drain asyncssh's redirect cleanup coroutines (Queue.join et al): + # they are stored un-awaited and only run inside wait_closed(); + # skipping this leaks them and GC prints RuntimeWarnings. + await process.wait_closed() async def _serve(self) -> None: self._server = await asyncssh.listen( From 0d123bdbd2476dcee227eb150c22ae8f30c97ef2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 08:50:57 +0200 Subject: [PATCH 34/66] docs: record PR #422 and the xfail audit in the phase C handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ced2d895..351f6c16 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -6,6 +6,10 @@ supersedes `handoff-phase-b-async-core.md` (kept for the B record). Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. +The rework is up as **draft PR pytest-dev/execnet#422** (branch pushed +to the `origin` fork). Keep it updated as C/D land; undraft once the +docs overhaul (D.1) at least covers migration notes. + Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). The suite is xdist-clean: `uv run pytest testing/ -n 12` passes (~7s) since the session-attach race fix @@ -122,6 +126,16 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; necessary but not sufficient. 4. **Close the open decision**: channel callbacks on the loop thread — keep or move. +5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 + consistent XPASSes were investigated — trio's single-loop dispatch + + FIFO admission makes them pass reliably when idle, but under + sustained load `test_gateway_status_busy` (numexecuting race: + `_track_start` runs in a separately scheduled task) and + `test_popen_stderr_tracing` (capfd race) still fail, so the + `flakytest` marks stay. `test_safe_terminate2`'s xpass is CPython + dummy-thread accounting, unrelated to trio. To retire the status + marks for real: retry-poll for `numexecuting == 2` like the tests + already do for `== 0`. Recommended order: C.1+C.2 first (spec axes + loop placement) since the compat mapping unblocks the ExecModel retirement, then `exec=task`; run From 2c7841678694539da379266560313049e4de4522 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 10:02:11 +0200 Subject: [PATCH 35/66] feat: introduce the boundary kit (Wakener/Mailbox/OneShot) in execnet.portal P1 of the host-boundary rethink (handoff-boundary-protocol-rethink.md): loop->consumer egress unifies on a pluggable thread-safe Wakener with two carriers on top -- Mailbox (item streams, generalizing the KI-safe SyncReceiver drain pattern with get_nowait/timeouts) and OneShot (single results). SyncReceiver is replaced by Mailbox; the SyncBridgeGateway write-ack and _done_sync move off ad-hoc threading.Events onto OneShot. Pure refactor: no behavior change, suite stays xdist-clean. Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 129 +++++++++++++++++++++ src/execnet/_trio_host.py | 40 ++++--- src/execnet/_trio_worker.py | 8 +- src/execnet/portal.py | 166 +++++++++++++++++++++++---- testing/test_namespaces.py | 8 +- testing/test_portal.py | 76 ++++++++++++ 6 files changed, 379 insertions(+), 48 deletions(-) create mode 100644 handoff-boundary-protocol-rethink.md create mode 100644 testing/test_portal.py diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md new file mode 100644 index 00000000..52336693 --- /dev/null +++ b/handoff-boundary-protocol-rethink.md @@ -0,0 +1,129 @@ +# Handoff: host boundary protocol rethink (P1–P5, precedes Phase C proper) + +Decided with Ronny 2026-07-26. This supersedes the *ordering* of +`handoff-phase-c-worker-axes.md` (still valid for C content): the boundary +rethink lands first, then C's `loop=`/`exec=` work continues on the +smaller core. The wire protocol (Message framing) is untouched — this is +about how things cross between the trio host loop and its consumers. + +## Rationale + +Consumer→loop is already universal: `LoopPortal.post` rides +`TrioToken.run_sync_soon`, thread-safe from any foreign context (plain +thread, gevent greenlet, asyncio callback, another trio loop). One +ingress, strict FIFO — unchanged. + +Loop→consumer is hardwired to threading primitives (`Channel._items` +execmodel queue, `threading.Event` for waitclose/join/write-acks, +`SyncReceiver`), which is the only reason `ExecModel` still exists. The +fix: the loop never blocks and never knows who listens — it fires a +thread-safe wakeup callable the consumer supplied. Every event loop has +exactly one such primitive: + +| consumer | wakeup primitive | +|---|---| +| plain thread | `threading.Event.set` | +| asyncio | `loop.call_soon_threadsafe` | +| gevent | `hub.loop.async_()` watcher — `send()` is gevent's one documented thread-safe op | +| foreign trio loop | `TrioToken.run_sync_soon` | + +## The boundary kit (grows in `execnet.portal`) + +1. **`Wakener`** — protocol, one thread-safe method `notify()`. The + entire integration surface for a new event loop (eventlet, Qt, …) + is implementing this; no more ExecModel. +2. **`Mailbox[T]`** — unbounded deque + Wakener. `put()` from the loop + (append + notify, never blocks the loop); `get(timeout)` blocking for + sync backends — generalizes the KI-safe `SyncReceiver` drain pattern + (Event.wait + drain, re-check after clear) — and `await`-able for + asyncio/trio wakeners. Replaces `Channel._items`, `SyncReceiver`, + `MultiChannel`'s queue. +3. **`OneShot[T]`** — single-result future on the same wakener. + Replaces the per-send write-ack `threading.Event`; adds + `portal.start(async_fn) -> OneShot` so asyncio callers await + management ops instead of blocking a thread in `portal.run`. + +## Channel unification (decided: full rebase) + +`AsyncGateway._dispatch` becomes the ONLY message router. `RawChannel` +grows a consumer hook: a bound facade diverts payloads into a `Mailbox` +instead of the internal memory channel (callbacks: invoked on the loop +thread, preserving current semantics; moving them later is just posting +via the wakener). + +Sync `Channel` becomes a genuine facade over (raw id, mailbox, portal): + +- `receive()` = `mailbox.get(timeout)` + `loads_internal` at the call + site — deserialization moves off the loop thread; `DataFormatError` + now surfaces at `receive()`. +- `send()` = portal-posted frame + `OneShot` write-ack (non-loop threads + keep block-until-written, 120s → OSError, KI-safe). +- `waitclose()`/`join()` = closed-events on the wakener. +- `__del__` best-effort close keeps `portal.post` (post_message path). +- Unknown inbound ids still auto-create (`_channel_for`). + +Deleted outright: `_receivelock`, `ChannelFactory` dispatch paths +(`_local_receive`/`_local_close`), `SyncBridgeGateway._dispatch`, the +`Message.received` handler-table use, the duplicate close state machine. +`CHANNEL_EXEC`/`GATEWAY_START_*` become per-gateway hooks on the async +core. `TrioWorkerExec` FIFO pump survives unchanged; exec'd code gets +facade channels. + +## Execmodels: retired as machinery, preserved as presets + +execmodel names become presets over three orthogonal axes (spec + worker +CLI config; decided: **both** coordinator and worker side): + +| axis | values | meaning | +|---|---|---| +| `loop` | `main` \| `thread` | where the trio host runs (Phase C) | +| `exec` | `thread` \| `main` \| `task` | where remote_exec code runs (Phase C) | +| `wait` | `thread` \| `gevent` \| `asyncio` | which Wakener blocking facade waits park on (new) | + +Presets: `execmodel=thread` → `loop=main, exec=thread, wait=thread`; +`main_thread_only` → `loop=thread, exec=main, wait=thread`; `gevent` +returns as `loop=thread, exec=thread, wait=gevent` (greenlet channel +waits park the greenlet, not the hub; sends already greenlet-safe via +the portal). eventlet stays dead (third parties can implement Wakener). + +Dies: `ExecModel` ABC, `WorkerPool`, `Reply` (`multi.safe_terminate` +moves to plain threads or delegates to `FacadeAsyncGroup.terminate`). +Stays as deprecated preset-mappers: `get_execmodel`, `set_execmodel`, +`group.execmodel`, `spec.execmodel` (xdist compat). + +## asyncio (decided: land now) + +`execnet.aio` namespace mirroring `execnet.trio`'s surface; every op is +portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full +asyncio-native API over the trio host loop, all transports, no anyio +port. Phase E (anyio-native core) becomes optional purity/perf work. + +## Staging + +1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port + SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. +2. **P2 — channel unification** (big one): sync Channel onto + RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, + no intermediate primitive-swap (suite pins semantics either way). + Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. +3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, + `wait=` joins the C.1 spec axes + worker CLI config. +4. **P4 — `execnet.aio`**. +5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). + +Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D +docs cover the three namespaces + `execnet.aio` + the axes/presets. + +## Invariants (unchanged, re-mapped) + +- Non-loop-thread sends block until the frame hit the OS write (OneShot + write-ack; 120s → OSError); loop-thread sends only enqueue; all sends + one portal FIFO. +- After close: `OSError("cannot send (already closed?)")`. +- exec admission order = message arrival order (TrioWorkerExec._pump). +- Channel callbacks run on the loop thread (revisit-later stands; the + kit makes moving them cheap). +- `Group.terminate(timeout)` never hangs (~2×timeout). +- Sync blocking waits stay KeyboardInterrupt-interruptible on the main + thread (thread Wakener keeps the Event.wait + drain pattern). +- No source shipping; `import execnet` must not import trio. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 4d05690d..68b278d7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal +from .portal import OneShot if TYPE_CHECKING: from .gateway import Gateway @@ -217,7 +218,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync = threading.Event() + self._done_sync: OneShot[None] = OneShot() self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -249,7 +250,7 @@ async def _finalize(self) -> None: # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. - self._done_sync.set() + self._done_sync.set(None) gateway._trace("[trio-bridge] terminating execution") # May sleep/SIGINT; keep it off the Trio scheduling thread. await trio.to_thread.run_sync( @@ -266,20 +267,17 @@ def enqueue_message(self, message: Message) -> None: """ frame = message.pack() wait = not self.host.portal.is_loop_thread() - done = threading.Event() if wait else None - errors: list[BaseException] = [] - - def on_written(error: BaseException | None) -> None: - if error is not None: - errors.append(error) - if done is not None: - done.set() + # The ack carries the write failure as a value (never raised into + # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable + # from a stream error. + ack: OneShot[BaseException | None] | None = OneShot() if wait else None def post() -> None: try: - self.enqueue_frame(frame, on_written) + self.enqueue_frame(frame, ack.set if ack is not None else None) except OSError as exc: - on_written(exc) + if ack is not None: + ack.set(exc) with self._send_lock: if self._send_closed: @@ -290,12 +288,14 @@ def post() -> None: self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None - if done is None: + if ack is None: return - if not done.wait(timeout=120.0): - raise OSError("cannot send (write timed out)") - if errors: - raise OSError("cannot send (already closed?)") from errors[0] + try: + error = ack.wait(timeout=120.0) + except TimeoutError: + raise OSError("cannot send (write timed out)") from None + if error is not None: + raise OSError("cannot send (already closed?)") from error def post_message(self, message: Message) -> None: """Best-effort non-waiting send (Channel.__del__ during GC).""" @@ -320,7 +320,11 @@ def request_close_write(self) -> None: self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done_sync.wait(timeout) + try: + self._done_sync.wait(timeout) + except TimeoutError: + return False + return True def is_alive(self) -> bool: return not self._done_sync.is_set() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bfaeea43..aa3e68d9 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -16,7 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace -from .portal import SyncReceiver +from .portal import Mailbox if TYPE_CHECKING: from . import _trio_host @@ -48,9 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: SyncReceiver[ - tuple[Channel, ExecItem, threading.Event] | None - ] = SyncReceiver() + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the diff --git a/src/execnet/portal.py b/src/execnet/portal.py index a204cc5a..223fdd97 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -1,30 +1,39 @@ -"""Cross-thread / cross-loop communication primitives. +"""Cross-thread / cross-loop communication primitives — the boundary kit. A :class:`LoopPortal` is a handle to a running trio loop that foreign -threads use to run functions on the loop or push work into it. Because -each direction only needs the *receiving* loop's token, two trio loops in -two threads can communicate by holding each other's portal (the sync -facade's host loop and a user loop; the worker loop and the process main -thread). - -:class:`SyncReceiver` covers the opposite direction: a plain thread -(typically the process main thread) receiving items produced on a loop, -in a way that stays interruptible by KeyboardInterrupt. +threads use to run functions on the loop or push work into it (the +consumer -> loop direction). Because each direction only needs the +*receiving* loop's token, two trio loops in two threads can communicate +by holding each other's portal. + +The loop -> consumer direction never blocks the loop and never knows who +is listening: the loop fires a :class:`Wakener`, a single thread-safe +``notify()`` supplied by the consumer. Implementing that one method is +the entire integration surface for an event loop backend (threads here; +asyncio/gevent wakeners come with their facades). On top of it sit the +two carriers: + +* :class:`Mailbox` -- an item stream (channel payloads, exec requests), +* :class:`OneShot` -- a single result (write acknowledgements, call + results a consumer wants to await instead of block on). """ from __future__ import annotations import queue import threading +import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any from typing import Generic +from typing import Protocol from typing import TypeVar +from typing import cast import trio -__all__ = ["LoopPortal", "SyncReceiver"] +__all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -64,35 +73,142 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: self._token.run_sync_soon(sync_fn, *args) -class SyncReceiver(Generic[T]): - """Receive items on a plain thread, KeyboardInterrupt-friendly. +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. - ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields - KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. - So producers put into an unbounded queue and set a wake event, and - :meth:`get` waits on the event and drains the queue. + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. """ def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wake = threading.Event() + self._wakener = ThreadWakener() if wakener is None else wakener def put(self, item: T) -> None: """Thread-safe; usable from a loop thread (never blocks).""" self._items.put(item) - self._wake.set() + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() - def get(self) -> T: - """Block until an item is available (interruptible on main thread).""" + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout while True: - self._wake.wait() + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None try: return self._items.get_nowait() except queue.Empty: - # Re-check after clearing so a put between get_nowait and - # clear cannot be lost. - self._wake.clear() + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() try: return self._items.get_nowait() except queue.Empty: continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 5f44f8a9..a8bfef0c 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -38,7 +38,13 @@ def test_trio_namespace_exposes_async_core() -> None: def test_portal_namespace() -> None: - assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.__all__ == [ + "LoopPortal", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + ] assert execnet.portal.LoopPortal is not None diff --git a/testing/test_portal.py b/testing/test_portal.py new file mode 100644 index 00000000..1a3759af --- /dev/null +++ b/testing/test_portal.py @@ -0,0 +1,76 @@ +"""The boundary kit: Mailbox and OneShot on the thread wakener.""" + +from __future__ import annotations + +import queue +import threading + +import pytest + +from execnet.portal import Mailbox +from execnet.portal import OneShot + + +class TestMailbox: + def test_put_get_fifo(self) -> None: + box: Mailbox[int] = Mailbox() + for n in range(3): + box.put(n) + assert [box.get(), box.get(), box.get()] == [0, 1, 2] + + def test_get_nowait_empty(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(queue.Empty): + box.get_nowait() + + def test_get_timeout(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(TimeoutError): + box.get(timeout=0.01) + + def test_get_blocks_until_put_from_other_thread(self) -> None: + box: Mailbox[str] = Mailbox() + threading.Timer(0.05, box.put, args=["item"]).start() + assert box.get(timeout=5.0) == "item" + + def test_get_after_stale_wakeup(self) -> None: + # A consumed notify leaves the wakener set; the drain pattern must + # still block (and then receive) rather than spin or miss items. + box: Mailbox[int] = Mailbox() + box.put(1) + assert box.get() == 1 + threading.Timer(0.05, box.put, args=[2]).start() + assert box.get(timeout=5.0) == 2 + + +class TestOneShot: + def test_set_then_wait(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(42) + assert shot.is_set() + assert shot.wait() == 42 + # a resolved OneShot stays readable + assert shot.wait(timeout=0.01) == 42 + + def test_wait_timeout(self) -> None: + shot: OneShot[int] = OneShot() + with pytest.raises(TimeoutError): + shot.wait(timeout=0.01) + assert not shot.is_set() + + def test_set_error_reraises(self) -> None: + shot: OneShot[None] = OneShot() + shot.set_error(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + shot.wait() + + def test_wait_blocks_until_set_from_other_thread(self) -> None: + shot: OneShot[str] = OneShot() + threading.Timer(0.05, shot.set, args=["done"]).start() + assert shot.wait(timeout=5.0) == "done" + + def test_single_resolution_asserted(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(1) + with pytest.raises(AssertionError): + shot.set(2) From 2a40a559c68170f53751d3eef4116b659c70e583 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:05:02 +0200 Subject: [PATCH 36/66] feat!: rebase the sync Channel onto the async core (single dispatch) P2 of the boundary rethink: AsyncGateway._dispatch is now the only Message router. RawChannel grows a consumer hook (set_consumer) that diverts inbound payloads/closes to a bound facade and replays anything that arrived first; the sync Channel becomes a genuine facade -- a Mailbox fed from the loop (deserialization at the receive() call site), callbacks invoked on the loop thread with the switch-over running there too, sends unchanged through gateway._send. Deleted: the classic Message.received handler table, ChannelFactory's dispatch paths (_local_receive, callback/raw-receiver registries) and BaseGateway._receivelock; RawTunnelStream (the via tunnel now uses RawChannelStream over the session's raw channel on both master and worker relay sides). ChannelFactory remains as registry + id allocator, keeping callback channels strongly alive while registered. An unconsumed remotely-closed RawChannel now stays registered until a consumer claims it: with call-site deserialization a passed channel can bind after its data AND close already arrived, and previously the close forgot the raw channel so the late binder found a fresh empty one and lost the payloads (latent race for async open_channel(id) too). The boundary kit moved to the trio-free execnet._boundary so gateway_base can use Mailbox without `import execnet` importing trio; the standalone-source tests inline it / import the package instead. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 161 +++++++++++++++ src/execnet/_trio_gateway.py | 48 ++++- src/execnet/_trio_host.py | 206 +++++++++++-------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 387 ++++++++++++++++------------------- src/execnet/portal.py | 152 +------------- testing/test_basics.py | 28 ++- testing/test_serializer.py | 9 +- 8 files changed, 542 insertions(+), 451 deletions(-) create mode 100644 src/execnet/_boundary.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py new file mode 100644 index 00000000..f80f6998 --- /dev/null +++ b/src/execnet/_boundary.py @@ -0,0 +1,161 @@ +"""Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. + +Importable without loading any event loop (``import execnet`` must not +import trio); ``execnet.portal`` re-exports these next to LoopPortal. +""" + +from __future__ import annotations + +import queue +import threading +import time +from typing import Any +from typing import Generic +from typing import Protocol +from typing import TypeVar +from typing import cast + +__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] + +T = TypeVar("T") + + +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. + + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. + """ + + def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wakener = ThreadWakener() if wakener is None else wakener + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() + + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout + while True: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None + try: + return self._items.get_nowait() + except queue.Empty: + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fa4f2ba9..fc364772 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -159,6 +159,12 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + # Diversion hooks for a bound facade (sync channel): when set, + # inbound payloads/closes route out of the loop instead of + # buffering for receive_bytes. + self._consumer_payload: Callable[[bytes], None] | None = None + self._consumer_close: Callable[[RemoteError | None, bool], None] | None = None + self._pending_close: tuple[RemoteError | None, bool] | None = None def __repr__(self) -> str: state = "closed" if self._closed else "open" @@ -228,9 +234,40 @@ def _pending_error(self) -> BaseException: or EOFError(f"raw channel {self.id} closed") ) + def set_consumer( + self, + on_payload: Callable[[bytes], None], + on_close: Callable[[RemoteError | None, bool], None], + ) -> None: + """Divert inbound payloads and the close to callbacks (loop thread). + + Already-buffered payloads flush to ``on_payload`` first, and a close + that arrived before binding is replayed to ``on_close`` -- so a + consumer bound late (facade channels bind via a portal post) sees + the exact inbound order. + """ + self._consumer_payload = on_payload + self._consumer_close = on_close + while True: + try: + data = self._payloads.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + break + on_payload(data) + if self._pending_close is not None: + error, sendonly = self._pending_close + self._pending_close = None + on_close(error, sendonly) + if not sendonly: + # the late binder has now claimed the buffered close + self.gateway._forget_channel(self.id) + # dispatch-loop internals (inline on the gateway's serve task) def _feed(self, data: bytes) -> None: + if self._consumer_payload is not None: + self._consumer_payload(data) + return try: self._payload_send.send_nowait(data) except (trio.BrokenResourceError, trio.ClosedResourceError): @@ -243,8 +280,17 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._receive_closed.set() if not sendonly: self._closed = True - self.gateway._forget_channel(self.id) + if self._consumer_close is not None: + # Without a consumer the closed channel stays registered: + # a passed-channel reference may still bind late and must + # find the buffered payloads and this close, not a fresh + # empty channel under the same id. + self.gateway._forget_channel(self.id) self._payload_send.close() + if self._consumer_close is not None: + self._consumer_close(error, sendonly) + else: + self._pending_close = (error, sendonly) class RawChannelStream: diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68b278d7..ed4f876e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,12 +7,13 @@ from __future__ import annotations +import functools import itertools import json -import math import subprocess import sys import threading +import weakref from collections.abc import Awaitable from collections.abc import Callable from contextlib import suppress @@ -26,6 +27,7 @@ from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannelStream from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args @@ -33,6 +35,7 @@ from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import RemoteError from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace @@ -51,72 +54,6 @@ ssh_trio_args = ssh_transport_args -_CHANNEL_EOF = object() - - -class RawTunnelStream: - """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - - The frame-native via tunnel: the master relays whole sub-protocol - frames as verbatim CHANNEL_DATA payloads (no serialization, no - double-framing), so this stream only buffers payloads; the - sub-session's FrameDecoder sees exact frame boundaries. - """ - - def __init__(self, gateway: BaseGateway, channelid: int) -> None: - self._gateway = gateway - self.channelid = channelid - self._send, self._recv = trio.open_memory_channel[Any](math.inf) - self._buf = bytearray() - self._eof = False - self._closed = False - gateway._channelfactory.register_raw_receiver( - channelid, self._on_data, self._on_close - ) - - def _on_data(self, data: bytes) -> None: - self._send.send_nowait(data) - - def _on_close(self, error: Any) -> None: - self._send.send_nowait(_CHANNEL_EOF if error is None else error) - - async def send_all(self, data: bytes) -> None: - self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) - - async def receive_some(self, max_bytes: int | None = None) -> bytes: - if not self._buf and not self._eof: - item = await self._recv.receive() - if item is _CHANNEL_EOF: - self._eof = True - elif isinstance(item, Exception): - self._eof = True - raise EOFError(f"via tunnel closed: {item}") from None - else: - assert isinstance(item, bytes) - self._buf += item - if max_bytes is None: - max_bytes = len(self._buf) - out = bytes(self._buf[:max_bytes]) - del self._buf[:max_bytes] - return out - - async def send_eof(self) -> None: - self._close_tunnel() - - async def aclose(self) -> None: - self._close_tunnel() - - def _close_tunnel(self) -> None: - if self._closed: - return - self._closed = True - self._gateway._channelfactory.unregister_raw_receiver(self.channelid) - # Unblock our own reader: no more payloads will be routed here. - self._send.send_nowait(_CHANNEL_EOF) - with suppress(OSError): - self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) - - async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. @@ -229,16 +166,52 @@ def __init__( # -- engine hooks (run on the host loop) -- def _dispatch(self, message: Message) -> None: + """Route one message: sync-facade concerns here, the rest to the core. + + CHANNEL_DATA/CLOSE/CLOSE_ERROR/LAST_MESSAGE fall through to the + async core, which routes them to the RawChannel whose consumer is + the bound sync ``Channel``. + """ gateway = self.sync_gateway + code = message.msgcode try: - with gateway._receivelock: - message.received(gateway) + if code == Message.STATUS: + self._answer_status(message) + elif code == Message.CHANNEL_EXEC: + channel = gateway._channelfactory.new(message.channelid) + gateway._local_schedulexec(channel=channel, sourcetask=message.data) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data, gateway) + assert isinstance(data, tuple) + if message.channelid == 0: + gateway._strconfig = data + else: + gateway._channelfactory.new(message.channelid)._strconfig = data + elif code == Message.GATEWAY_START_SOCKET: + handle_start_socket(gateway, message.channelid, message.data) + elif code == Message.GATEWAY_START_SUB: + handle_start_sub(gateway, message.channelid, message.data) + else: + super()._dispatch(message) except (GatewayReceivedTerminate, EOFError): raise except Exception as exc: gateway._trace("dispatch failed:", gateway._geterrortext(exc)) raise EOFError("error dispatching message") from exc + def _answer_status(self, message: Message) -> None: + # we use the channelid to send back information + # but don't instantiate a channel object + gateway = self.sync_gateway + execpool = getattr(gateway, "_execpool", None) + d = { + "numchannels": len(gateway._channelfactory._channels), + "numexecuting": execpool.active_count() if execpool is not None else 0, + "execmodel": gateway.execmodel.backend, + } + gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) + async def _finalize(self) -> None: gateway = self.sync_gateway with self._send_lock: @@ -247,6 +220,9 @@ async def _finalize(self) -> None: gateway._error = self._error gateway._trace("[trio-bridge] finishing channels") gateway._channelfactory._finished_receiving() + # EOF the loop-side raw channels that have no sync consumer + # (via-tunnel relays and readers blocked in receive_bytes). + self._finish_channels() # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. @@ -259,6 +235,70 @@ async def _finalize(self) -> None: # -- sync session interface (any thread) -- + def bind_sync_channel(self, channel: Any) -> None: + """Route inbound data for ``channel.id`` to the sync channel. + + Callable from any thread: binding happens on the loop via the + portal so it cannot interleave with dispatch, and the raw channel + replays anything (payloads, a close) that arrived first. The loop + side only holds a weakref, preserving the factory's weak-registry + semantics (GC of the last user reference sends the close message); + channels with callbacks are kept alive by the factory instead. + """ + ref = weakref.ref(channel) + channelid = channel.id + + def bind() -> None: + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) + + with suppress(trio.RunFinishedError): + self.host.portal.post(bind) + + def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: + channel = ref() + if channel is not None: + channel._deliver_payload(data) + # dead ref: data for a deleted channel is dropped, like before + + def _sync_close( + self, + ref: weakref.ref[Any], + channelid: int, + error: RemoteError | None, + sendonly: bool, + ) -> None: + channel = ref() + if channel is None: + # channel already in "deleted" state + if error is not None: + error.warn() + self.sync_gateway._channelfactory._no_longer_opened(channelid) + return + channel._close_from_remote(error, sendonly=sendonly) + + def release_channel(self, channelid: int) -> None: + """Drop the loop-side raw channel for ``channelid`` (best-effort).""" + with suppress(trio.RunFinishedError): + self.host.portal.post(self._forget_channel, channelid) + + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: + """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. + + Falls back to running inline once the loop is gone (no more + deliveries can interleave then anyway). + """ + portal = self.host.portal + if portal.is_loop_thread(): + return sync_fn() + try: + return portal.run_sync(sync_fn) + except trio.RunFinishedError: + return sync_fn() + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -471,11 +511,13 @@ async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision master = self.group[spec.via] + session = master._trio_session + assert isinstance(session, SyncBridgeGateway) channelid = master._channelfactory.allocate_id() + # Create the raw channel before the request goes out so no relayed + # frame can arrive unrouted (we are on the loop: no dispatch races). + io = RawChannelStream(session._channel_for(channelid)) request = _provision.spawn_request(spec) - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") return io @@ -602,7 +644,7 @@ async def _start_sub_and_relay( Runs on the master's Trio host (the ``via`` transport). The tunnel is frame-native both ways: coordinator payloads arrive verbatim through the - raw-receiver registry and go to the sub's stdin unchanged (each payload + session's raw channel and go to the sub's stdin unchanged (each payload one whole frame), while the sub's stdout runs through a FrameDecoder so every CHANNEL_DATA sent back carries exactly one frame -- except the initial ready byte, which is forwarded on its own for the handshake. @@ -621,21 +663,17 @@ def send_close_error(text: str) -> None: except Exception as exc: send_close_error(f"could not spawn via sub-gateway: {exc}") return - send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - gateway._channelfactory.register_raw_receiver( - channelid, - send_ch.send_nowait, - lambda error: send_ch.send_nowait(_CHANNEL_EOF), - ) + session = gateway._trio_session + assert isinstance(session, SyncBridgeGateway) + raw = session._channel_for(channelid) async def coordinator_to_sub() -> None: assert process.stdin is not None if preamble: await process.stdin.send_all(preamble) - async for data in recv_ch: - if data is _CHANNEL_EOF: - break - await process.stdin.send_all(data) + with suppress(RemoteError): + async for data in raw: + await process.stdin.send_all(data) with trio.move_on_after(5): await process.stdin.aclose() @@ -664,7 +702,7 @@ async def sub_to_coordinator() -> None: gateway._trace("via sub relay failed:", exc) send_close_error(f"via sub-gateway relay failed: {exc}") finally: - gateway._channelfactory.unregister_raw_receiver(channelid) + session._forget_channel(channelid) with trio.move_on_after(5): await process.wait() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index aa3e68d9..6506199d 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -76,7 +76,7 @@ def _track_finish(self) -> None: self._idle.set() def schedule(self, channel: Channel, sourcetask: bytes) -> None: - """Called from the Trio receiver while holding ``_receivelock``. + """Called from the session dispatch on the Trio host thread. Must not block: deadlock checks and exec run in a nursery task. """ diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2923eaa3..47bf7d32 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -12,9 +12,12 @@ from __future__ import annotations import abc +import builtins import os +import queue as _queue import struct import sys +import threading import traceback import weakref from _thread import interrupt_main @@ -28,6 +31,8 @@ from typing import cast from typing import overload +from ._boundary import Mailbox + class WriteIO(Protocol): def write(self, data: bytes, /) -> None: ... @@ -390,10 +395,37 @@ def trace(*msg: object) -> None: class Message: - """Encapsulates Messages and their wire protocol.""" + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 - # message code -> name, handler - _types: dict[int, tuple[str, Callable[[Message, BaseGateway], None]]] = {} + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.msgcode = msgcode @@ -431,103 +463,10 @@ def from_io(io: ReadIO) -> Message: def to_io(self, io: WriteIO) -> None: io.write(self.pack()) - def received(self, gateway: BaseGateway) -> None: - handler = self._types[self.msgcode][1] - handler(self, gateway) - def __repr__(self) -> str: - name = self._types[self.msgcode][0] + name = self._types[self.msgcode] return f"" - def _status(message: Message, gateway: BaseGateway) -> None: - # we use the channelid to send back information - # but don't instantiate a channel object - d = { - "numchannels": len(gateway._channelfactory._channels), - # TODO(typing): Attribute `_execpool` is only on WorkerGateway. - "numexecuting": gateway._execpool.active_count(), # type: ignore[attr-defined] - "execmodel": gateway.execmodel.backend, - } - gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) - gateway._send(Message.CHANNEL_CLOSE, message.channelid) - - STATUS = 0 - _types[STATUS] = ("STATUS", _status) - - def _reconfigure(message: Message, gateway: BaseGateway) -> None: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - strconfig: tuple[bool, bool] = data - if message.channelid == 0: - gateway._strconfig = strconfig - else: - gateway._channelfactory.new(message.channelid)._strconfig = strconfig - - RECONFIGURE = 1 - _types[RECONFIGURE] = ("RECONFIGURE", _reconfigure) - - def _gateway_terminate(message: Message, gateway: BaseGateway) -> None: - raise GatewayReceivedTerminate(gateway) - - GATEWAY_TERMINATE = 2 - _types[GATEWAY_TERMINATE] = ("GATEWAY_TERMINATE", _gateway_terminate) - - def _channel_exec(message: Message, gateway: BaseGateway) -> None: - channel = gateway._channelfactory.new(message.channelid) - gateway._local_schedulexec(channel=channel, sourcetask=message.data) - - CHANNEL_EXEC = 3 - _types[CHANNEL_EXEC] = ("CHANNEL_EXEC", _channel_exec) - - def _channel_data(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_receive(message.channelid, message.data) - - CHANNEL_DATA = 4 - _types[CHANNEL_DATA] = ("CHANNEL_DATA", _channel_data) - - def _channel_close(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid) - - CHANNEL_CLOSE = 5 - _types[CHANNEL_CLOSE] = ("CHANNEL_CLOSE", _channel_close) - - def _channel_close_error(message: Message, gateway: BaseGateway) -> None: - error_message = loads_internal(message.data) - assert isinstance(error_message, str) - remote_error = RemoteError(error_message) - gateway._channelfactory._local_close(message.channelid, remote_error) - - CHANNEL_CLOSE_ERROR = 6 - _types[CHANNEL_CLOSE_ERROR] = ("CHANNEL_CLOSE_ERROR", _channel_close_error) - - def _channel_last_message(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid, sendonly=True) - - CHANNEL_LAST_MESSAGE = 7 - _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) - - def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: - # Start a one-shot socketserver on this (Trio) gateway's host and reply - # with the bound (host, port) on the request channel. Handled natively - # instead of shipping source via remote_exec. - from . import _trio_host - - _trio_host.handle_start_socket(gateway, message.channelid, message.data) - - GATEWAY_START_SOCKET = 8 - _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - - def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: - # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this - # (Trio) gateway's host and relay its Message protocol over the request - # channel (the ``via`` transport). - from . import _trio_host - - _trio_host.handle_start_sub(gateway, message.channelid, message.data) - - GATEWAY_START_SUB = 9 - _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) - class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. @@ -617,7 +556,13 @@ class TimeoutError(IOError): class Channel: - """Communication channel between two Python Interpreter execution points.""" + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ RemoteError = RemoteError TimeoutError = TimeoutError @@ -632,9 +577,12 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - self._items = self.gateway.execmodel.queue.Queue() + # serialized payloads (or ENDMARKER); None once a callback is set + self._mailbox: Mailbox[Any] | None = Mailbox() + self._callback: Callable[[Any], Any] | None = None + self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = self.gateway.execmodel.Event() + self._receiveclosed = threading.Event() self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -648,33 +596,36 @@ def setcallback( """Set a callback function for receiving items. All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver thread + Afterwards the callback will execute in the receiver (loop) thread for each received data item and calls to ``receive()`` will raise an error. If an endmarker is specified the callback will eventually be called with the endmarker when the channel closes. """ - _callbacks = self.gateway._channelfactory._callbacks - with self.gateway._receivelock: - if self._items is None: + + def switch() -> None: + # Runs on the loop thread (inline without a session), so the + # switch-over cannot interleave with payload delivery. + mailbox = self._mailbox + if mailbox is None: raise OSError(f"{self!r} has callback already registered") - items = self._items - self._items = None + self._mailbox = None while 1: try: - olditem = items.get(block=False) - except self.gateway.execmodel.queue.Empty: + olditem = mailbox.get_nowait() + except _queue.Empty: if not (self._closed or self._receiveclosed.is_set()): - _callbacks[self.id] = (callback, endmarker, self._strconfig) + self._callback = callback + self._endmarker = endmarker + self.gateway._channelfactory._register_callback_channel(self) break - else: - if olditem is ENDMARKER: - items.put(olditem) # for other receivers - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - else: - callback(olditem) + if olditem is ENDMARKER: + if endmarker is not NO_ENDMARKER_WANTED: + callback(endmarker) + break + callback(loads_internal(olditem, self)) + + self.gateway._run_on_loop(switch) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -701,7 +652,7 @@ def __del__(self) -> None: # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) if Message is not None: - if self._items is None: # has_callback + if self._mailbox is None: # has_callback msgcode = Message.CHANNEL_LAST_MESSAGE else: msgcode = Message.CHANNEL_CLOSE @@ -711,6 +662,8 @@ def __del__(self) -> None: self.gateway, "_send_nonblocking", self.gateway._send ) send(msgcode, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) def _getremoteerror(self): try: @@ -722,6 +675,50 @@ def _getremoteerror(self): pass return None + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread).""" + if self._closed: + return # late data for a locally closed channel: drop + callback = self._callback + if callback is None: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no mailbox and no callback: closed for receiving -- drop + else: + try: + callback(loads_internal(data, self)) + except Exception as exc: + self.gateway._trace("exception during callback: %s" % exc) + errortext = self.gateway._geterrortext(exc) + self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) + ) + self._close_from_remote(RemoteError(errortext)) + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread).""" + if remoteerror: + self._remoteerrors.append(remoteerror) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + def _fire_endmarker(self) -> None: + callback = self._callback + if callback is not None: + self._callback = None + if self._endmarker is not NO_ENDMARKER_WANTED: + callback(self._endmarker) + # # public API for channel objects # @@ -788,10 +785,12 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - queue = self._items - if queue is not None: - queue.put(ENDMARKER) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) def waitclose(self, timeout: float | None = None) -> None: """Wait until this channel is closed (or the remote side @@ -841,18 +840,18 @@ def receive(self, timeout: float | None = None) -> Any: reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ - itemqueue = self._items - if itemqueue is None: + mailbox = self._mailbox + if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") try: - x = itemqueue.get(timeout=timeout) - except self.gateway.execmodel.queue.Empty: + x = mailbox.get(timeout) + except builtins.TimeoutError: raise self.TimeoutError("no item after %r seconds" % timeout) from None if x is ENDMARKER: - itemqueue.put(x) # for other receivers + mailbox.put(x) # for other receivers raise self._getremoteerror() or EOFError() else: - return x + return loads_internal(x, self) def __iter__(self) -> Iterator[Any]: return self @@ -886,21 +885,22 @@ def reconfigure( class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message -- and keeps channels with a + registered callback strongly alive until they close. + """ + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # Channel ID => (callback, end marker, strconfig) - self._callbacks: dict[ - int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] - ] = {} - # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim - # (no serialization) -- the low-level raw channel routing used by - # byte-shaped consumers such as the via tunnel. - self._raw_receivers: dict[ - int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] - ] = {} - self._writelock = gateway.execmodel.Lock() + # channels kept strongly alive while their callback is registered + self._callback_channels: dict[int, Channel] = {} + self._writelock = threading.Lock() self.gateway = gateway self.count = startcount self.finished = False @@ -918,6 +918,7 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] except KeyError: channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) return channel def allocate_id(self) -> int: @@ -929,42 +930,21 @@ def allocate_id(self) -> int: self.count += 2 return id - def register_raw_receiver( - self, - id: int, - feed: Callable[[bytes], None], - on_close: Callable[[RemoteError | None], None], - ) -> None: - """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. - - ``on_close`` fires once when the channel closes (with the peer's - RemoteError, if any) or when receiving finishes. - """ - self._raw_receivers[id] = (feed, on_close) - - def unregister_raw_receiver(self, id: int) -> None: - self._raw_receivers.pop(id, None) - def channels(self) -> list[Channel]: return self._list(self._channels.values()) # - # internal methods, called from the receiver thread + # internal methods, called from the loop thread (or local close paths) # + def _register_callback_channel(self, channel: Channel) -> None: + self._callback_channels[channel.id] = channel + def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - item = self._callbacks.pop(id, None) - if item is not None: - callback, endmarker, _strconfig = item - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) + self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - raw = self._raw_receivers.pop(id, None) - if raw is not None: - _feed, on_close = raw - on_close(remoteerror) - return + """Close ``id`` as if the peer had closed it (no message is sent).""" channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -972,57 +952,13 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non remoteerror.warn() self._no_longer_opened(id) else: - # state transition to "closed" state - if remoteerror: - channel._remoteerrors.append(remoteerror) - queue = channel._items - if queue is not None: - queue.put(ENDMARKER) - self._no_longer_opened(id) - if not sendonly: # otherwise #--> "sendonly" - channel._closed = True # --> "closed" - channel._receiveclosed.set() - - def _local_receive(self, id: int, data) -> None: - # executes in receiver thread - raw = self._raw_receivers.get(id) - if raw is not None: - feed, _on_close = raw - feed(data) - return - channel = self._channels.get(id) - try: - callback, _endmarker, strconfig = self._callbacks[id] - except KeyError: - queue = channel._items if channel is not None else None - if queue is None: - pass # drop data - else: - item = loads_internal(data, channel) - queue.put(item) - else: - try: - data = loads_internal(data, channel, strconfig) - callback(data) # even if channel may be already closed - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, id, dumps_internal(errortext) - ) - self._local_close(id, errortext) + channel._close_from_remote(remoteerror, sendonly=sendonly) def _finished_receiving(self) -> None: with self._writelock: self.finished = True for id in self._list(self._channels): self._local_close(id, sendonly=True) - for id in self._list(self._callbacks): - self._no_longer_opened(id) - for id in self._list(self._raw_receivers): - item = self._raw_receivers.pop(id, None) - if item is not None: - item[1](None) class ChannelFile: @@ -1099,7 +1035,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.id = id self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) - self._receivelock = self.execmodel.RLock() # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext @@ -1109,8 +1044,36 @@ def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) def _attach_trio_session(self, session: Any) -> None: - """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + """Attach the Trio bridge session doing the Message IO.""" self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) def _terminate_execution(self) -> None: pass diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 223fdd97..5cc3e30e 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -20,19 +20,18 @@ from __future__ import annotations -import queue -import threading -import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any -from typing import Generic -from typing import Protocol from typing import TypeVar -from typing import cast import trio +from ._boundary import Mailbox +from ._boundary import OneShot +from ._boundary import ThreadWakener +from ._boundary import Wakener + __all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -71,144 +70,3 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: ``trio.RunFinishedError`` once the loop has shut down. """ self._token.run_sync_soon(sync_fn, *args) - - -class Wakener(Protocol): - """Thread-safe consumer wakeup fired by the loop. - - ``notify()`` must never block and must be safe from any thread; it is - the only thing the loop side ever calls. The blocking mailbox/oneshot - waits additionally need :meth:`wait`/:meth:`clear` executed in the - consumer's own context (a thread here, a greenlet for a gevent - wakener); loop-native consumers such as asyncio wait on their own - side of ``notify`` instead. - """ - - def notify(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... - - def clear(self) -> None: ... - - -class ThreadWakener: - """Plain-thread wakener on a ``threading.Event``. - - ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on - the main thread (a C-level ``queue.SimpleQueue.get`` does not), which - is why the carriers wait on the wakener and drain a queue instead of - blocking in the queue itself. - """ - - def __init__(self) -> None: - self._event = threading.Event() - - def notify(self) -> None: - self._event.set() - - def wait(self, timeout: float | None = None) -> bool: - return self._event.wait(timeout) - - def clear(self) -> None: - self._event.clear() - - -class Mailbox(Generic[T]): - """Loop -> consumer item stream: an unbounded queue plus a wakener. - - :meth:`put` never blocks and is safe from any thread (including the - loop thread). :meth:`get` blocks in the consumer's context via the - wakener; after draining to empty it clears and re-checks so a ``put`` - racing the clear cannot be lost. Multiple consumers are allowed. - """ - - def __init__(self, wakener: Wakener | None = None) -> None: - self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wakener = ThreadWakener() if wakener is None else wakener - - def put(self, item: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - self._items.put(item) - self._wakener.notify() - - def get_nowait(self) -> T: - """Return the next item or raise ``queue.Empty``.""" - return self._items.get_nowait() - - def get(self, timeout: float | None = None) -> T: - """Block until an item is available; TimeoutError after ``timeout``.""" - deadline = None if timeout is None else time.monotonic() + timeout - while True: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if remaining <= 0 or not self._wakener.wait(remaining): - # Final non-blocking check: a put may have raced the - # timeout (its notify landing after our last wait). - try: - return self._items.get_nowait() - except queue.Empty: - raise TimeoutError( - "no item after %r seconds" % timeout - ) from None - try: - return self._items.get_nowait() - except queue.Empty: - # Empty: clear, then re-check so a put between get_nowait - # and clear cannot be lost. - self._wakener.clear() - try: - return self._items.get_nowait() - except queue.Empty: - continue - - -class OneShot(Generic[T]): - """A single result crossing loop -> consumer exactly once. - - The loop side calls :meth:`set` (or :meth:`set_error`) at most once; - the consumer blocks in :meth:`wait`, which returns the value, - re-raises the stored error, or raises ``TimeoutError``. - """ - - _NOTSET = object() - - def __init__(self, wakener: Wakener | None = None) -> None: - self._wakener = ThreadWakener() if wakener is None else wakener - self._value: Any = self._NOTSET - self._error: BaseException | None = None - self._done = False - - def is_set(self) -> bool: - return self._done - - def set(self, value: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - assert not self._done, "OneShot already resolved" - self._value = value - self._done = True - self._wakener.notify() - - def set_error(self, error: BaseException) -> None: - """Resolve with an error that :meth:`wait` will re-raise.""" - assert not self._done, "OneShot already resolved" - self._error = error - self._done = True - self._wakener.notify() - - def wait(self, timeout: float | None = None) -> T: - """Block until resolved; TimeoutError after ``timeout`` seconds.""" - deadline = None if timeout is None else time.monotonic() + timeout - while not self._done: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if (remaining <= 0 or not self._wakener.wait(remaining)) and ( - not self._done - ): - raise TimeoutError("not resolved after %r seconds" % timeout) - if self._error is not None: - raise self._error - return cast("T", self._value) diff --git a/testing/test_basics.py b/testing/test_basics.py index 473cd741..29ceef19 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -15,6 +15,7 @@ import pytest import execnet +from execnet import _boundary from execnet import gateway from execnet import gateway_base from execnet.gateway_base import ChannelFactory @@ -66,6 +67,21 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") +def standalone_gateway_base_source() -> str: + """gateway_base's source as a self-contained script. + + The module imports the trio-free boundary kit relatively; for the + run-on-any-python checks the kit source is inlined at the import site + (minus its own future import, which must stay file-leading). + """ + boundary_source = inspect.getsource(_boundary).replace( + "from __future__ import annotations\n", "" + ) + return inspect.getsource(gateway_base).replace( + "from ._boundary import Mailbox\n", boundary_source + ) + + IO_MESSAGE_EXTRA_SOURCE = """ from io import BytesIO @@ -124,7 +140,7 @@ def checker(anypython: str, tmp_path: Path) -> Checker: def test_io_message(checker: Checker) -> None: - out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) + out = checker.run_check(standalone_gateway_base_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout @@ -147,7 +163,7 @@ def send(self, data): def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - inspect.getsource(gateway_base) + standalone_gateway_base_source() + """ class Arg(Exception): pass @@ -324,12 +340,20 @@ class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: + _trio_session = None + def _trace(self, *args) -> None: pass def _send(self, *k) -> None: pass + def _bind_channel(self, channel) -> None: + pass + + def _release_channel(self, id) -> None: + pass + FakeGateway.execmodel = execmodel # type: ignore[attr-defined] return ChannelFactory(FakeGateway()) # type: ignore[arg-type] diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 06a84cd2..cb2d422c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,9 @@ import execnet -# We use the execnet folder in order to avoid triggering a missing apipkg. -pyimportdir = os.fspath(Path(execnet.__file__).parent) +# The package parent: the serializer is imported as execnet.gateway_base +# (it needs its trio-free sibling execnet._boundary; only stdlib beyond that). +pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) class PythonWrapper: @@ -24,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -40,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer from io import BytesIO data = {data!r} io = BytesIO(data) From 811fd9956374b2cf2685094d3aca2f30bcec26cd Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:15:41 +0200 Subject: [PATCH 37/66] feat!: retire the ExecModel machinery; add the wait= spec axis P3 of the boundary rethink. The ExecModel ABC, ThreadExecModel, MainThreadOnlyExecModel, WorkerPool and Reply are gone: protocol IO always runs on the Trio host, blocking waits go through the boundary kit, and execution placement is the worker config axes' job. What survives is a single deprecated ExecModel preset carrying the backend name plus the stdlib-delegating members (pytest-xdist builds its remote test queue on execmodel.RLock/Event, so the duck-type stays live); get_execmodel/set_execmodel/spec.execmodel keep working on top of it. safe_terminate now runs its term/kill pairs on daemon threads with the same bounded-wait contract; MultiChannel.make_receive_queue hands out a plain stdlib queue. New wait= axis end to end: the boundary kit gains Flag (idempotent event on a wakener) and a wakener-factory registry ("thread" built in; gevent/asyncio register later); specs accept wait= (validated in makegateway), the worker CLI config ships it, and every blocking carrier (channel mailbox, receive-closed flag, write-acks, join) is created through gateway._new_wakener() on both sides of the wire. Test suite: test_threadpool.py removed with WorkerPool; the pool fixture became a ThreadPoolExecutor; direct execmodel.queue/Event/sleep uses in tests moved to the stdlib equivalents. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 70 ++++++++- src/execnet/_provision.py | 1 + src/execnet/_trio_host.py | 6 +- src/execnet/_trio_worker.py | 34 +++-- src/execnet/gateway.py | 2 + src/execnet/gateway_base.py | 274 ++++-------------------------------- src/execnet/multi.py | 63 ++++++--- src/execnet/xspec.py | 1 + testing/conftest.py | 7 +- testing/test_basics.py | 19 ++- testing/test_channel.py | 5 +- testing/test_gateway.py | 2 +- testing/test_multi.py | 7 +- testing/test_termination.py | 19 ++- testing/test_threadpool.py | 222 ----------------------------- testing/test_xspec.py | 8 ++ 16 files changed, 213 insertions(+), 527 deletions(-) delete mode 100644 testing/test_threadpool.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index f80f6998..ce211ae1 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -9,13 +9,22 @@ import queue import threading import time +from collections.abc import Callable from typing import Any from typing import Generic from typing import Protocol from typing import TypeVar from typing import cast -__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] +__all__ = [ + "Flag", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + "make_wakener", + "register_wakener", +] T = TypeVar("T") @@ -159,3 +168,62 @@ def wait(self, timeout: float | None = None) -> T: if self._error is not None: raise self._error return cast("T", self._value) + + +class Flag: + """An idempotent event on a wakener: may be set any number of times. + + Each Flag owns its wakener exclusively -- sharing one wakener between + carriers would lose wakeups (another carrier's ``clear`` can swallow + this one's ``notify``). + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._flag = False + + def is_set(self) -> bool: + return self._flag + + def set(self) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._flag = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> bool: + """Block until set; returns whether the flag is set.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._flag: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + break + return self._flag + + +# wait= axis: named wakener factories; each call returns a fresh instance +# (carriers own their wakener exclusively, see Flag). Backends register +# here ("gevent", "asyncio") next to the built-in "thread". +_WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { + "thread": ThreadWakener, +} + + +def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: + """Register a wakener factory for the ``wait=`` spec axis.""" + _WAKENER_FACTORIES[name] = factory + + +def wakener_names() -> list[str]: + return list(_WAKENER_FACTORIES) + + +def make_wakener(name: str) -> Wakener: + """Create a fresh wakener for the named wait backend.""" + try: + factory = _WAKENER_FACTORIES[name] + except KeyError: + raise ValueError(f"unknown wait backend {name!r}") from None + return factory() diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b1c16335..f5114ad8 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -168,6 +168,7 @@ def worker_cli_arg(spec: Any) -> str: { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, + "wait": spec.wait or "thread", "coordinator_version": execnet.__version__, } ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed4f876e..e29af4d2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -155,7 +155,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync: OneShot[None] = OneShot() + self._done_sync: OneShot[None] = OneShot(sync_gateway._new_wakener()) self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -310,7 +310,9 @@ def enqueue_message(self, message: Message) -> None: # The ack carries the write failure as a value (never raised into # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable # from a stream error. - ack: OneShot[BaseException | None] | None = OneShot() if wait else None + ack: OneShot[BaseException | None] | None = ( + OneShot(self.sync_gateway._new_wakener()) if wait else None + ) def post() -> None: try: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6506199d..89aad122 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -201,12 +201,13 @@ def kill(self) -> None: def _build_worker_gateway( - host: _trio_host.TrioHost, id: str, model: ExecModel + host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" ) -> tuple[WorkerGateway, TrioWorkerExec, bool]: """Construct the WorkerGateway + Trio exec pool (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + gateway._wait_backend = wait main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) @@ -215,14 +216,16 @@ def _build_worker_gateway( gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: - gateway._executetask_complete = model.Event() + gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() return gateway, trio_exec, main_thread_only -def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: +def _run_worker( + host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" +) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -252,7 +255,7 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) -def serve_popen_trio(id: str, execmodel: str = "thread") -> None: +def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -266,10 +269,12 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_make_fd_io, read_fd, write_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) -def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: +def serve_socket_trio( + id: str, execmodel: str, socket_fd: int, wait: str = "thread" +) -> None: """Serve a WorkerGateway over an inherited socket fd. Used for the socketserver (an accepted TCP connection) and, in future, a @@ -282,7 +287,7 @@ def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_trio_host.adopt_socket, socket_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) def _rough_version(version: str) -> tuple[int, ...]: @@ -341,9 +346,18 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) if ns.socket_fd is not None: - serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + serve_socket_trio( + config["id"], + config["execmodel"], + ns.socket_fd, + wait=config.get("wait", "thread"), + ) else: - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + serve_popen_trio( + id=config["id"], + execmodel=config["execmodel"], + wait=config.get("wait", "thread"), + ) if __name__ == "__main__": diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index b1d3ed46..bec4c778 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -43,6 +43,8 @@ def __init__(self, io: IO, spec: XSpec) -> None: """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec + if spec.wait: + self._wait_backend = spec.wait @property def remoteaddress(self) -> str: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 47bf7d32..6a286302 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -11,7 +11,6 @@ from __future__ import annotations -import abc import builtins import os import queue as _queue @@ -31,7 +30,10 @@ from typing import cast from typing import overload +from ._boundary import Flag from ._boundary import Mailbox +from ._boundary import Wakener +from ._boundary import make_wakener class WriteIO(Protocol): @@ -58,74 +60,23 @@ def wait(self) -> int | None: ... def kill(self) -> None: ... -class Event(Protocol): - """Protocol for types which look like threading.Event.""" - - def is_set(self) -> bool: ... - - def set(self) -> None: ... - - def clear(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... +class ExecModel: + """Deprecated preset name for an execution model. + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ -class ExecModel(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def backend(self) -> str: - raise NotImplementedError() + def __init__(self, backend: str) -> None: + self.backend = backend def __repr__(self) -> str: return "" % self.backend - @property - @abc.abstractmethod - def queue(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def subprocess(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def socket(self): - raise NotImplementedError() - - @abc.abstractmethod - def start(self, func, args=()) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def get_ident(self) -> int: - raise NotImplementedError() - - @abc.abstractmethod - def sleep(self, delay: float) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def fdopen(self, fd, mode, bufsize=1, closefd=True): - raise NotImplementedError() - - @abc.abstractmethod - def Lock(self): - raise NotImplementedError() - - @abc.abstractmethod - def RLock(self): - raise NotImplementedError() - - @abc.abstractmethod - def Event(self) -> Event: - raise NotImplementedError() - - -class ThreadExecModel(ExecModel): - backend = "thread" - @property def queue(self): import queue @@ -160,200 +111,24 @@ def start(self, func, args=()) -> None: _thread.start_new_thread(func, args) def fdopen(self, fd, mode, bufsize=1, closefd=True): - import os - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) def Lock(self): - import threading - return threading.RLock() def RLock(self): - import threading - return threading.RLock() - def Event(self): - import threading - + def Event(self) -> threading.Event: return threading.Event() -class MainThreadOnlyExecModel(ThreadExecModel): - backend = "main_thread_only" - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend == "thread": - return ThreadExecModel() - elif backend == "main_thread_only": - return MainThreadOnlyExecModel() - else: - raise ValueError(f"unknown execmodel {backend!r}") - - -class Reply: - """Provide access to the result of a function execution that got dispatched - through WorkerPool.spawn().""" - - def __init__(self, task, threadmodel: ExecModel) -> None: - self.task = task - self._result_ready = threadmodel.Event() - self.running = True - - def get(self, timeout: float | None = None): - """get the result object from an asynchronous function execution. - if the function execution raised an exception, - then calling get() will reraise that exception - including its traceback. - """ - self.waitfinish(timeout) - try: - return self._result - except AttributeError: - raise self._exc from None - - def waitfinish(self, timeout: float | None = None) -> None: - if not self._result_ready.wait(timeout): - raise OSError(f"timeout waiting for {self.task!r}") - - def run(self) -> None: - func, args, kwargs = self.task - try: - try: - self._result = func(*args, **kwargs) - except BaseException as exc: - self._exc = exc - finally: - self._result_ready.set() - self.running = False - - -class WorkerPool: - """A WorkerPool allows to spawn function executions - to threads, returning a reply object on which you - can ask for the result (and get exceptions reraised). - - This implementation allows the main thread to integrate - itself into performing function execution through - calling integrate_as_primary_thread() which will return - when the pool received a trigger_shutdown(). - - By default allows unlimited number of spawns. - """ - - _primary_thread_task: Reply | None - - def __init__(self, execmodel: ExecModel, hasprimary: bool = False) -> None: - self.execmodel = execmodel - self._running_lock = self.execmodel.Lock() - self._running: set[Reply] = set() - self._shuttingdown = False - self._waitall_events: list[Event] = [] - if hasprimary: - if self.execmodel.backend not in ("thread", "main_thread_only"): - raise ValueError("hasprimary=True requires thread model") - self._primary_thread_task_ready: Event | None = self.execmodel.Event() - else: - self._primary_thread_task_ready = None - - def integrate_as_primary_thread(self) -> None: - """Integrate the thread with which we are called as a primary - thread for executing functions triggered with spawn().""" - assert self.execmodel.backend in ("thread", "main_thread_only"), self.execmodel - primary_thread_task_ready = self._primary_thread_task_ready - assert primary_thread_task_ready is not None - # interacts with code at REF1 - while 1: - primary_thread_task_ready.wait() - reply = self._primary_thread_task - if reply is None: # trigger_shutdown() woke us up - break - self._perform_spawn(reply) - # we are concurrent with trigger_shutdown and spawn - with self._running_lock: - if self._shuttingdown: - break - # Only clear if _try_send_to_primary_thread has not - # yet set the next self._primary_thread_task reply - # after waiting for this one to complete. - if reply is self._primary_thread_task: - primary_thread_task_ready.clear() - - def trigger_shutdown(self) -> None: - with self._running_lock: - self._shuttingdown = True - if self._primary_thread_task_ready is not None: - self._primary_thread_task = None - self._primary_thread_task_ready.set() - - def active_count(self) -> int: - return len(self._running) - - def _perform_spawn(self, reply: Reply) -> None: - reply.run() - with self._running_lock: - self._running.remove(reply) - if not self._running: - while self._waitall_events: - waitall_event = self._waitall_events.pop() - waitall_event.set() - - def _try_send_to_primary_thread(self, reply: Reply) -> bool: - # REF1 in 'thread' model we give priority to running in main thread - # note that we should be called with _running_lock hold - primary_thread_task_ready = self._primary_thread_task_ready - if primary_thread_task_ready is not None: - if not primary_thread_task_ready.is_set(): - self._primary_thread_task = reply - # wake up primary thread - primary_thread_task_ready.set() - return True - elif ( - self.execmodel.backend == "main_thread_only" - and self._primary_thread_task is not None - ): - self._primary_thread_task.waitfinish() - self._primary_thread_task = reply - # wake up primary thread (it's okay if this is already set - # because we waited for the previous task to finish above - # and integrate_as_primary_thread will not clear it when - # it enters self._running_lock if it detects that a new - # task is available) - primary_thread_task_ready.set() - return True - return False - - def spawn(self, func, *args, **kwargs) -> Reply: - """Asynchronously dispatch func(*args, **kwargs) and return a Reply.""" - reply = Reply((func, args, kwargs), self.execmodel) - with self._running_lock: - if self._shuttingdown: - raise ValueError("pool is shutting down") - self._running.add(reply) - if not self._try_send_to_primary_thread(reply): - self.execmodel.start(self._perform_spawn, (reply,)) - return reply - - def terminate(self, timeout: float | None = None) -> bool: - """Trigger shutdown and wait for completion of all executions.""" - self.trigger_shutdown() - return self.waitall(timeout=timeout) - - def waitall(self, timeout: float | None = None) -> bool: - """Wait until all active spawns have finished executing.""" - with self._running_lock: - if not self._running: - return True - # if a Reply still runs, we let run_and_release - # signal us -- note that we are still holding the - # _running_lock to avoid race conditions - my_waitall_event = self.execmodel.Event() - self._waitall_events.append(my_waitall_event) - return my_waitall_event.wait(timeout=timeout) + if backend in ("thread", "main_thread_only"): + return ExecModel(backend) + raise ValueError(f"unknown execmodel {backend!r}") sysex = (KeyboardInterrupt, SystemExit) @@ -578,11 +353,11 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a callback is set - self._mailbox: Mailbox[Any] | None = Mailbox() + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) self._callback: Callable[[Any], Any] | None = None self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = threading.Event() + self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -1028,6 +803,9 @@ class BaseGateway: _trio_session: Any = None # Set by the receiver on EOF without a prior termination message. _error: BaseException | None = None + #: wait= axis: which wakener backend this gateway's blocking waits + #: park on (channels, write-acks, join) + _wait_backend: str = "thread" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1051,6 +829,10 @@ def _attach_trio_session(self, session: Any) -> None: for channel in self._channelfactory.channels(): session.bind_sync_channel(channel) + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier (wait= axis).""" + return make_wakener(self._wait_backend) + def _bind_channel(self, channel: Channel) -> None: """Divert the session's inbound routing for ``channel.id`` to it.""" session = self._trio_session @@ -1134,7 +916,7 @@ class WorkerGateway(BaseGateway): _trio_exec: Any = None # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). _execpool: Any = None - _executetask_complete: Event | None = None + _executetask_complete: threading.Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: trio_exec = self._trio_exec diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ca3b69dd..ba73bf62 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -7,6 +7,9 @@ from __future__ import annotations import atexit +import queue +import threading +import time import types from collections.abc import Callable from collections.abc import Iterable @@ -20,9 +23,9 @@ from typing import TypeAlias from typing import overload +from ._boundary import wakener_names from .gateway_base import Channel from .gateway_base import ExecModel -from .gateway_base import WorkerPool from .gateway_base import get_execmodel from .gateway_base import trace from .xspec import XSpec @@ -152,6 +155,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute execmodel=model 'thread' or 'main_thread_only' execution model + wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. @@ -165,6 +169,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + if spec.wait is not None and spec.wait not in wakener_names(): + raise ValueError( + f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" + ) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -313,10 +321,10 @@ def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): try: return self._queue # type: ignore[has-type] except AttributeError: - self._queue = None + self._queue: queue.Queue[tuple[Channel, Any]] | None = None for ch in self._channels: if self._queue is None: - self._queue = ch.gateway.execmodel.queue.Queue() + self._queue = queue.Queue() def putreceived(obj, channel: Channel = ch) -> None: self._queue.put((channel, obj)) # type: ignore[union-attr] @@ -351,32 +359,45 @@ def safe_terminate( """Run terminate/kill pairs in parallel with a hard wait bound. Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - Waiting for the worker pool is also bounded so a stuck kill cannot hang - the caller forever (see issues #43 / #221). + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). """ - workerpool = WorkerPool(execmodel) + errors: list[BaseException] = [] def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - termreply = workerpool.spawn(termfunc) - try: - termreply.get(timeout=timeout) - except OSError: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): killfunc() + return + if term_errors: + errors.append(term_errors[0]) - replylist = [ - workerpool.spawn(termkill, termfunc, killfunc) - for termfunc, killfunc in list_of_paired_functions + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions ] + for thread in threads: + thread.start() # Allow term timeout plus a kill attempt; never block indefinitely. wait_timeout = None if timeout is None else timeout * 2 - for reply in replylist: - try: - reply.waitfinish(timeout=wait_timeout) - except OSError: - # termkill still running (typically stuck in killfunc). - continue - reply.get() # propagate worker exceptions, if any - workerpool.waitall(timeout=wait_timeout) + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] default_group = Group() diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index 0559ed8c..afc58f4c 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -29,6 +29,7 @@ class XSpec: ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None + wait: str | None = None def __init__(self, string: str) -> None: self._spec = string diff --git a/testing/conftest.py b/testing/conftest.py index 4d97bf35..7c3ed7a6 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -5,6 +5,7 @@ from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import pytest @@ -12,7 +13,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool from execnet.gateway_base import get_execmodel collect_ignore = ["build", "doc/_build"] @@ -184,5 +184,6 @@ def execmodel(request: pytest.FixtureRequest) -> ExecModel: @pytest.fixture -def pool(execmodel: ExecModel) -> WorkerPool: - return WorkerPool(execmodel=execmodel) +def executor() -> Iterator[ThreadPoolExecutor]: + with ThreadPoolExecutor() as tpe: + yield tpe diff --git a/testing/test_basics.py b/testing/test_basics.py index 29ceef19..c58e4728 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -71,15 +71,23 @@ def standalone_gateway_base_source() -> str: """gateway_base's source as a self-contained script. The module imports the trio-free boundary kit relatively; for the - run-on-any-python checks the kit source is inlined at the import site - (minus its own future import, which must stay file-leading). + run-on-any-python checks the kit source is inlined in place of those + imports (minus its own future import, which must stay file-leading). """ boundary_source = inspect.getsource(_boundary).replace( "from __future__ import annotations\n", "" ) - return inspect.getsource(gateway_base).replace( - "from ._boundary import Mailbox\n", boundary_source - ) + lines = [] + inlined = False + for line in inspect.getsource(gateway_base).splitlines(keepends=True): + if line.startswith("from ._boundary import"): + if not inlined: + inlined = True + lines.append(boundary_source) + else: + lines.append(line) + assert inlined + return "".join(lines) IO_MESSAGE_EXTRA_SOURCE = """ @@ -341,6 +349,7 @@ class TestPureChannel: def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: _trio_session = None + _new_wakener = staticmethod(_boundary.ThreadWakener) def _trace(self, *args) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index ba02b57b..f06b8632 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -4,6 +4,7 @@ from __future__ import annotations +import queue import time import pytest @@ -270,14 +271,14 @@ def test_channel_endmarker_callback(self, gw: Gateway) -> None: assert l[3] == 999 def test_channel_endmarker_callback_error(self, gw: Gateway) -> None: - q = gw.execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" raise ValueError() """ ) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert err diff --git a/testing/test_gateway.py b/testing/test_gateway.py index b7098773..da519d4e 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -599,7 +599,7 @@ def test_main_thread_only_concurrent_remote_exec_deadlock( import threading channel.send(threading.current_thread() is threading.main_thread()) # Wait forever, ensuring that the deadlock case triggers. - channel.gateway.execmodel.Event().wait() + threading.Event().wait() """ ) ) diff --git a/testing/test_multi.py b/testing/test_multi.py index e0390298..2f1b634e 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -5,6 +5,7 @@ from __future__ import annotations import gc +import threading from collections.abc import Callable from time import sleep @@ -308,12 +309,12 @@ def test_safe_terminate_does_not_hang_when_kill_blocks( out, so a blocking killfunc made Group.terminate() hang indefinitely (seen via pytest-xdist teardown). """ - kill_started = execmodel.Event() - release_kill = execmodel.Event() + kill_started = threading.Event() + release_kill = threading.Event() other_killed: list[int] = [] def term_slow() -> None: - execmodel.sleep(10) + sleep(10) def kill_hang() -> None: kill_started.set() diff --git a/testing/test_termination.py b/testing/test_termination.py index ca119304..01c57427 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -1,10 +1,12 @@ import os import pathlib +import queue import shutil import signal import subprocess import sys from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor import pytest from test_gateway import TESTTIMEOUT @@ -12,7 +14,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool execnetdir = pathlib.Path(execnet.__file__).parent.parent @@ -23,7 +24,7 @@ def test_exit_blocked_worker_execution_gateway( - anypython: str, makegateway: Callable[[str], Gateway], pool: WorkerPool + anypython: str, makegateway: Callable[[str], Gateway], executor: ThreadPoolExecutor ) -> None: gateway = makegateway("popen//python=%s" % anypython) gateway.remote_exec( @@ -37,8 +38,7 @@ def doit() -> int: gateway.exit() return 17 - reply = pool.spawn(doit) - x = reply.get(timeout=5.0) + x = executor.submit(doit).result(timeout=5.0) assert x == 17 @@ -48,7 +48,7 @@ def test_endmarker_delivery_on_remote_killterm( if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("test and execnet not compatible to greenlets yet") gw = makegateway("popen") - q = execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" import os, time @@ -60,7 +60,7 @@ def test_endmarker_delivery_on_remote_killterm( assert isinstance(pid, int) os.kill(pid, signal.SIGTERM) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert isinstance(err, EOFError) @@ -113,10 +113,8 @@ def test_terminate_implicit_does_trykill( pytester: pytest.Pytester, anypython: str, capfd: pytest.CaptureFixture[str], - pool: WorkerPool, + executor: ThreadPoolExecutor, ) -> None: - if pool.execmodel.backend not in ("thread", "main_thread_only"): - pytest.xfail("only os threading model supported") if sys.version_info >= (3, 12): pytest.xfail( "since python3.12 this test triggers RuntimeError: can't create new thread at interpreter shutdown" @@ -149,8 +147,7 @@ def flush(self): # sync with start-up assert popen.stdout is not None popen.stdout.readline() - reply = pool.spawn(popen.communicate) - reply.get(timeout=50) + executor.submit(popen.communicate).result(timeout=50) _out, err = capfd.readouterr() lines = [x for x in err.splitlines() if "*sys-package" not in x] assert not lines or "Killed" in err diff --git a/testing/test_threadpool.py b/testing/test_threadpool.py deleted file mode 100644 index 510ae020..00000000 --- a/testing/test_threadpool.py +++ /dev/null @@ -1,222 +0,0 @@ -import os -from pathlib import Path - -import pytest - -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool - - -def test_execmodel(execmodel: ExecModel, tmp_path: Path) -> None: - assert execmodel.backend - p = tmp_path / "somefile" - p.write_text("content") - fd = os.open(p, os.O_RDONLY) - f = execmodel.fdopen(fd, "r") - assert f.read() == "content" - f.close() - - -def test_execmodel_basic_attrs(execmodel: ExecModel) -> None: - m = execmodel - assert callable(m.start) - assert m.get_ident() - - -def test_simple(pool: WorkerPool) -> None: - reply = pool.spawn(lambda: 42) - assert reply.get() == 42 - - -def test_some(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - num = 4 - - def f(i: int) -> None: - q.put(i) - while q.qsize(): - execmodel.sleep(0.01) - - for i in range(num): - pool.spawn(f, i) - for i in range(num): - q.get() - # assert len(pool._running) == 4 - assert pool.waitall(timeout=1.0) - # execmodel.sleep(1) helps on windows? - assert len(pool._running) == 0 - - -def test_running_semnatics(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def first() -> None: - q.get() - - reply = pool.spawn(first) - assert reply.running - assert pool.active_count() == 1 - q.put(1) - assert pool.waitall() - assert pool.active_count() == 0 - assert not reply.running - - -def test_waitfinish_on_reply(pool: WorkerPool) -> None: - l = [] - reply = pool.spawn(lambda: l.append(1)) - reply.waitfinish() - assert l == [1] - reply = pool.spawn(lambda: 0 / 0) - reply.waitfinish() # no exception raised - pytest.raises(ZeroDivisionError, reply.get) - - -@pytest.mark.xfail(reason="WorkerPool does not implement limited size") -def test_limited_size(execmodel: ExecModel) -> None: - pool = WorkerPool(execmodel, size=1) # type: ignore[call-arg] - q = execmodel.queue.Queue() - q2 = execmodel.queue.Queue() - q3 = execmodel.queue.Queue() - - def first() -> None: - q.put(1) - q2.get() - - pool.spawn(first) - assert q.get() == 1 - - def second() -> None: - q3.put(3) - - # we spawn a second pool to spawn the second function - # which should block - pool2 = WorkerPool(execmodel) - pool2.spawn(pool.spawn, second) - assert not pool2.waitall(1.0) - assert q3.qsize() == 0 - q2.put(2) - assert pool2.waitall() - assert pool.waitall() - - -def test_get(pool: WorkerPool) -> None: - def f() -> int: - return 42 - - reply = pool.spawn(f) - result = reply.get() - assert result == 42 - - -def test_get_timeout(execmodel: ExecModel, pool: WorkerPool) -> None: - def f() -> int: - execmodel.sleep(0.2) - return 42 - - reply = pool.spawn(f) - with pytest.raises(IOError): - reply.get(timeout=0.01) - - -def test_get_excinfo(pool: WorkerPool) -> None: - def f() -> None: - raise ValueError("42") - - reply = pool.spawn(f) - with pytest.raises(ValueError): - reply.get(1.0) - with pytest.raises(ValueError): - reply.get(1.0) - - -def test_waitall_timeout(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def f() -> None: - q.get() - - reply = pool.spawn(f) - assert not pool.waitall(0.01) - q.put(None) - reply.get(timeout=1.0) - assert pool.waitall(timeout=0.1) - - -@pytest.mark.skipif(not hasattr(os, "dup"), reason="no os.dup") -def test_pool_clean_shutdown( - pool: WorkerPool, capfd: pytest.CaptureFixture[str] -) -> None: - q = pool.execmodel.queue.Queue() - - def f() -> None: - q.get() - - pool.spawn(f) - assert not pool.waitall(timeout=1.0) - pool.trigger_shutdown() - with pytest.raises(ValueError): - pool.spawn(f) - - def wait_then_put() -> None: - pool.execmodel.sleep(0.1) - q.put(1) - - pool.execmodel.start(wait_then_put) - assert pool.waitall() - _out, err = capfd.readouterr() - assert err == "" - - -def test_primary_thread_integration(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - with pytest.raises(ValueError): - WorkerPool(execmodel=execmodel, hasprimary=True) - return - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - - def func() -> None: - queue.put(execmodel.get_ident()) - - pool.spawn(func) - ident1 = queue.get() - ident2 = queue.get() - assert ident1 == ident2 - pool.terminate() - - -def test_primary_thread_integration_shutdown(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - pytest.skip("can only run with threading") - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - queue.get() - - queue2 = execmodel.queue.Queue() - - def get_two() -> None: - queue.put(execmodel.get_ident()) - queue2.get() - - reply = pool.spawn(get_two) - # make sure get_two is running and blocked on queue2 - queue.get() - # then shut down - pool.trigger_shutdown() - # and let get_two finish - queue2.put(1) - reply.get() - assert pool.waitall(5.0) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 661e50af..850acbe1 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -119,6 +119,14 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) + def test_wait_axis(self, makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown wait backend"): + makegateway("popen//wait=nope") + gw = makegateway("popen//wait=thread") + assert gw._wait_backend == "thread" + channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") + assert channel.receive() == "thread" + @skip_win_pypy def test_popen_default(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("") From 845510e64a8e9e8542cae7234f6197ec812527f7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:19:08 +0200 Subject: [PATCH 38/66] feat: add execnet.aio -- the asyncio-native API over the Trio host P4 of the boundary rethink. A fourth public namespace mirroring execnet.trio's surface for asyncio applications: Group/Gateway/Channel are async context managers / awaitables whose operations run as tasks on a dedicated Trio host thread and resolve asyncio futures via loop.call_soon_threadsafe (a _HostBridge per group). All transports work because the protocol engine is the same AsyncGroup; no anyio port and no per-call executor threads. Received channel references arrive wrapped as aio Channels. Known v1 limitation (documented): cancelling a bridged await abandons the operation asyncio-side only; the host-side task runs to completion. `import execnet` stays free of trio/asyncio -- execnet.aio loads lazily like execnet.trio. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 8 +- src/execnet/aio.py | 321 ++++++++++++++++++++++++++++++++++++++++ testing/test_aio.py | 116 +++++++++++++++ 3 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 src/execnet/aio.py create mode 100644 testing/test_aio.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 2108f4b2..66c61821 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,14 +4,16 @@ pure python lib for connecting to local and remote Python Interpreters. -Three public namespaces: +Four public namespaces: * :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` names below are aliases into it. * :mod:`execnet.trio` — the trio-native API, awaited inside your own ``trio.run``. +* :mod:`execnet.aio` — the asyncio-native API, bridged over a Trio + host thread. * :mod:`execnet.portal` — cross-thread / cross-loop communication - primitives shared by both. + primitives shared by all of them. (c) 2012, Holger Krekel and others """ @@ -66,7 +68,7 @@ def __getattr__(name: str) -> Any: # Lazy namespace modules: keep ``import execnet`` from loading the # trio event loop machinery until it is actually used. - if name in ("trio", "portal"): + if name in ("aio", "portal", "trio"): import importlib return importlib.import_module(f".{name}", __name__) diff --git a/src/execnet/aio.py b/src/execnet/aio.py new file mode 100644 index 00000000..e3004a28 --- /dev/null +++ b/src/execnet/aio.py @@ -0,0 +1,321 @@ +"""The asyncio-native execnet API, bridged over a Trio host thread. + +Everything here is awaited inside your own asyncio event loop:: + + import asyncio + import execnet.aio + + async def main(): + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + asyncio.run(main()) + +Protocol IO keeps running on a dedicated Trio host thread (the same +engine as the blocking and trio-native APIs, all transports included); +each awaited operation runs as a task on that host and resolves an +asyncio future via ``loop.call_soon_threadsafe``. No anyio port and no +executor threads per call. + +Note: cancelling a bridged await abandons the operation on the asyncio +side only -- the host-side task runs to completion (a cancelled +``receive`` may still consume the next item). + +The serialization helpers and error types are shared with +:mod:`execnet.sync` and :mod:`execnet.trio`. +""" + +from __future__ import annotations + +import asyncio +import functools +import types +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +import trio + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_host import TrioHost +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +if TYPE_CHECKING: + from typing_extensions import Self + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", +] + +T = TypeVar("T") + + +class _HostBridge: + """Await trio-native coroutines on a TrioHost from asyncio.""" + + def __init__(self, host: TrioHost) -> None: + self._host = host + + async def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + loop = asyncio.get_running_loop() + future: asyncio.Future[T] = loop.create_future() + + def resolve(result: Any, error: BaseException | None) -> None: + if future.cancelled(): + return + if error is not None: + future.set_exception(error) + else: + future.set_result(result) + + def post_result(result: Any, error: BaseException | None) -> None: + # The asyncio loop may already be gone at interpreter/test + # teardown; the result is undeliverable then. + with suppress(RuntimeError): + loop.call_soon_threadsafe(resolve, result, error) + + async def runner() -> None: + try: + result = await async_fn(*args) + except trio.Cancelled: + # host shutdown: the nursery cancel must propagate + post_result(None, RuntimeError("execnet aio host was shut down")) + raise + except BaseException as exc: + post_result(None, exc) + else: + post_result(result, None) + + def spawn() -> None: + self._host.start_soon(runner) + + try: + self._host.portal.post(spawn) + except trio.RunFinishedError: + raise RuntimeError("execnet aio group is not running") from None + return await future + + +class _HostedGroup(AsyncGroup): + """Trio AsyncGroup living as a task on the host nursery.""" + + def __init__(self, termination_timeout: float) -> None: + super().__init__(termination_timeout) + self.shutdown = trio.Event() + self.finished = trio.Event() + + async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: + try: + async with self: + task_status.started(self) + await self.shutdown.wait() + finally: + self.finished.set() + + +class Channel: + """asyncio facade over a trio-native :class:`AsyncChannel`.""" + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, bridge: _HostBridge, channel: AsyncChannel) -> None: + self._bridge = bridge + self._channel = channel + + @property + def id(self) -> int: + return self._channel.id + + def __repr__(self) -> str: + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._channel.isclosed() + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side.""" + await self._bridge.call(self._channel.send, item) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + EOFError once the peer closed or sent EOF, RemoteError for a peer + close-with-error, TimeoutError after ``timeout`` seconds. A + received channel reference arrives as an :class:`~execnet.aio.Channel`. + """ + result = await self._bridge.call(self._channel.receive, timeout) + if isinstance(result, AsyncChannel): + return Channel(self._bridge, result) + return result + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._bridge.call(self._channel.send_eof) + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._bridge.call(self._channel.aclose, error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._bridge.call(self._channel.wait_closed) + + def __aiter__(self) -> Channel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + +class Gateway: + """asyncio facade over a trio-native :class:`AsyncGateway`.""" + + def __init__(self, bridge: _HostBridge, gateway: AsyncGateway) -> None: + self._bridge = bridge + self._gateway = gateway + + @property + def id(self) -> str: + return self._gateway.id + + @property + def remoteaddress(self) -> str | None: + return self._gateway.remoteaddress + + def __repr__(self) -> str: + return f"" + + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as ``Gateway.remote_exec``: a source + string, a pure function called with ``channel`` and ``**kwargs``, + or a module. + """ + channel = await self._bridge.call( + functools.partial(self._gateway.remote_exec, source, **kwargs) + ) + return Channel(self._bridge, channel) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + await self._bridge.call(self._gateway.terminate) + + +class Group: + """asyncio-native gateway group over a dedicated Trio host thread. + + An async context manager mirroring :class:`execnet.trio.AsyncGroup`; + leaving the ``async with`` block terminates every gateway with the + same bounded contract, then stops the host thread. + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._host: TrioHost | None = None + self._bridge: _HostBridge | None = None + self._group: _HostedGroup | None = None + + def __repr__(self) -> str: + state = "running" if self._group is not None else "idle" + return f"" + + async def __aenter__(self) -> Self: + assert self._host is None, "group already entered" + loop = asyncio.get_running_loop() + host = TrioHost(name="execnet-aio-group") + # start() blocks until the host loop is ready; keep it off the + # asyncio loop thread. + await loop.run_in_executor(None, host.start) + self._host = host + self._bridge = _HostBridge(host) + + async def start_group() -> _HostedGroup: + # runs on the host loop + group = _HostedGroup(self._termination_timeout) + started: _HostedGroup = await host._nursery.start(group.run) + return started + + self._group = await self._bridge.call(start_group) + return self + + async def __aexit__(self, *exc_info: object) -> None: + group, bridge, host = self._group, self._bridge, self._host + self._group = self._bridge = None + if group is not None and bridge is not None: + + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() + + with suppress(RuntimeError): + await bridge.call(stop_group) + if host is not None: + self._host = None + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, functools.partial(host.stop, 5.0)) + + async def makegateway(self, spec: str | XSpec = "popen") -> Gateway: + """Create a gateway for ``spec`` served on the group's host. + + All transports are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways. + """ + group, bridge = self._group, self._bridge + if group is None or bridge is None: + raise RuntimeError(f"{self!r} is not entered") + gateway = await bridge.call(group.makegateway, spec) + return Gateway(bridge, gateway) + + +@asynccontextmanager +async def open_popen_gateway(spec: str | XSpec = "popen") -> AsyncIterator[Gateway]: + """Spawn one popen worker and yield an asyncio Gateway to it. + + Convenience for a single-gateway :class:`Group`. + """ + async with Group() as group: + yield await group.makegateway(spec) diff --git a/testing/test_aio.py b/testing/test_aio.py new file mode 100644 index 00000000..6bdb7724 --- /dev/null +++ b/testing/test_aio.py @@ -0,0 +1,116 @@ +"""The asyncio-native API: execnet.aio bridged over the Trio host.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any +from typing import TypeVar + +import pytest + +import execnet.aio + +T = TypeVar("T") + +TESTTIMEOUT = 30.0 + + +def run(main: Awaitable[T]) -> T: + return asyncio.run(asyncio.wait_for(main, TESTTIMEOUT)) + + +def test_popen_roundtrip() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + run(main()) + + +def test_open_popen_gateway_iteration() -> None: + async def main() -> list[int]: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(4): channel.send(i * 2)" + ) + return [item async for item in channel] + + assert run(main()) == [0, 2, 4, 6] + + +def test_receive_timeout() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("channel.receive()") + with pytest.raises(channel.TimeoutError): + await channel.receive(timeout=0.05) + await channel.send(None) + + run(main()) + + +def test_remote_error() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError(17)") + with pytest.raises(execnet.aio.RemoteError, match="ValueError"): + await channel.receive() + + run(main()) + + +def test_channel_passing_wraps_aio() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + """ + c = channel.gateway.newchannel() + channel.send(c) + c.send(42) + """ + ) + passed = await channel.receive() + assert isinstance(passed, execnet.aio.Channel) + assert await passed.receive() == 42 + + run(main()) + + +def test_multiple_gateways_and_send_each() -> None: + async def main() -> list[Any]: + async with execnet.aio.Group() as group: + gateways = [await group.makegateway("popen") for _ in range(2)] + channels = [ + await gw.remote_exec("channel.send(channel.receive() * 2)") + for gw in gateways + ] + for i, channel in enumerate(channels): + await channel.send(i + 1) + return [await channel.receive() for channel in channels] + + assert run(main()) == [2, 4] + + +def test_group_not_entered() -> None: + async def main() -> None: + group = execnet.aio.Group() + with pytest.raises(RuntimeError, match="not entered"): + await group.makegateway("popen") + + run(main()) + + +def test_terminate_gateway_explicitly() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(1)") + assert await channel.receive() == 1 + await gateway.terminate() + + run(main()) From b01d00937d0428794b736bb06c42b714ec852bd1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:22:37 +0200 Subject: [PATCH 39/66] feat: gevent wait backend (wait=gevent) with opt-in tests P5, closing out the boundary rethink. execnet._gevent_support provides a GeventWakener whose notify() crosses from the trio host thread via a hub loop.async_ watcher (the one gevent primitive documented as thread-safe) while waits park only the calling greenlet. The hub-bound event/watcher are created lazily in the first waiter's hub so carriers can be constructed on the host loop; a pre-watcher notify is replayed, never lost. The boundary kit resolves wait=gevent by lazily importing the support module, so specs validate without gevent installed. Tests live behind the new `gevent` dependency group (pytest importorskip): wakener unit coverage plus gateway-level proof that channel.receive()/send() park the greenlet, not the hub -- no monkey-patching required. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 + src/execnet/_boundary.py | 12 +- src/execnet/_gevent_support.py | 76 ++++++++++++ src/execnet/aio.py | 1 + testing/test_gevent.py | 86 ++++++++++++++ uv.lock | 208 ++++++++++++++++++++++++++++++++- 6 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 src/execnet/_gevent_support.py create mode 100644 testing/test_gevent.py diff --git a/pyproject.toml b/pyproject.toml index 4e94c938..aedd6b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ testing = [ dev = [ "pytest-xdist>=3.8.0", ] +gevent = [ + "gevent>=26.7.0", +] testing = [ "execnet[testing]", ] diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index ce211ae1..9c7c69f4 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -205,10 +205,14 @@ def wait(self, timeout: float | None = None) -> bool: # wait= axis: named wakener factories; each call returns a fresh instance # (carriers own their wakener exclusively, see Flag). Backends register -# here ("gevent", "asyncio") next to the built-in "thread". +# here next to the built-in "thread"; entries in the lazy table import +# their module (which registers itself) on first use. _WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { "thread": ThreadWakener, } +_LAZY_WAKENER_MODULES: dict[str, str] = { + "gevent": "execnet._gevent_support", +} def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: @@ -217,11 +221,15 @@ def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: def wakener_names() -> list[str]: - return list(_WAKENER_FACTORIES) + return sorted(set(_WAKENER_FACTORIES) | set(_LAZY_WAKENER_MODULES)) def make_wakener(name: str) -> Wakener: """Create a fresh wakener for the named wait backend.""" + if name not in _WAKENER_FACTORIES and name in _LAZY_WAKENER_MODULES: + import importlib + + importlib.import_module(_LAZY_WAKENER_MODULES[name]) try: factory = _WAKENER_FACTORIES[name] except KeyError: diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py new file mode 100644 index 00000000..cd761571 --- /dev/null +++ b/src/execnet/_gevent_support.py @@ -0,0 +1,76 @@ +"""The gevent wait backend (``wait=gevent``). + +Importing this module registers the ``gevent`` wakener factory; the +boundary kit imports it lazily when a spec asks for ``wait=gevent``. + +A blocking wait then parks only the calling greenlet: the carrier's +event lives in the waiting greenlet's hub, and the loop side's +``notify()`` crosses threads through a ``loop.async_`` watcher -- the +one libev/libuv primitive gevent documents as safe to use from other +threads. +""" + +from __future__ import annotations + +import threading +from typing import Any + +import gevent.event +from gevent.hub import get_hub + +from ._boundary import register_wakener + + +class GeventWakener: + """Wakener parking greenlets instead of OS threads. + + ``notify()`` is thread-safe and non-blocking (called from the trio + host thread). The hub-bound pieces (event + async watcher) are + created lazily in the first waiter's hub, so construction is safe + from any thread -- including the host loop, which creates channels + for inbound ids. Waiters must share one hub (one gevent thread per + carrier), which is the ordinary gevent setup. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._notified = False + self._event: gevent.event.Event | None = None + self._watcher: Any = None + + def notify(self) -> None: + with self._lock: + self._notified = True + watcher = self._watcher + if watcher is not None: + watcher.send() + + def _ensure(self) -> gevent.event.Event: + """Create the hub-bound event/watcher in the waiter's context.""" + event = self._event + if event is None: + event = gevent.event.Event() + watcher = get_hub().loop.async_() + watcher.start(event.set) + with self._lock: + self._event = event + self._watcher = watcher + return event + + def wait(self, timeout: float | None = None) -> bool: + event = self._ensure() + with self._lock: + # a notify may have fired before the watcher existed + if self._notified and not event.is_set(): + event.set() + return event.wait(timeout) + + def clear(self) -> None: + with self._lock: + self._notified = False + event = self._event + if event is not None: + event.clear() + + +register_wakener("gevent", GeventWakener) diff --git a/src/execnet/aio.py b/src/execnet/aio.py index e3004a28..bbfe06e3 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -275,6 +275,7 @@ async def __aenter__(self) -> Self: async def start_group() -> _HostedGroup: # runs on the host loop group = _HostedGroup(self._termination_timeout) + assert host._nursery is not None started: _HostedGroup = await host._nursery.start(group.run) return started diff --git a/testing/test_gevent.py b/testing/test_gevent.py new file mode 100644 index 00000000..e95edfc7 --- /dev/null +++ b/testing/test_gevent.py @@ -0,0 +1,86 @@ +"""The gevent wait backend (wait=gevent): greenlet-parking blocking waits. + +Opt-in: requires the ``gevent`` dependency group (``uv sync --group +gevent``); skipped when gevent is not installed. No monkey-patching is +needed -- the wakener parks the waiting greenlet while the trio host +thread keeps running the protocol. +""" + +from __future__ import annotations + +import threading + +import pytest + +gevent = pytest.importorskip("gevent") + +import execnet # noqa: E402 +from execnet._boundary import Flag # noqa: E402 +from execnet._boundary import Mailbox # noqa: E402 +from execnet._boundary import make_wakener # noqa: E402 + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def gevent_gw(): + group = execnet.Group() + try: + yield group.makegateway("popen//wait=gevent") + finally: + group.terminate(timeout=5.0) + + +class TestGeventWakener: + def test_mailbox_wakes_greenlet_from_foreign_thread(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + threading.Timer(0.05, box.put, args=["item"]).start() + result = gevent.spawn(box.get, 5.0) + assert result.get(timeout=TESTTIMEOUT) == "item" + + def test_notify_before_first_wait_is_not_lost(self) -> None: + flag = Flag(make_wakener("gevent")) + flag.set() + assert flag.wait(timeout=1.0) + + def test_wait_parks_greenlet_not_hub(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + box.put("done") + + waiter = gevent.spawn(box.get, 5.0) + gevent.spawn(other) + # if get() blocked the hub, other() could never run and put() + assert waiter.get(timeout=TESTTIMEOUT) == "done" + assert progressed == [0, 1, 2, 3, 4] + + +class TestGeventGateway: + def test_receive_parks_greenlet_not_hub(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(channel.receive())") + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + # sending from a greenlet blocks-until-written on the + # gevent wakener, parking only this greenlet + channel.send("hello") + + waiter = gevent.spawn(channel.receive, TESTTIMEOUT) + gevent.spawn(other) + # if receive() blocked the hub, other() could never send and + # the remote echo could never arrive -> this would hang + assert waiter.get(timeout=TESTTIMEOUT) == "hello" + assert progressed == [0, 1, 2, 3, 4] + + def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(1)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 + gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) diff --git a/uv.lock b/uv.lock index 15185f2a..b885a4b7 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -401,6 +401,9 @@ testing = [ dev = [ { name = "pytest-xdist" }, ] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -420,6 +423,7 @@ provides-extras = ["testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] +gevent = [{ name = "gevent", specifier = ">=26.7.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -431,6 +435,151 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] +[[package]] +name = "gevent" +version = "26.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event" }, + { name = "zope-interface" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/60/878d0cdef05d952ac7f17ffe385143fb0f3720afce0f6ff5ddbf7aac0342/gevent-26.7.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:80e98fc808bd9cc5c911d78a443d214bf0c8f96c9fdd296893df7e40364d5f37", size = 2198781, upload-time = "2026-07-22T16:48:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/69/79/6ce781b60049060e9d89b3d0fe60940353adeb39856aaad5ee925fd127e9/gevent-26.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf4b946b47cc6fdbdf9221f891db9a44df92166435c027760ee7dbdfb4039adc", size = 2229318, upload-time = "2026-07-22T17:02:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/48898f35c2092d699b755b01918551358db72b554c63f553c8f027d2bf31/gevent-26.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:55ce0b7f87f9befcc788d77eb039b1de89a35f37afc31942e12c7ae090a563b8", size = 1700480, upload-time = "2026-07-22T16:26:16.794Z" }, + { url = "https://files.pythonhosted.org/packages/93/51/53370896942523c333699394ccad379d186648dbfb913f42ce094ddfb4b3/gevent-26.7.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f7143823ef99bc657534a2b6e8cbadedc910750cc0b4f4b4438a58d9fe43ab2", size = 1783048, upload-time = "2026-07-22T18:11:25.996Z" }, + { url = "https://files.pythonhosted.org/packages/ba/56/5a2cb36d75d3b626d6ffa116673b34442bf5afd6f7ab4b98e512c1b008d6/gevent-26.7.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:ca4019899830471910129968251c795c8aee59e225fd16326ae01c1f93f3cfa6", size = 1880257, upload-time = "2026-07-22T18:10:40.919Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ed/ee7eb2f03a38a4f33b0f327bab16d3158b3a794588a84dba739cbf4a3e68/gevent-26.7.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e4042da317a96d12110831cc404855f0c501a5a5aa476a7a18c3b480a5a59233", size = 1819378, upload-time = "2026-07-22T18:29:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/e2a202c03cff4f49bba54cc321c3afc29eee9d6fc452f4aa94d2f00e7d3d/gevent-26.7.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5c97ca98e1aae427a267eae0fbfe8d0884327e6b1cd51fc2ef6642b8b0b82701", size = 2136837, upload-time = "2026-07-22T16:48:29.939Z" }, + { url = "https://files.pythonhosted.org/packages/ac/64/4892fbca47aa4e06b86aef96d6146bd61175b23b7db431ec7937d73b2f72/gevent-26.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d5d1864bc3db92d1f82d1790395eda99f98b47fd9f7ec02c4e182d7828a8251", size = 1794058, upload-time = "2026-07-22T18:07:10.644Z" }, + { url = "https://files.pythonhosted.org/packages/21/23/90bb7d0c6f59d2973bb8f4bd3164be00e466823dea01568e0cb36f2afb77/gevent-26.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15fd2d88ed5370f8084079758758df91f26d2f68575e1ee76fce604ddba83e5e", size = 2159797, upload-time = "2026-07-22T17:02:13.229Z" }, + { url = "https://files.pythonhosted.org/packages/a3/67/4d1e315ee3052530fa8537e0cdf1f9c3a6606740f7372f420c7d12d5cf82/gevent-26.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:514bda3fff741d7e5ab108ee1d31550a7f4b2fd3dc6e3b6f38dfb8685efdafaa", size = 1682414, upload-time = "2026-07-22T16:26:22.18Z" }, + { url = "https://files.pythonhosted.org/packages/02/b1/d1b1de89677ee39e641ad8501ed72c2b99f1ddba1c14c462675f40b37a66/gevent-26.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:0f26f9a8c32ac0a73f6084c59b63deeacb350e7f1fee5301d95c5e0683a390d4", size = 1562794, upload-time = "2026-07-22T16:27:53.205Z" }, + { url = "https://files.pythonhosted.org/packages/2b/66/104590ad3a9e671b3ef77ad19c1cc50e7f1c8c220b27ddfaf34e5f88bd9c/gevent-26.7.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:92f256285fb43a57f152bd2e51a59cde1cd0b20869ae1e6da583b6beab88ed8a", size = 2953977, upload-time = "2026-07-22T16:23:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/350d87161378633714184828bdc57c66f9b525eca5249ad1294dc7f8cf58/gevent-26.7.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0e4fea187c5df7168b9538b4f543fcb0fcbaeb93be3d6cd499c324652c740704", size = 1800960, upload-time = "2026-07-22T18:11:27.242Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6c/ddfe298c2ecb1cfc72a03dd69ba751759f7e7835d3135f171b284eb09ed1/gevent-26.7.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:ce732fe08d0ea65de07eff6e46bade8ac6a6fdb65cc748c713f3d31ae122529e", size = 1900387, upload-time = "2026-07-22T18:10:42.973Z" }, + { url = "https://files.pythonhosted.org/packages/f9/89/2647bbbf1da35a1c271a54452e76065308cbaf684ee32a79f989ca4265f6/gevent-26.7.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:c25b3522072137aecf3389031039230190038f888e257f490b3897d0e0620f74", size = 1848046, upload-time = "2026-07-22T18:29:08.074Z" }, + { url = "https://files.pythonhosted.org/packages/31/52/4f9b4c536b5a0424e328d5a5466640d185020f1f0b08b2de0ca939cc0a0b/gevent-26.7.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:eaaa75c9014df3f8c310c64f53f1152af8c6be32e82734396bed91e1d0e6f35c", size = 2132200, upload-time = "2026-07-22T16:48:31.203Z" }, + { url = "https://files.pythonhosted.org/packages/42/88/1daffa63b257c381df68018e4d3da72b429955cf95a171a46d3220422c8d/gevent-26.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f1956032a9926ac9b4152b2a50bc5a2cc020722ec16928ccaf32e227ee0aae47", size = 1814237, upload-time = "2026-07-22T18:07:12.438Z" }, + { url = "https://files.pythonhosted.org/packages/73/4c/b996cdddca78eac435195cd97dff590c65a9e0052640bcb6f8b6f1e30b1d/gevent-26.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d989a1ad6cc54f5c69bb7304360f98b4fda80da2b773f1047db9fba61ae7379a", size = 2157938, upload-time = "2026-07-22T17:02:14.887Z" }, + { url = "https://files.pythonhosted.org/packages/08/fd/44419d7559a95e238ee45d29df30bad78a91d343cb27b13a6af552b907bb/gevent-26.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:e0c9ce2d80fc0f8894d748a1045ff26ad188e294bad656b29839271800827c85", size = 1685319, upload-time = "2026-07-22T16:26:13.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/7a/7df1762ccd9a40ce1ca626f50cb7a75758f2c930aabcc73c91cf00cb3448/gevent-26.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:959effe0c56cdee0bf761e5c4e78ab62880be147a2f2aa31112ca2f7e5754e53", size = 1558945, upload-time = "2026-07-22T16:26:56.737Z" }, + { url = "https://files.pythonhosted.org/packages/75/63/0fcfbe3f5696e56424f331ec41e0e447cea79c384848d6019e3f7f340f4b/gevent-26.7.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b1b89eb5566f75aa8b2bbdb0308e1ac8d9113ca7cff85b45366aea9faad639a1", size = 2976844, upload-time = "2026-07-22T16:24:39.275Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6c/ea2d0afbe760c18df5bd1631dbe5a73d840d9b141cb71e6810157c2ae28a/gevent-26.7.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:449857ce058183442e2d71d83ff0c587a3ddff631e93c6d19a6dffb4814eccad", size = 1802332, upload-time = "2026-07-22T18:11:29.155Z" }, + { url = "https://files.pythonhosted.org/packages/59/90/36f2258f1bfe8601224f6159066103b832c31e1451ce8dc2cd2408b6ecf4/gevent-26.7.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:8260a3f38b05fcf3c283417b18617562dbec74f5784f748e4ba3866789d7f3a4", size = 1901253, upload-time = "2026-07-22T18:10:44.402Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/86dd67e5c2dfab016a9c935b4338d6cf8f9bfa72dc5a0f3fb879d993127a/gevent-26.7.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:30894398d06747b433c8923a6a77ede61259ce6822a99f6c6e7fa0216ccb73c3", size = 1850489, upload-time = "2026-07-22T18:29:09.694Z" }, + { url = "https://files.pythonhosted.org/packages/f1/33/f5651942a5967483298b6ce6f45572d33120dd0fd01c8991d3fca5b1e8ee/gevent-26.7.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0b753522498118c9489753de7c612d4baed0edf384d9df2bf9492233ba1c20ff", size = 2129813, upload-time = "2026-07-22T16:48:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/c0/09/abe8217a8fcd3f0e94c9eec024a5499c0f267cb269ccea6c0e2e812319b9/gevent-26.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:055a643026dc28daff2be228555a2097937448cc9b58307edebcf81b9d78ff4b", size = 1815121, upload-time = "2026-07-22T18:07:13.988Z" }, + { url = "https://files.pythonhosted.org/packages/40/d6/dbae1cd2d27b62664cefa086035530eb21203d45b12f466272441c048c9c/gevent-26.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e1dc6a2712de67fd210e1f1a408601f6908b042f6420e188106f2f37f94ec71", size = 2155913, upload-time = "2026-07-22T17:02:16.35Z" }, + { url = "https://files.pythonhosted.org/packages/fc/42/90b662f4eb27d7727d4619d5c6be872117f1a9f187b243ec7f6fef988ce7/gevent-26.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:44e5280296129c0915addaefdb37d6e9bc124a77a433b1b1c8ddf1853c53f4e7", size = 1682483, upload-time = "2026-07-22T16:26:30.492Z" }, + { url = "https://files.pythonhosted.org/packages/e1/84/7297c56b9fff463c4ba2f685dbb913a855df903046dc68d14e8655a29ffe/gevent-26.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f08b1aa6729f794409ca137e25f671e0d9bbda4451200c5e28a769375365388", size = 1556053, upload-time = "2026-07-22T16:26:14.365Z" }, + { url = "https://files.pythonhosted.org/packages/c1/bb/ab60d496cbdc0293ebbd6c2070b34da0632bd7a2ca20163c17e18d2d2dc9/gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948", size = 2992503, upload-time = "2026-07-22T16:24:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/5c/35/75f27c06a82a5b22600aaccbd9567d89bb4091be43e96c02981f10aff23d/gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480", size = 1809173, upload-time = "2026-07-22T18:11:30.724Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/a6b32b4db3fa76bd8a070f0f46f5306123bf6336e7a0ca0cd2f9b99473df/gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6", size = 1906630, upload-time = "2026-07-22T18:10:45.836Z" }, + { url = "https://files.pythonhosted.org/packages/e1/87/832495d8fcc05ff7432f038b7c4decbd5632425a2cd5da2ce73cb2d800c4/gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0", size = 1855278, upload-time = "2026-07-22T18:29:11.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/2f36c0fa389fa2b7ceb5a8972b0e7da7bc770f9135315cf4246c607ca5fc/gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11", size = 2136155, upload-time = "2026-07-22T16:48:33.865Z" }, + { url = "https://files.pythonhosted.org/packages/b5/98/09f2cfaa23dbce48e3271e95b0d003f93acece6b5cfd40f4cebe3850d79b/gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0", size = 1822108, upload-time = "2026-07-22T18:07:15.397Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d8/05a294165c17569f04284ad3c889684c8780544885b4cdf77b1432947d0c/gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad", size = 2162814, upload-time = "2026-07-22T17:02:18.066Z" }, + { url = "https://files.pythonhosted.org/packages/59/89/58a545c4eda33e106d6887a0387adc2249abc14c779e3eb88bbfdf3768d6/gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a", size = 1706971, upload-time = "2026-07-22T16:27:02.263Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cd/413f293e54961e5c89c54235370e3603ec0f561e7ace8357980410efbf78/gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609", size = 1585078, upload-time = "2026-07-22T16:30:48.145Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/47f29f632aaa38aa12410f57f1732fc50bfd4d4006d2e7e022ce731cabc9/gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404", size = 2996208, upload-time = "2026-07-22T16:23:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b3/4620f1ce81ecec9890229806c73f07dd022e40f76a552bd430391e7316c4/gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17", size = 1811545, upload-time = "2026-07-22T18:11:32.224Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/d285212ffd5585d511299e13e61d76262def8e826e9f20c92cb85df406f2/gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce", size = 1910418, upload-time = "2026-07-22T18:10:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/c5d666e6065652918cfb6e6a3cf8f721d0e152c57cec217ad81792a1323b/gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2", size = 1857891, upload-time = "2026-07-22T18:29:13.13Z" }, + { url = "https://files.pythonhosted.org/packages/a1/67/e945ed458fa98b34572876bfd0d35fe4fa3f1159f43660b71d982b7cb63e/gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4", size = 2138121, upload-time = "2026-07-22T16:48:35.358Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/622468fa1a3c4cf51f20e14813f5cc1592fe6e44a42ccca9195a0b18c769/gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107", size = 1825114, upload-time = "2026-07-22T18:07:16.667Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7a/151a2afcacf487ca25faf8b1bdd6c5b4ace2f7c1e6b4eaffe0a5e6e1df61/gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e", size = 2165990, upload-time = "2026-07-22T17:02:19.817Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/9d/58f80897f4121f5c218bb931cf6d3b6514873f02ad0b729f744352926b9f/greenlet-3.5.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190", size = 293072, upload-time = "2026-07-22T11:38:14.299Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9f/b4bc9bbd6a7855cbd8ad8a83c874eeeca56c24de9132b3323f81c03a30ba/greenlet-3.5.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353", size = 609393, upload-time = "2026-07-22T12:26:37.964Z" }, + { url = "https://files.pythonhosted.org/packages/05/0e/744b5e063af127d2e3c74fe0f1aef15573064c83b6066883524f5b258b17/greenlet-3.5.4-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606", size = 622750, upload-time = "2026-07-22T12:28:59.546Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5c/53d6b94742a6f1ee1877c7ff76262c909e137f3f7383ce96a8ab78e1ae31/greenlet-3.5.4-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52", size = 629659, upload-time = "2026-07-22T12:43:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/d6/6b/d78ea2908e8e08985348f28ac396c2950be7ab66321dfe0054c73bd1f456/greenlet-3.5.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7", size = 622920, upload-time = "2026-07-22T11:51:06.83Z" }, + { url = "https://files.pythonhosted.org/packages/f2/34/957fc5577180ef2f57be82580ee1f59fdefad4f6c623c7d5e1b6980a76fb/greenlet-3.5.4-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c", size = 425580, upload-time = "2026-07-22T12:39:48.326Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/3ce7009c948920b01527f8d9da29f501a31ac3d98318829e981fd879b850/greenlet-3.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7", size = 1582262, upload-time = "2026-07-22T12:25:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4c/0408366102a33829f7bdd6a992dad75abbf75e86cc1e76caf19e57311d29/greenlet-3.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df", size = 1648906, upload-time = "2026-07-22T11:51:08.627Z" }, + { url = "https://files.pythonhosted.org/packages/13/52/ebfe8f6a1aeb8e430540b406c844ecc4e3367072b0192f69dcb85eeeec2b/greenlet-3.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616", size = 246036, upload-time = "2026-07-22T11:38:30.073Z" }, + { url = "https://files.pythonhosted.org/packages/61/16/71eefcf68267bbf06a9b6bff57d0b222e49432326e85d74348b67694b8d4/greenlet-3.5.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb", size = 294266, upload-time = "2026-07-22T11:37:56.142Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/a0b19adfc35d07e10acb626e9d22a3893b95f1309c42c4a20161dec16800/greenlet-3.5.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686", size = 613712, upload-time = "2026-07-22T12:26:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a121978b3337407d05a1ce5f79b4aa5998a43a9d8422f9726029b90b4471/greenlet-3.5.4-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7", size = 625582, upload-time = "2026-07-22T12:29:00.814Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4a/f301f1d85c69a86b90b5d581a73e8927bba4e79450037e6e2cbca05eb4fd/greenlet-3.5.4-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7", size = 633429, upload-time = "2026-07-22T12:43:42.073Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/080f16cf870e929e592f55767f01d6c98d2ee83bfdc36c3b892f2d0459ab/greenlet-3.5.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071", size = 624663, upload-time = "2026-07-22T11:51:08.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/26884072b0eb343a4d5fee903341bfe5171b32b7f14553886e2b6349135a/greenlet-3.5.4-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937", size = 428238, upload-time = "2026-07-22T12:39:49.973Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/8f3ca88370b817369008faeceeee85970adc16c92a70a3e5fe5fea495a57/greenlet-3.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72", size = 1585010, upload-time = "2026-07-22T12:25:02.539Z" }, + { url = "https://files.pythonhosted.org/packages/51/c2/45877154689709ebce9a0b83c2235e6ca0f31577889b02af308c8cc5f8fb/greenlet-3.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59", size = 1651283, upload-time = "2026-07-22T11:51:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7d/8711a75cb61d85246277c07ff6e1a6504621ba473d808c11ad225ffca43f/greenlet-3.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6", size = 246434, upload-time = "2026-07-22T11:43:15.557Z" }, + { url = "https://files.pythonhosted.org/packages/00/62/e290b3bce433da8f0324ac02da0b128d683482229f1a8b789fa47818a4cd/greenlet-3.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da", size = 244990, upload-time = "2026-07-22T11:39:22.626Z" }, + { url = "https://files.pythonhosted.org/packages/f3/04/81bd731d6d1e3a469d9a4c36f5eb069bcf0cbb2d5d342c9fec22245b91fc/greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4", size = 295909, upload-time = "2026-07-22T11:38:09.261Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dd/f5f22903a6ae70f5ea328ed0beaec92ad903f0e3b7d2845133b354abc4b8/greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17", size = 612011, upload-time = "2026-07-22T12:26:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/8e/10/92a4a88d12b915d74ea5b6d288e4afefda4771647caa34442c156f7a454f/greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a", size = 624299, upload-time = "2026-07-22T12:29:02.089Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f9/03e26be3487c5238e81f2b84714959a86ea8515a869828cf41f4fc54b34e/greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf", size = 629603, upload-time = "2026-07-22T12:43:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/0b14bb9db2989f32cd9fe7f76afedea01ee8bee3f87c07e69f24adfe7e63/greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f", size = 621541, upload-time = "2026-07-22T11:51:09.464Z" }, + { url = "https://files.pythonhosted.org/packages/57/6b/7c55ca72ef80d57c16c4a55210f82582622462dc4485799a30f4ec6f3372/greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f", size = 432554, upload-time = "2026-07-22T12:39:51.379Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/25e9a2d9eb6b2e8b7ca4e80a3a26cb887cce6c8e0a87c921164f11bc5574/greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d", size = 1581444, upload-time = "2026-07-22T12:25:03.818Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/4c9bf2e2c408dcc0556edce69efa9f802e82223573c53240136a086821f1/greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9", size = 1645842, upload-time = "2026-07-22T11:51:12.295Z" }, + { url = "https://files.pythonhosted.org/packages/b5/41/303ecb26a3a56122c0f4d4073ee078881847bd6b6f463ae0ec57ec20223b/greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3", size = 247169, upload-time = "2026-07-22T11:38:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/ef56864b4c35fcb3eb3b41b869f6cc46f4cd3f5e2c68e74acde8ac433951/greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0", size = 245565, upload-time = "2026-07-22T11:38:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, + { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, + { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, + { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, + { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, + { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, + { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, + { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, + { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, + { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1158,3 +1307,60 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/43/9cd98bee951d23848de690ba2809f87e3b22c67c370987acc960da15ad37/zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe", size = 210951, upload-time = "2026-05-26T06:49:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/8f1a29966bcf863e3a2121edcafb81c55715de7886bcc9544749cc79e7da/zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c", size = 211309, upload-time = "2026-05-26T06:49:02.732Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9f/37e564eaaf85e3abc1ada40a79fa43f2ab45bdb67431b0ec0fe29e4763e2/zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37", size = 254881, upload-time = "2026-05-26T06:49:04.303Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/e6501d8ea7a2cac3217e03f404e1f98c1df7191d83cfe86b1895fbba5dac/zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a", size = 259811, upload-time = "2026-05-26T06:49:06.373Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/bfa25ef480b02af6e9452c478483fec75e87c9e2b60c407fd0b1f6054b9c/zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58", size = 260358, upload-time = "2026-05-26T06:49:08.317Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/2b518072fea76242da64451d501c69b7b5ccdef9b57fead584ccf1c180d5/zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170", size = 214822, upload-time = "2026-05-26T06:49:10.441Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/83ad110fb847413affe71609bb50e59e1aa082e1236030122227c7c283d3/zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057", size = 211426, upload-time = "2026-05-26T06:49:12.595Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a7/6b6e0c31ac240cb9fc015ae9ed45ca54be886c18fcf7bfa2377a4d7a8785/zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f", size = 211850, upload-time = "2026-05-26T06:49:14.474Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/7599ecabcf80ce4fef2e1ef3c5ac0d4696b61f03f724cc44022f4d226af9/zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376", size = 260711, upload-time = "2026-05-26T06:49:16.568Z" }, + { url = "https://files.pythonhosted.org/packages/03/3e/1774b0ee46ccbb5498ee3c33ece40315b6ef58bc71957be94bd345340bc1/zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f", size = 265277, upload-time = "2026-05-26T06:49:18.656Z" }, + { url = "https://files.pythonhosted.org/packages/b6/09/e533b2ffabaae4e5d5730d6768a591cf335defe8e37bec2ad905d09be656/zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2", size = 266369, upload-time = "2026-05-26T06:49:20.174Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/3ebe6a4c122b2d5340db45cbe7e490663d3228b172710ec71060cd5d541e/zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955", size = 215161, upload-time = "2026-05-26T06:49:21.799Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/056ad97af5b16db1975ee98ec7ab03d2ce3f3355efad904ced1dbce0e39f/zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646", size = 213481, upload-time = "2026-05-26T06:49:25.085Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" }, + { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, + { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" }, + { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" }, + { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" }, + { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" }, +] From 5df051ce664f023213af9c36b1078748321972b8 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:23:28 +0200 Subject: [PATCH 40/66] docs: record the landed boundary rethink (P1-P5) in the handoff Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 48 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 52336693..59cd1141 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -98,21 +98,39 @@ portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full asyncio-native API over the trio host loop, all transports, no anyio port. Phase E (anyio-native core) becomes optional purity/perf work. -## Staging - -1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port - SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. -2. **P2 — channel unification** (big one): sync Channel onto - RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, - no intermediate primitive-swap (suite pins semantics either way). - Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. -3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, - `wait=` joins the C.1 spec axes + worker CLI config. -4. **P4 — `execnet.aio`**. -5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). - -Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D -docs cover the three namespaces + `execnet.aio` + the axes/presets. +## Staging — ALL LANDED 2026-07-26 (commits 2c78416..b01d009) + +1. **P1 — boundary kit** (`2c78416`): Wakener/Mailbox/OneShot + + ThreadWakener; SyncReceiver, write-acks, `_done_sync` ported. +2. **P2 — channel unification** (`2a40a55`): sync Channel onto + RawChannel(+set_consumer)+Mailbox, classic dispatch deleted. The kit + lives in trio-free `execnet._boundary` (portal re-exports) so + `import execnet` stays trio-free. Fix worth knowing: an unconsumed + remotely-closed RawChannel stays registered until a consumer claims + it — call-site deserialization means a passed channel can bind after + its data AND close arrived (was a payload-loss race, latent in the + async core's `open_channel(id)` too). +3. **P3 — retirement** (`811fd99`): ExecModel ABC/WorkerPool/Reply gone; + one deprecated ExecModel preset KEEPS the stdlib-delegate members + because pytest-xdist's remote worker calls + `channel.gateway.execmodel.RLock()/Event()`; `wait=` axis end to end + (spec validation, worker CLI config, `BaseGateway._new_wakener`, + `_boundary` registry + Flag); safe_terminate on daemon threads; + test_threadpool.py deleted. +4. **P4 — `execnet.aio`** (`845510e`): implemented as a per-call + _HostBridge (host task + `loop.call_soon_threadsafe` future), NOT the + originally sketched AsyncioWakener/Mailbox — real awaitables over the + trio-native AsyncGroup/AsyncChannel, simpler and semantically exact. + The "asyncio" wait= registry entry is therefore unused/unregistered. + v1 limitation: cancelling a bridged await abandons it aio-side only. +5. **P5 — gevent wakener** (`b01d009`): `execnet._gevent_support`, + lazily imported by `make_wakener("gevent")`; hub-bound pieces created + in the first waiter's hub (carriers may be constructed on the host + loop); tests behind the `gevent` dependency group. + +Next: Phase C `loop=main` / `exec=task` on the smaller core +(handoff-phase-c-worker-axes.md), and Phase D docs cover the four +namespaces + the axes/presets. ## Invariants (unchanged, re-mapped) From 7969f7e37daea12c5b6543c0ed2680db96d81275 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:47:43 +0200 Subject: [PATCH 41/66] docs: record the decided Phase C plan (axes matrix, trio-only core) Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 351f6c16..40c222b6 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,7 +71,46 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — what is missing (nothing of C exists yet) +## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) + +Predates below gap analysis; where they conflict, this section wins. +Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) +landed first — `wait=` axis exists, ExecModel is a preset, the sync +Channel is a facade over the async core. + +**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). +The main thread is the scarce resource (signals, interrupt_main, GUI, +tests calling asyncio.run/trio.run themselves): + +| loop= | exec= | main thread | exec'd code | channel | consumer | +|---|---|---|---|---|---| +| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | +| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | +| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | +| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | +| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | +| main | main | — | — | — | invalid | + +Interaction rule: exec'd code may start its own event loop only when it +does not share a thread with ours (exec=thread / exec=main); exec=task +sources are ``async def`` and join our loop — a sync source under +exec=task is a hard error. ``wait=`` composes orthogonally. + +**Backend decisions**: the core STAYS trio-only and trio stays the +default and a hard dependency (pure-python wheels; deployment cost +accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; +asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands +as a per-gateway spec axis, but reserved: only ``trio`` validates +(exactly how ``wait=`` landed with only ``thread``), keeping an +asyncio-core door open without paying for it. Consequence: exec=task +sources are trio-typed. + +Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + +presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → +C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; +`execmodel=main_thread_only` → `loop=thread, exec=main`. + +## Phase C — original gap analysis (pre-decision record) Target axes: `loop=thread|main` × `exec=thread|main|task`. Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; From b2f43c38d0bb53265d481064e36f6216c5542b5e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:35:31 +0200 Subject: [PATCH 42/66] refactor: extract worker exec placement strategies behind the FIFO pump C1 of the profile plan (approved 2026-07-26): TrioWorkerExec becomes a pure admission pump (message-arrival FIFO preserved) delegating placement to a strategy object -- PoolExec (worker threads, today's "thread") and MainExec (serialized main thread + deadlock guard, today's "main_thread_only"), selected via WORKER_EXEC_STRATEGIES by execmodel name. _run_worker keys main-thread integration off strategy.needs_primary_thread instead of a hardcoded flag. No behavior change; upcoming profiles (trio task exec, classic hybrid, gevent greenlets, later subinterpreters) slot in as new strategies. Co-Authored-By: Claude Fable 5 --- src/execnet/_gevent_support.py | 2 +- src/execnet/_trio_worker.py | 177 +++++++++++++++++++++++---------- src/execnet/multi.py | 5 +- 3 files changed, 129 insertions(+), 55 deletions(-) diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py index cd761571..a999ab96 100644 --- a/src/execnet/_gevent_support.py +++ b/src/execnet/_gevent_support.py @@ -63,7 +63,7 @@ def wait(self, timeout: float | None = None) -> bool: # a notify may have fired before the watcher existed if self._notified and not event.is_set(): event.set() - return event.wait(timeout) + return bool(event.wait(timeout)) def clear(self) -> None: with self._lock: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 89aad122..c5152cc2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -6,6 +6,7 @@ import os import sys import threading +from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any @@ -26,40 +27,128 @@ ExecItem = tuple[Any, ...] +class PoolExec: + """Exec strategy: run each request on a worker thread (trio's pool). + + The building block of the ``thread`` profile; exec'd code may freely + start its own event loops (it never shares a thread with ours). + """ + + #: whether _run_worker must hand this strategy the process main thread + needs_primary_thread = False + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + """FIFO admission gate; always open for pool placement.""" + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + + def integrate_as_primary_thread(self) -> None: + raise RuntimeError("pool exec strategy does not use the main thread") + + def trigger_shutdown(self) -> None: + pass + + +class MainExec: + """Exec strategy: serialize each request onto the process main thread. + + The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist + runs this way). Admission waits for the previous request to finish and + closes the channel with the deadlock text when it cannot within a + second (a second concurrent remote_exec would deadlock the requester). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return False + complete.clear() + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running exec tasks until shutdown.""" + while True: + task = self._primary.get() + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + +# execmodel profile -> exec strategy. Placement strategies to come: +# "trio" (tasks on a main-thread loop), classic hybrid for "thread" +# (primary main + pool overflow), "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. +WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { + "thread": PoolExec, + "main_thread_only": MainExec, +} + + class TrioWorkerExec: - """Schedule ``remote_exec`` work from the Trio host nursery. + """FIFO admission pump feeding an exec placement strategy. - * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). - * ``main_thread_only``: hand off to the process main thread (GUI-safe). + Exec requests flow through a single pump task so admission happens + strictly in message-arrival order (trio task scheduling order is + deliberately unordered, so per-request tasks would race for e.g. the + main_thread_only slot). Where an admitted request runs is the + strategy's business (:data:`WORKER_EXEC_STRATEGIES`). """ def __init__( self, host: _trio_host.TrioHost, gateway: WorkerGateway, - *, - main_thread_only: bool, + strategy: Any, ) -> None: self.host = host self.gateway = gateway - self.main_thread_only = main_thread_only + self.strategy = strategy self._lock = threading.Lock() self._running = 0 self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( - Mailbox() - ) - # Exec requests flow through a single pump task so admission happens - # strictly in message-arrival order (trio task scheduling order is - # deliberately unordered, so per-request tasks would race for the - # main_thread_only slot). self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) self._pump_started = False + @property + def needs_primary_thread(self) -> bool: + return bool(self.strategy.needs_primary_thread) + def active_count(self) -> int: with self._lock: return self._running @@ -78,7 +167,7 @@ def _track_finish(self) -> None: def schedule(self, channel: Channel, sourcetask: bytes) -> None: """Called from the session dispatch on the Trio host thread. - Must not block: deadlock checks and exec run in a nursery task. + Must not block: admission checks and exec run in a nursery task. """ item = loads_internal(sourcetask) assert isinstance(item, tuple) @@ -95,48 +184,23 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: async def _pump(self) -> None: """Admit queued exec requests in FIFO order, then run each as a task.""" async for channel, item in self._pending_recv: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) - if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - continue - complete.clear() - self.host.start_soon(self._run_exec, channel, item) + if await self.strategy.admit(channel, item): + self.host.start_soon(self._run_exec, channel, item) async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: - if self.main_thread_only: - done = threading.Event() - self._primary.put((channel, item, done)) - await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) - else: - await trio.to_thread.run_sync( - self.gateway.executetask, - (channel, item), - abandon_on_cancel=True, - ) + await self.strategy.run(channel, item) finally: self._track_finish() def integrate_as_primary_thread(self) -> None: - """Block the main thread running main_thread_only exec tasks.""" - while True: - task = self._primary.get() - if task is None: - break - channel, item, done = task - try: - self.gateway.executetask((channel, item)) - finally: - done.set() + self.strategy.integrate_as_primary_thread() def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary.put(None) + self.strategy.trigger_shutdown() def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) @@ -202,30 +266,37 @@ def kill(self) -> None: def _build_worker_gateway( host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" -) -> tuple[WorkerGateway, TrioWorkerExec, bool]: - """Construct the WorkerGateway + Trio exec pool (no IO yet).""" +) -> tuple[WorkerGateway, TrioWorkerExec]: + """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) gateway._wait_backend = wait - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. + try: + strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] + except KeyError: + raise ValueError( + f"execmodel {model.backend!r} has no worker exec strategy " + f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" + ) from None + strategy = strategy_factory(gateway) + trio_exec = TrioWorkerExec(host, gateway, strategy) + # Duck-type as the exec pool for STATUS / _terminate_execution. gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None - if main_thread_only: + if strategy.needs_primary_thread: gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() - return gateway, trio_exec, main_thread_only + return gateway, trio_exec def _run_worker( host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" ) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) + gateway, trio_exec = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -235,7 +306,7 @@ async def _start() -> _trio_host.SyncBridgeGateway: host.call(_start) try: - if main_thread_only: + if trio_exec.needs_primary_thread: trace("integrating as primary thread (trio worker)") trio_exec.integrate_as_primary_thread() gateway.join() diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ba73bf62..4bbfa7cf 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -154,7 +154,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread' or 'main_thread_only' execution model + execmodel=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' (pool + threads) or 'main_thread_only' (serialized on + the worker main thread, GUI/signal-safe). wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process From d8f22c5f269536bac36a8de2f2d503a150b870eb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:44:20 +0200 Subject: [PATCH 43/66] feat!: answer info natively and apply chdir/nice/env from worker config C2 of the profile plan: coordinator bookkeeping no longer goes through remote_exec, so it can never claim an exec slot -- with main-thread profiles an info call used to occupy the main thread (or trip the main_thread_only deadlock guard) and push the real workload (pytest) onto a worker thread. - New Message.GATEWAY_INFO (code 10), answered by the dispatch loop like STATUS with the sys/env payload (gateway_base.gateway_info); Gateway._rinfo() sends it instead of executing rinfo_source, which is deleted. Handled in both the sync bridge and the async core. - chdir/nice/env ship in the worker config JSON (worker_cli_arg; the via spawn_request inherits it) and are applied by the worker before serving; the post-start remote_exec setup block in multi.makegateway is gone. This also makes setup valid for the upcoming pure-async profile, which cannot run sync sources at all. Regression test: _rinfo(update=True) while an exec occupies the worker, for both execmodels. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 23 +++++++++++++++-------- src/execnet/_trio_gateway.py | 6 ++++++ src/execnet/_trio_host.py | 8 ++++++++ src/execnet/_trio_worker.py | 20 ++++++++++++++++++++ src/execnet/gateway.py | 34 ++++++++++++---------------------- src/execnet/gateway_base.py | 19 +++++++++++++++++++ src/execnet/multi.py | 21 ++------------------- testing/test_basics.py | 16 ---------------- testing/test_gateway.py | 13 +++++++++++++ 9 files changed, 95 insertions(+), 65 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index f5114ad8..ffe9ffaa 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,14 +164,21 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet - return json.dumps( - { - "id": f"{spec.id}-worker", - "execmodel": spec.execmodel, - "wait": spec.wait or "thread", - "coordinator_version": execnet.__version__, - } - ) + config: dict[str, Any] = { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "wait": spec.wait or "thread", + "coordinator_version": execnet.__version__, + } + # Startup setup applied by the worker before serving (never through + # remote_exec: an exec slot must not be claimed by bookkeeping). + if spec.chdir: + config["chdir"] = spec.chdir + if spec.nice: + config["nice"] = int(spec.nice) + if spec.env: + config["env"] = spec.env + return json.dumps(config) def _worker_tokens(config: str) -> list[str]: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fc364772..66a892e8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -46,6 +46,7 @@ from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace @@ -678,6 +679,11 @@ def _dispatch(self, message: Message) -> None: } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.GATEWAY_INFO: + self._send_nowait( + Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) + ) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) elif code == Message.RECONFIGURE: data = loads_internal(message.data) assert isinstance(data, tuple) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e29af4d2..9a1f1338 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal @@ -177,6 +178,13 @@ def _dispatch(self, message: Message) -> None: try: if code == Message.STATUS: self._answer_status(message) + elif code == Message.GATEWAY_INFO: + gateway._send( + Message.CHANNEL_DATA, + message.channelid, + dumps_internal(gateway_info()), + ) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c5152cc2..56e64ef3 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -377,6 +377,25 @@ def _rough_version(version: str) -> tuple[int, ...]: return tuple(parts) +def _apply_worker_setup(config: dict[str, Any]) -> None: + """Apply chdir/nice/env from the worker config, before serving starts. + + This replaces the classic post-start ``remote_exec`` setup: valid for + every profile (a trio worker cannot run sync sources) and never + claims an exec slot. + """ + path = config.get("chdir") + if path: + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + nice = config.get("nice") + if nice and hasattr(os, "nice"): + os.nice(nice) + for name, value in config.get("env", {}).items(): + os.environ[name] = value + + def _check_version(coordinator_version: str) -> None: """Warn on a real (major/minor) execnet version mismatch across the wire. @@ -416,6 +435,7 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) + _apply_worker_setup(config) if ns.socket_fd is not None: serve_socket_trio( config["id"], diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index bec4c778..672a0830 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,7 +26,6 @@ "RemoteStatus", "_find_non_builtin_globals", "_source_of_function", - "rinfo_source", ] @@ -94,13 +93,19 @@ def reconfigure( self._send(Message.RECONFIGURE, data=data) def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote.""" + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ if update or not hasattr(self, "_cache_rinfo"): - ch = self.remote_exec(rinfo_source) - try: - self._cache_rinfo = RInfo(ch.receive()) - finally: - ch.waitclose() + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) return self._cache_rinfo def hasreceiver(self) -> bool: @@ -166,18 +171,3 @@ def __getattr__(self, name: str) -> Any: ... RemoteStatus = RInfo - - -def rinfo_source(channel) -> None: - import os - import sys - - channel.send( - dict( - executable=sys.executable, - version_info=sys.version_info[:5], - platform=sys.platform, - cwd=os.getcwd(), - pid=os.getpid(), - ) - ) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 6a286302..c0ac5252 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -187,6 +187,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE = 7 GATEWAY_START_SOCKET = 8 GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 # message code -> name _types: dict[int, str] = { @@ -200,6 +201,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: @@ -243,6 +245,23 @@ def __repr__(self) -> str: return f"" +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4bbfa7cf..ac88b181 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -183,25 +183,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) - if spec.chdir or spec.nice or spec.env: - channel = gw.remote_exec( - """ - import os - path, nice, env = channel.receive() - if path: - if not os.path.exists(path): - os.mkdir(path) - os.chdir(path) - if nice and hasattr(os, 'nice'): - os.nice(nice) - if env: - for name, value in env.items(): - os.environ[name] = value - """ - ) - nice = (spec.nice and int(spec.nice)) or 0 - channel.send((spec.chdir, nice, spec.env)) - channel.waitclose() + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. return gw def allocate_id(self, spec: XSpec) -> None: diff --git a/testing/test_basics.py b/testing/test_basics.py index c58e4728..470bec6d 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -153,22 +153,6 @@ def test_io_message(checker: Checker) -> None: assert "all passed" in out.stdout -def test_rinfo_source(checker: Checker) -> None: - out = checker.run_check( - f""" -class Channel: - def send(self, data): - assert eval(repr(data), {{}}) == data -channel = Channel() -{inspect.getsource(gateway.rinfo_source)} -print ('all passed') -""" - ) - - print(out.stdout) - assert "all passed" in out.stdout - - def test_geterrortext(checker: Checker) -> None: out = checker.run_check( standalone_gateway_base_source() diff --git a/testing/test_gateway.py b/testing/test_gateway.py index da519d4e..ae51cf0d 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -285,6 +285,19 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: + # info is a native protocol request: it must work (and not claim + # an exec slot) while an exec occupies the worker -- under + # main_thread_only an info-by-remote_exec used to either steal + # the main thread or trip the concurrency deadlock guard. + channel = gw.remote_exec("channel.send(channel.receive())") + try: + rinfo = gw._rinfo(update=True) + assert rinfo.pid != os.getpid() + finally: + channel.send("done") + assert channel.receive(TESTTIMEOUT) == "done" + class TestPopenGateway: gwtype = "popen" From cd2c8717fa481dd60a41e62da1439783558fa95b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:55:17 +0200 Subject: [PATCH 44/66] feat: pure-async worker profile (execmodel=trio) C3 of the profile plan: the whole worker is one trio run owning the process main thread -- no host thread, no sync bridge, no sync channels. AsyncGateway gains a pluggable CHANNEL_EXEC handler (and live numexecuting for STATUS); TaskExec runs each source as a task on the worker nursery with an AsyncChannel. Sources must be async: strings execute with top-level await (PyCF_ALLOW_TOP_LEVEL_AWAIT), functions must be async def -- sync sources are rejected with a RemoteError before they can starve the loop. Gateway termination cancels running exec tasks. execmodel values are validated coordinator-side in makegateway against the profile registry (EXECMODEL_PROFILES); the worker CLI branches to the async entry (stdio and inherited-socket transports). Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 11 ++- src/execnet/_trio_worker.py | 139 +++++++++++++++++++++++++++++++-- src/execnet/gateway_base.py | 10 ++- src/execnet/multi.py | 6 ++ testing/test_execmodel_trio.py | 124 +++++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 testing/test_execmodel_trio.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 66a892e8..81ea76d0 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -461,6 +461,12 @@ class AsyncGateway: _error: BaseException | None = None #: where the peer lives, when the transport knows (ssh host, socket addr) remoteaddress: str | None = None + #: CHANNEL_EXEC handler ``(gateway, channelid, data) -> None`` -- set by + #: an exec strategy (the pure-async worker's TaskExec); without one, + #: exec requests are rejected. + _exec_handler: Callable[[AsyncGateway, int, bytes], None] | None = None + #: the exec strategy (for STATUS numexecuting), when serving as a worker + _task_exec: Any = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -671,10 +677,13 @@ def _dispatch(self, message: Message) -> None: self._channel_for(channelid)._close_from_remote(None, sendonly=True) elif code == Message.GATEWAY_TERMINATE: raise GatewayReceivedTerminate(self) + elif code == Message.CHANNEL_EXEC and self._exec_handler is not None: + self._exec_handler(self, channelid, message.data) elif code == Message.STATUS: + task_exec = self._task_exec status = { "numchannels": len(self._channels), - "numexecuting": 0, + "numexecuting": task_exec.active_count() if task_exec else 0, "execmodel": "trio", } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 56e64ef3..49e53a69 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -15,6 +15,7 @@ from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import geterrortext from .gateway_base import loads_internal from .gateway_base import trace from .portal import Mailbox @@ -106,16 +107,93 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -# execmodel profile -> exec strategy. Placement strategies to come: -# "trio" (tasks on a main-thread loop), classic hybrid for "thread" -# (primary main + pool overflow), "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# execmodel profile -> exec strategy for the sync-facade worker; the +# "trio" profile serves a plain AsyncGateway instead (TaskExec below). +# Placement strategies to come: classic hybrid for "thread" (primary +# main + pool overflow), "gevent" (greenlets on a main-thread hub), and +# eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": PoolExec, "main_thread_only": MainExec, } +class TaskExec: + """Exec strategy for the pure-async profile (``execmodel=trio``). + + Sources run as tasks on the worker's own loop, in the one and only + thread of the process, and receive an ``AsyncChannel``. Sources must + be async: a plain function or a source string without top-level + ``await`` is rejected before it can starve the loop. Gateway + termination cancels running exec tasks (``trio.Cancelled`` inside the + source). + """ + + def __init__(self, gateway: Any, nursery: trio.Nursery) -> None: + self.gateway = gateway + self.nursery = nursery + self._running = 0 + + def active_count(self) -> int: + return self._running + + def handle_exec(self, gateway: Any, channelid: int, data: bytes) -> None: + """CHANNEL_EXEC hook; runs inline on the dispatch task.""" + item = loads_internal(data) + assert isinstance(item, tuple) + channel = gateway.open_channel(channelid) + self.nursery.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Any, item: ExecItem) -> None: + import ast + import inspect + + source, file_name, call_name, kwargs = item + self._running += 1 + try: + trace(f"async execution starts[{channel.id}]: {source[:50]!r}") + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + co = compile( + source + "\n", + file_name or "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) + if not toplevel_await and not call_name: + await channel.aclose( + "sync source under execmodel=trio: the source must use" + " top-level await (or pass an async function)" + ) + return + if toplevel_await: + await eval(co, loc) + else: + exec(co, loc) # define the function (no top-level awaits) + if call_name: + function = loc[call_name] + result = function(channel, **kwargs) + if not hasattr(result, "__await__"): + await channel.aclose( + f"sync function {call_name!r} under execmodel=trio:" + " remote_exec functions must be async" + ) + return + await result + except trio.Cancelled: + raise + except EOFError: + trace("ignoring EOFError from async exec") + except BaseException as exc: + trace(f"async exec got exception: {exc!r}") + await channel.aclose(geterrortext(exc)) + return + finally: + self._running -= 1 + trace("async execution finished") + await channel.aclose() + + class TrioWorkerExec: """FIFO admission pump feeding an exec placement strategy. @@ -326,6 +404,51 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) +async def _serve_async_worker(stream: Any, id: str) -> None: + """Serve a plain AsyncGateway with task-based exec (execmodel=trio). + + The whole worker is this one trio run on the process main thread: the + dispatch loop and every exec'd source share it. Termination cancels + running exec tasks. + """ + from ._trio_gateway import AsyncGateway + + gateway = AsyncGateway(stream, id=id, _startcount=2) + async with trio.open_nursery() as nursery: + task_exec = TaskExec(gateway, nursery) + gateway._exec_handler = task_exec.handle_exec + gateway._task_exec = task_exec + await nursery.start(gateway._serve) + await gateway.wait_closed() + nursery.cancel_scope.cancel() + + +def serve_popen_async(id: str) -> None: + """Serve the pure-async worker over the stdio pipes (execmodel=trio).""" + from ._trio_gateway import staple_fd_stream + + read_fd, write_fd = _prepare_protocol_fds() + os.write(write_fd, b"1") + + async def main() -> None: + await _serve_async_worker(staple_fd_stream(read_fd, write_fd), id) + + trio.run(main) + os._exit(0) + + +def serve_socket_async(id: str, socket_fd: int) -> None: + """Serve the pure-async worker over an inherited socket fd.""" + from . import _trio_host + + async def main() -> None: + stream = await _trio_host.adopt_socket(socket_fd) + await _serve_async_worker(stream, id) + + trio.run(main) + os._exit(0) + + def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -436,7 +559,13 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) _apply_worker_setup(config) - if ns.socket_fd is not None: + if config["execmodel"] == "trio": + # pure-async profile: one thread, the loop owns the main thread + if ns.socket_fd is not None: + serve_socket_async(config["id"], ns.socket_fd) + else: + serve_popen_async(config["id"]) + elif ns.socket_fd is not None: serve_socket_trio( config["id"], config["execmodel"], diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c0ac5252..77fd81df 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -123,10 +123,18 @@ def Event(self) -> threading.Event: return threading.Event() +#: worker profiles: where exec'd code runs relative to the protocol loop +EXECMODEL_PROFILES = ( + "thread", # exec on pool threads, loop on a side thread + "main_thread_only", # exec serialized on the main thread (GUI/pytest) + "trio", # pure async: loop owns the main thread, async sources as tasks +) + + def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend in ("thread", "main_thread_only"): + if backend in EXECMODEL_PROFILES: return ExecModel(backend) raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ac88b181..87b17786 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -24,6 +24,7 @@ from typing import overload from ._boundary import wakener_names +from .gateway_base import EXECMODEL_PROFILES from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import get_execmodel @@ -172,6 +173,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + elif spec.execmodel not in EXECMODEL_PROFILES: + raise ValueError( + f"unknown execmodel {spec.execmodel!r}" + f" (known profiles: {list(EXECMODEL_PROFILES)})" + ) if spec.wait is not None and spec.wait not in wakener_names(): raise ValueError( f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py new file mode 100644 index 00000000..d23a72cf --- /dev/null +++ b/testing/test_execmodel_trio.py @@ -0,0 +1,124 @@ +"""The pure-async worker profile (execmodel=trio). + +One single thread in the worker: the trio loop owns the main thread and +exec'd sources run as tasks on it, talking through AsyncChannels. Sync +sources are rejected before they can starve the loop. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest +import trio as trio_lib + +import execnet +import execnet.trio +from execnet.gateway import Gateway + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: + return makegateway("popen//execmodel=trio") + + +class TestSyncCoordinator: + def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + "await channel.send(await channel.receive() + 1)" + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_async_function_source(self, trio_gw: Gateway) -> None: + async def source(channel, delta) -> None: + value = await channel.receive() + await channel.send(value + delta) + + channel = trio_gw.remote_exec(source, delta=5) + channel.send(37) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_iteration_and_eof(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + for i in range(3): + await channel.send(i * 2) + """ + ) + assert list(channel) == [0, 2, 4] + + def test_single_thread_on_main(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + import threading + await channel.send( + ( + threading.active_count(), + threading.current_thread() is threading.main_thread(), + ) + ) + """ + ) + active, on_main = channel.receive(TESTTIMEOUT) + assert on_main + assert active == 1 + + def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: + # two execs run as tasks on one loop: the first parks in receive + # while the second completes -- no threads involved. + blocked = trio_gw.remote_exec( + "await channel.send(await channel.receive())" + ) + side = trio_gw.remote_exec("await channel.send('side')") + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_sync_source_rejected(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec("x = 40 + 2") + with pytest.raises(channel.RemoteError, match="sync source"): + channel.receive(TESTTIMEOUT) + + def test_sync_function_rejected(self, trio_gw: Gateway) -> None: + def source(channel) -> None: + pass + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="must be async"): + channel.receive(TESTTIMEOUT) + + def test_remote_error_traceback(self, trio_gw: Gateway) -> None: + async def source(channel) -> None: + raise ValueError(17) + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="ValueError"): + channel.receive(TESTTIMEOUT) + + def test_status_and_rinfo(self, trio_gw: Gateway) -> None: + assert trio_gw.remote_status().execmodel == "trio" + rinfo = trio_gw._rinfo() + assert rinfo.pid + assert rinfo.version_info + + +def test_unknown_execmodel_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown execmodel"): + makegateway("popen//execmodel=nope") + + +def test_trio_native_coordinator() -> None: + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen//execmodel=trio") + channel = await gateway.remote_exec( + "await channel.send(await channel.receive() * 2)" + ) + await channel.send(21) + with trio_lib.fail_after(TESTTIMEOUT): + assert await channel.receive() == 42 + + trio_lib.run(main) From 36e904e52dd22e0a8aea0e3f8761107d0363cc76 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 17:18:34 +0200 Subject: [PATCH 45/66] feat!: restore classic hybrid exec for execmodel=thread C4 of the profile plan: the thread profile regains its pre-port shape -- a request arriving while the worker main thread is idle claims it (the primary workload gets a true main thread again), and requests arriving while it is busy overflow to pool threads instead of queueing. The claim is decided during FIFO admission, so the first exec always lands on the main thread. HybridExec composes MainExec (primary mailbox + integrate loop) with PoolExec overflow; main-thread workers now integrate as primary instead of parking in join(). Together with C2 (native info/setup) this cannot be starved by coordinator bookkeeping: only real remote_exec work claims the slot. xdist is unaffected (it forces main_thread_only); the -n 12 suite runs are the live canary. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_worker.py | 45 +++++++++++++++++++++++++++++++--- testing/test_execmodel_trio.py | 8 ++---- testing/test_gateway.py | 32 ++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 49e53a69..c6042bb8 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -107,13 +107,50 @@ def trigger_shutdown(self) -> None: self._primary.put(None) +class HybridExec(MainExec): + """Exec strategy: primary on the main thread, overflow on pool threads. + + The classic ``thread`` execmodel shape: a request arriving while the + main thread is idle claims it (pytest and friends get a true main + thread); requests arriving while it is busy run on worker threads + instead of queueing. The claim is decided during FIFO admission so + the *first* request always gets the main thread. + """ + + def __init__(self, gateway: WorkerGateway) -> None: + super().__init__(gateway) + self._pool = PoolExec(gateway) + self._claim_lock = threading.Lock() + self._primary_busy = False + self._claimed: set[int] = set() + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + with self._claim_lock: + if not self._primary_busy: + self._primary_busy = True + self._claimed.add(channel.id) + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + with self._claim_lock: + claimed = channel.id in self._claimed + self._claimed.discard(channel.id) + if not claimed: + await self._pool.run(channel, item) + return + try: + await super().run(channel, item) + finally: + with self._claim_lock: + self._primary_busy = False + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: classic hybrid for "thread" (primary -# main + pool overflow), "gevent" (greenlets on a main-thread hub), and -# eventually subinterpreters. +# Placement strategies to come: "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { - "thread": PoolExec, + "thread": HybridExec, "main_thread_only": MainExec, } diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index d23a72cf..84585ba9 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -26,9 +26,7 @@ def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: class TestSyncCoordinator: def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: - channel = trio_gw.remote_exec( - "await channel.send(await channel.receive() + 1)" - ) + channel = trio_gw.remote_exec("await channel.send(await channel.receive() + 1)") channel.send(41) assert channel.receive(TESTTIMEOUT) == 42 @@ -69,9 +67,7 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: # two execs run as tasks on one loop: the first parks in receive # while the second completes -- no threads involved. - blocked = trio_gw.remote_exec( - "await channel.send(await channel.receive())" - ) + blocked = trio_gw.remote_exec("await channel.send(await channel.receive())") side = trio_gw.remote_exec("await channel.send('side')") assert side.receive(TESTTIMEOUT) == "side" blocked.send("go") diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ae51cf0d..6b6d5cf5 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -9,6 +9,7 @@ import shutil import signal import sys +import time from collections.abc import Callable from textwrap import dedent @@ -285,6 +286,37 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test_hybrid_primary_then_overflow( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # classic thread-model shape: the first exec claims the true main + # thread; while it is busy, further execs overflow to worker + # threads; once released the main thread is claimable again. + gw = makegateway("popen//execmodel=thread") + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = gw.remote_exec(report) + assert first.receive(TESTTIMEOUT) is True + second = gw.remote_exec(report) + assert second.receive(TESTTIMEOUT) is False + second.send(None) + second.waitclose(TESTTIMEOUT) + first.send(None) + first.waitclose(TESTTIMEOUT) + # the primary slot frees shortly after the exec finishes + for _ in range(50): + probe = gw.remote_exec(report) + on_main = probe.receive(TESTTIMEOUT) + probe.send(None) + probe.waitclose(TESTTIMEOUT) + if on_main: + break + time.sleep(0.05) + assert on_main + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: # info is a native protocol request: it must work (and not claim # an exec slot) while an exec occupies the worker -- under From cbce1830630cd02388992e8d9773b0639693b8ae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:21:54 +0200 Subject: [PATCH 46/66] feat: gevent worker profile and gevent-safe management ops C5 of the profile plan, completing the gevent/loop integration: - execmodel=gevent: GreenletExec runs the hub on the worker main thread and spawns a greenlet per remote_exec; the worker's wait backend defaults to gevent so channel operations park greenlets, letting concurrent execs cooperate on one thread. The integrate loop itself waits on a gevent wakener (a thread-blocking get would stall the hub). - Provisioning: execnet grows a [gevent] extra; every uv launcher (popen python=, ssh, via sub-spawn) derives an additional `--with gevent` from the worker config when the profile or wait backend needs it. - Coordinator side: TrioHost.call_pending resolves a OneShot from a host task; makegateway (wait!=thread), SyncIOHandle.wait/kill, and Group.terminate wait on it with the gateway's wakener so a gevent app's management ops park only the calling greenlet. wait=thread keeps the KI-deferred portal.run path unchanged. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 ++ src/execnet/_provision.py | 23 +++++++++-- src/execnet/_trio_host.py | 63 +++++++++++++++++++++++++++-- src/execnet/_trio_worker.py | 54 ++++++++++++++++++++++++- src/execnet/gateway_base.py | 3 +- src/execnet/multi.py | 24 ++++++++++- testing/test_gevent.py | 81 +++++++++++++++++++++++++++++++++++++ uv.lock | 6 ++- 8 files changed, 246 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aedd6b59..6e12c8da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ classifiers = [ execnet-socketserver = "execnet.script.socketserver:main" [project.optional-dependencies] +gevent = [ + "gevent", +] testing = [ "pre-commit", "pytest>8.0", diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index ffe9ffaa..4da4f689 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,10 +164,12 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet + # the gevent profile parks its greenlets on gevent wakeners + default_wait = "gevent" if spec.execmodel == "gevent" else "thread" config: dict[str, Any] = { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, - "wait": spec.wait or "thread", + "wait": spec.wait or default_wait, "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through @@ -203,13 +205,28 @@ def _uv_tokens(python: str | None) -> list[str]: return prefix +def _extra_with_tokens(config: str) -> list[str]: + """Additional ``--with`` requirements the worker env needs. + + Derived from the worker config itself so every uv launcher (popen, + ssh, via sub-spawn) provisions the same: the gevent profile / wait + backend needs gevent importable in the worker. + """ + parsed = json.loads(config) + if parsed.get("execmodel") == "gevent" or parsed.get("wait") == "gevent": + return ["--with", "gevent"] + return [] + + def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + config = worker_cli_arg(spec) return [ *_uv_tokens(spec.python), "--with", coordinator_requirement(), - *worker_module_tokens(spec), + *_extra_with_tokens(config), + *_worker_tokens(config), ] @@ -228,7 +245,7 @@ def _remote_shell_command( wheel bytes are returned as the preamble to stream before the protocol. """ worker = _worker_tokens(config) - uv = _uv_tokens(python) + uv = [*_uv_tokens(python), *_extra_with_tokens(config)] if wheel is None: assert requirement is not None return shlex.join([*uv, "--with", requirement, *worker]), b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 9a1f1338..597cbdb4 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -111,7 +111,7 @@ async def _wait() -> int | None: return code try: - return self._session.host.call(_wait) + return self._session.host_call(_wait) except Exception: return process.returncode @@ -126,7 +126,7 @@ async def _kill() -> None: await process.wait() try: - self._session.host.call(_kill) + self._session.host_call(_kill) except Exception as exc: trace("ERROR killing trio process:", exc) @@ -293,6 +293,21 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Blocking host call that parks correctly for the wait backend. + + ``wait=thread`` keeps the KI-deferred ``portal.run`` path; other + backends (gevent) wait on a OneShot with the gateway's wakener so + only the calling greenlet parks, not the whole hub. + """ + gateway = self.sync_gateway + if gateway._wait_backend == "thread": + return self.host.call(async_fn, *args) + pending = self.host.call_pending( + async_fn, *args, wakener=gateway._new_wakener() + ) + return pending.wait() + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. @@ -428,6 +443,39 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: return self.portal.run(async_fn, *args) + def call_pending( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + wakener: Any = None, + ) -> OneShot[T]: + """Run ``async_fn`` as a host task, resolving a :class:`OneShot`. + + The non-blocking counterpart of :meth:`call` for consumers that + must not block their OS thread (a gevent hub: waiting on the + OneShot with a gevent wakener parks only the calling greenlet). + Unlike ``portal.run`` the wait is KeyboardInterrupt-interruptible. + """ + result: OneShot[T] = OneShot(wakener) + + async def runner() -> None: + try: + value = await async_fn(*args) + except trio.Cancelled: + if not result.is_set(): + result.set_error(RuntimeError("trio host was shut down")) + raise + except BaseException as exc: + result.set_error(exc) + else: + result.set(value) + + def spawn() -> None: + self.start_soon(runner) + + self.portal.post(spawn) + return result + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: return self.portal.run_sync(sync_fn, *args) @@ -556,7 +604,16 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - bridge = host.call(async_group.makegateway, spec) + if spec.wait and spec.wait != "thread": + # e.g. a gevent app: wait on a OneShot so only the calling + # greenlet parks while the gateway comes up, not the whole hub. + from ._boundary import make_wakener + + bridge = host.call_pending( + async_group.makegateway, spec, wakener=make_wakener(spec.wait) + ).wait() + else: + bridge = host.call(async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c6042bb8..d8c6cc0e 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -145,13 +145,63 @@ async def run(self, channel: Channel, item: ExecItem) -> None: self._primary_busy = False +class GreenletExec: + """Exec strategy: greenlets on a gevent hub owning the main thread. + + ``execmodel=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's ``wait=gevent`` wakeners make channel + operations park the greenlet, so concurrent remote_execs cooperate on + the one main thread. Requires gevent in the worker environment + (provisioning adds the ``gevent`` requirement automatically). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + from ._boundary import make_wakener + + self.gateway = gateway + # The integrate loop blocks in get() on the hub thread: a gevent + # wakener parks only its root greenlet, letting exec greenlets run. + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox(make_wakener("gevent")) + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Run the hub on the main thread, spawning a greenlet per exec.""" + import gevent + + def run_exec(channel: Channel, item: ExecItem, done: threading.Event) -> None: + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + while True: + task = self._primary.get() + if task is None: + break + gevent.spawn(run_exec, *task) + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# Future placement strategies slot in here (e.g. subinterpreters). WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": HybridExec, "main_thread_only": MainExec, + "gevent": GreenletExec, } diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 77fd81df..2b183c1e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -125,9 +125,10 @@ def Event(self) -> threading.Event: #: worker profiles: where exec'd code runs relative to the protocol loop EXECMODEL_PROFILES = ( - "thread", # exec on pool threads, loop on a side thread + "thread", # hybrid: primary on the main thread, overflow on pool threads "main_thread_only", # exec serialized on the main thread (GUI/pytest) "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec ) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 87b17786..7c0810a7 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -248,13 +248,35 @@ def terminate(self, timeout: float | None = None) -> None: # each with a GATEWAY_TERMINATE + timeout grace, then kill; # bounded at roughly twice the timeout (issues #43 / #221). try: - self._trio_host.call(self._async_group.terminate, timeout) + self._host_terminate(timeout) except Exception as exc: trace("group terminate error:", exc) for gw in self._gateways_to_join: gw.join() self._gateways_to_join[:] = [] + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for wait backends. + + A member gateway created with a non-thread ``wait=`` implies the + caller may be a greenlet: wait on a OneShot instead of blocking + the OS thread (which would stall the hub for the whole grace). + """ + backends = {gw._wait_backend for gw in self._gateways_to_join} | { + gw._wait_backend for gw in self + } + backends.discard("thread") + if backends: + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(backends.pop()), + ).wait() + else: + self._trio_host.call(self._async_group.terminate, timeout) + def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, diff --git a/testing/test_gevent.py b/testing/test_gevent.py index e95edfc7..91f74ef4 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -84,3 +84,84 @@ def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: channel = gevent_gw.remote_exec("channel.send(1)") assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) + + def test_makegateway_parks_greenlet_not_hub(self) -> None: + # management ops (makegateway/terminate) from a greenlet must not + # stall the hub: they wait on a OneShot with a gevent wakener. + group = execnet.Group() + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + + try: + ticker = gevent.spawn(other) + maker = gevent.spawn(group.makegateway, "popen//wait=gevent") + gw = maker.get(timeout=TESTTIMEOUT) + channel = gw.remote_exec("channel.send(42)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 + ticker.get(timeout=TESTTIMEOUT) + assert progressed == [0, 1, 2, 3, 4] + finally: + gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + + +class TestGeventWorkerProfile: + """execmodel=gevent: exec'd code runs as greenlets on the main-thread hub.""" + + @pytest.fixture + def worker_gw(self): + group = execnet.Group() + try: + yield group.makegateway("popen//execmodel=gevent") + finally: + group.terminate(timeout=5.0) + + def test_execs_are_greenlets_on_main_thread(self, worker_gw) -> None: + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = worker_gw.remote_exec(report) + second = worker_gw.remote_exec(report) + # both run concurrently on the one main thread: greenlets + assert first.receive(TESTTIMEOUT) is True + assert second.receive(TESTTIMEOUT) is True + first.send(None) + second.send(None) + first.waitclose(TESTTIMEOUT) + second.waitclose(TESTTIMEOUT) + + def test_execs_cooperate_via_gevent(self, worker_gw) -> None: + # the first exec parks in channel.receive() (gevent wakener) while + # the second completes -- with a blocked hub this would deadlock. + blocked = worker_gw.remote_exec("channel.send(channel.receive())") + side = worker_gw.remote_exec( + """ + import gevent + gevent.sleep(0.01) + channel.send('side') + """ + ) + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_status_reports_gevent(self, worker_gw) -> None: + assert worker_gw.remote_status().execmodel == "gevent" + + +def test_provisioning_adds_gevent_requirement() -> None: + from execnet._provision import _extra_with_tokens + from execnet._provision import worker_cli_arg + from execnet.xspec import XSpec + + spec = XSpec("popen//id=g1//execmodel=gevent") + config = worker_cli_arg(spec) + assert '"wait": "gevent"' in config + assert _extra_with_tokens(config) == ["--with", "gevent"] + plain = worker_cli_arg(XSpec("popen//id=g2//execmodel=thread")) + assert _extra_with_tokens(plain) == [] diff --git a/uv.lock b/uv.lock index b885a4b7..3b8bb46e 100644 --- a/uv.lock +++ b/uv.lock @@ -387,6 +387,9 @@ dependencies = [ ] [package.optional-dependencies] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "asyncssh" }, { name = "hatch" }, @@ -411,6 +414,7 @@ testing = [ [package.metadata] requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, + { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -419,7 +423,7 @@ requires-dist = [ { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] -provides-extras = ["testing"] +provides-extras = ["gevent", "testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] From 87c58430e31f7c849db7cef2fbb35d788a6324be Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:35:25 +0200 Subject: [PATCH 47/66] docs: record the landed Phase C worker profiles in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 74 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 40c222b6..01382e39 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,44 +71,48 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) +## Phase C — LANDED: use-case worker profiles on `execmodel=` (2026-07-26) -Predates below gap analysis; where they conflict, this section wins. -Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) -landed first — `wait=` axis exists, ExecModel is a preset, the sync -Channel is a facade over the async core. +Where this conflicts with anything below or elsewhere, this section +wins. The axes framing (`loop=`/`exec=` as spec keys, a reserved +`backend=` axis) was **dropped** in the final rethink with Ronny: +different use-cases get named profiles, `execmodel=` is the public mode +key (xdist already passes it), and `wait=` is the only other public +knob. Implemented in commits `b2f43c3..cbce183`: -**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). -The main thread is the scarce resource (signals, interrupt_main, GUI, -tests calling asyncio.run/trio.run themselves): - -| loop= | exec= | main thread | exec'd code | channel | consumer | +| `execmodel=` | loop thread | exec'd code runs | channel | worker wait | extra deps | |---|---|---|---|---|---| -| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | -| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | -| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | -| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | -| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | -| main | main | — | — | — | invalid | - -Interaction rule: exec'd code may start its own event loop only when it -does not share a thread with ours (exec=thread / exec=main); exec=task -sources are ``async def`` and join our loop — a sync source under -exec=task is a hard error. ``wait=`` composes orthogonally. - -**Backend decisions**: the core STAYS trio-only and trio stays the -default and a hard dependency (pure-python wheels; deployment cost -accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; -asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands -as a per-gateway spec axis, but reserved: only ``trio`` validates -(exactly how ``wait=`` landed with only ``thread``), keeping an -asyncio-core door open without paying for it. Consequence: exec=task -sources are trio-typed. - -Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + -presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → -C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; -`execmodel=main_thread_only` → `loop=thread, exec=main`. +| `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | +| `main_thread_only` | side thread | main thread, serialized (deadlock guard) | sync | thread | — | +| `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | +| `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | + +Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating +to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: +PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec +serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync +bridge at all in the trio profile). Subinterpreters: future strategy +slot, not built. `EXECMODEL_PROFILES` (gateway_base) validates +coordinator-side in makegateway. + +**Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) +answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the +worker config JSON and apply at startup. Coordinator bookkeeping can no +longer claim an exec slot (previously an info call could occupy the main +thread so pytest landed on a worker thread) — `rinfo_source` and the +post-start remote_exec setup block are gone. + +Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a +host task) backs makegateway / `SyncIOHandle.wait/kill` / +`Group.terminate` whenever the wait backend is not `thread`, so a gevent +app's management ops park only the calling greenlet. `wait=thread` +keeps the KI-deferred `portal.run` path. + +Still decided/standing: core stays trio-only (anyio/asyncio-core port +rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd +code may start its own loop in every profile except `trio`. + +## Phase C — original gap analysis (pre-decision record, superseded) ## Phase C — original gap analysis (pre-decision record) From 3031810da1bcf6d907d067565c6e9125a5438383 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:04 +0200 Subject: [PATCH 48/66] feat!: run channel callbacks in an off-loop consumer task setcallback no longer delivers inline on the loop/reader thread and no longer pins the channel in a ChannelFactory._callback_channels registry. Instead Channel.setcallback -> BaseGateway._start_channel_consumer -> SyncBridgeGateway.attach_consumer starts a per-channel consumer *task* on the host loop that drains the channel into an inbox and runs each callback in a trio.to_thread pool thread (bounded by TrioHost.callback_limiter). The task holds a strong reference to the channel, so a callback channel's lifecycle is bound to consumption (and to GC once the stream closes) rather than to a strong-ref registry. Delivery keeps flowing through the existing _sync_payload -> Channel._deliver_payload path (diverted to the inbox once a consumer is attached), so nothing rebinds the raw channel and there is no race with the bind posted at newchannel(). waitclose() now waits on a new Channel._consumer_done flag for callback channels, so it still returns only after every callback -- including the endmarker -- has run. A slow callback no longer blocks the reader, and callbacks for different channels run concurrently (per-channel order kept). Deleted: Channel._callback/_endmarker/_fire_endmarker, ChannelFactory. _callback_channels/_register_callback_channel, and the callback branch of _deliver_payload (now uniformly mailbox.put); __del__'s CHANNEL_LAST_MESSAGE case folds away (a callback channel is never GC'd while open). Resolves the standing "callbacks on the loop thread" open decision (moved). Co-Authored-By: Claude Opus 4.8 (1M context) --- handoff-boundary-protocol-rethink.md | 6 +- handoff-phase-c-worker-axes.md | 54 +++++++- src/execnet/_trio_host.py | 181 +++++++++++++++++++++++++-- src/execnet/gateway_base.py | 181 +++++++++++++++------------ testing/test_channel.py | 44 +++++++ 5 files changed, 369 insertions(+), 97 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 59cd1141..70146c7d 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -139,8 +139,10 @@ namespaces + the axes/presets. one portal FIFO. - After close: `OSError("cannot send (already closed?)")`. - exec admission order = message arrival order (TrioWorkerExec._pump). -- Channel callbacks run on the loop thread (revisit-later stands; the - kit makes moving them cheap). +- Channel callbacks run off the loop thread: a per-channel consumer task + drains into a threadpool call (moved 2026-07-26; the kit made it cheap, + as predicted). See `handoff-phase-c-worker-axes.md` "Callback consumer + tasks". - `Group.terminate(timeout)` never hangs (~2×timeout). - Sync blocking waits stay KeyboardInterrupt-interruptible on the main thread (thread Wakener keeps the Event.wait + drain pattern). diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 01382e39..102cb78f 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -63,8 +63,12 @@ File map (src/execnet/): - exec admission order = message arrival order (`TrioWorkerExec._pump`; `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — trio shuffles its run batch, so never rely on task-spawn order). -- Channel callbacks run on the receiver (loop) thread — keep for now - (open decision: revisit in D). +- Channel callbacks now run in a threadpool thread driven by a per-channel + consumer *task* on the loop (RESOLVED 2026-07-26, was the "revisit in D" + open decision — moved off the loop). A slow callback no longer blocks the + reader; per-channel order is still strict, and `waitclose()` still returns + only after every callback (incl. the endmarker) has run. See "Callback + consumer tasks" below. - `Group.terminate(timeout)` never hangs (~2×timeout bound, issues #43/#221). - Sync blocking waits (send-ack, receive, waitclose, join) stay on @@ -167,8 +171,8 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; this branch plus real `-n` smoke runs (crash/endmarker tests, `main_thread_only` GUI case). Our own suite running `-n 12` green is necessary but not sufficient. -4. **Close the open decision**: channel callbacks on the loop thread — - keep or move. +4. **Open decision CLOSED (2026-07-26)**: channel callbacks moved off the + loop thread — see "Callback consumer tasks" below. 5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 consistent XPASSes were investigated — trio's single-loop dispatch + FIFO admission makes them pass reliably when idle, but under @@ -185,6 +189,48 @@ compat mapping unblocks the ExecModel retirement, then `exec=task`; run the xdist verification (D.3) mid-C as an early canary rather than only at the end. +## Callback consumer tasks (`setcallback`, LANDED 2026-07-26) + +`setcallback` no longer delivers inline on the loop thread and no longer +pins the channel in a `ChannelFactory._callback_channels` registry. Instead +`Channel.setcallback` → `BaseGateway._start_channel_consumer` → +`SyncBridgeGateway.attach_consumer`, which on the loop: + +- moves any already-buffered mailbox items into a fresh per-channel inbox + (a `trio` memory channel), diverts future raw payloads there + (`raw.set_consumer(inbox.send_nowait, on_close)`), and nulls `_mailbox`; +- starts a consumer *task* on the host root nursery (`_run_consumer`) that is + handed a **strong** reference to the sync `Channel` — so the channel's + lifecycle is now bound to the task (and to GC once the stream closes), + replacing the strong-ref registry. + +`_run_consumer` does `async for data in inbox:` and runs each callback via +`trio.to_thread.run_sync(..., limiter=host.callback_limiter)` — off the loop, +one thread of a bounded pool (`DEFAULT_CALLBACK_THREADS=40`), sequential per +channel (order preserved), concurrent across channels. A raising callback +(or a `loads_internal` failure) → `CHANNEL_CLOSE_ERROR` + local close +(`_consumer_failed`). On EOF/close the endmarker fires (shielded, bounded by +`CONSUMER_ENDMARKER_GRACE`) and the task sets `channel._consumer_done`. + +Key invariant preserved: `waitclose()` waits on `_consumer_done` (not +`_receiveclosed`) for callback channels, so it still returns only after every +callback — including the endmarker — has run (`test_waiting_for_callbacks`, +`test_channel_endmarker_callback`). Deleted along the way: `_callback`, +`_endmarker`, `_fire_endmarker`, `_callback_channels`, +`_register_callback_channel`, and the callback branch of `_deliver_payload` +(now uniformly `mailbox.put`). `__del__`'s `CHANNEL_LAST_MESSAGE` case folds +away (a callback channel is never GC'd while open). + +Stress coverage: `testing/test_channel_stress.py` (Hypothesis) with a +`--stress=N` pytest option (profiles registered in +`testing/conftest.py::pytest_configure`; default quick profile). Hypothesis +also surfaced and we fixed a latent serializer bug: `_save_integral` only +bounds-checked the upper int4 limit, so a negative int below `-2**31` +overflowed `struct.pack('!i', …)` instead of taking the long path +(`FOUR_BYTE_INT_MIN` added; regression `test_serializer.test_int_boundaries`). + +`hypothesis` was added to the `testing` extra in `pyproject.toml`. + ## Invariants (do not regress) - No source shipping, ever. Workers import installed execnet+trio; diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 597cbdb4..e119ca68 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,8 @@ import functools import itertools import json +import math +import queue as _queue import subprocess import sys import threading @@ -23,6 +25,7 @@ import trio +from ._boundary import Flag from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup @@ -31,6 +34,8 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args +from .gateway_base import ENDMARKER +from .gateway_base import NO_ENDMARKER_WANTED from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -43,6 +48,17 @@ from .portal import LoopPortal from .portal import OneShot +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 +#: bound on how long the endmarker callback may run during host shutdown +CONSUMER_ENDMARKER_GRACE = 10.0 + + +def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> None: + """Deserialize one payload and invoke the receiver callback (in a thread).""" + callback(loads_internal(data, channel)) + + if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway @@ -255,16 +271,21 @@ def bind_sync_channel(self, channel: Any) -> None: """ ref = weakref.ref(channel) channelid = channel.id + with suppress(trio.RunFinishedError): + self.host.portal.post(self._install_sync_consumer, ref, channelid) - def bind() -> None: - raw = self._channel_for(channelid) - raw.set_consumer( - functools.partial(self._sync_payload, ref), - functools.partial(self._sync_close, ref, channelid), - ) + def _install_sync_consumer(self, ref: weakref.ref[Any], channelid: int) -> None: + """Route ``channelid``'s raw payloads/close to the sync channel (loop). - with suppress(trio.RunFinishedError): - self.host.portal.post(bind) + Idempotent: re-binding to the same hooks just re-flushes the (empty) + raw buffer, so ``attach_consumer`` can call this itself instead of + depending on the separately-posted bind having run first. + """ + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: channel = ref() @@ -293,6 +314,141 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + # -- receiver callbacks: a consumer task per channel -- + + def attach_consumer( + self, + channel: Any, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Switch ``channel`` to callback mode: a loop task drains it. + + The switch runs on the loop (so it cannot interleave with delivery): + it moves any already-buffered items into the task's inbox, points the + channel's delivery/close at that inbox, and starts the consumer task. + It deliberately does *not* touch the raw channel's consumer (the sync + payload/close hooks bound by :meth:`bind_sync_channel` stay in place); + delivery keeps flowing through ``Channel._deliver_payload`` / + ``_close_from_remote``, which divert to the inbox once ``_has_consumer`` + is set. Rebinding the raw channel here would race the still-queued + ``bind()`` post (``run_sync`` and ``run_sync_soon`` are not mutually + ordered) and could be clobbered back to the mailbox. + + The task is handed a strong reference to ``channel`` and thus keeps it + alive for as long as it consumes -- the channel's lifecycle is bound to + the task (and to GC once the stream closes), not to a registry. + """ + + def switch() -> None: + mailbox = channel._mailbox + if mailbox is None: + raise OSError(f"{channel!r} has callback already registered") + inbox_send, inbox_recv = trio.open_memory_channel[bytes](math.inf) + + def feed(data: bytes) -> None: + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + inbox_send.send_nowait(data) + + def close_inbox() -> None: + with suppress(trio.ClosedResourceError): + inbox_send.close() + + # Drain items buffered before the switch into the task's inbox, + # preserving order. An ENDMARKER means the channel already closed. + saw_end = False + while True: + try: + item = mailbox.get_nowait() + except _queue.Empty: + break + if item is ENDMARKER: + saw_end = True + break + inbox_send.send_nowait(item) + channel._mailbox = None + done = Flag(channel.gateway._new_wakener()) + channel._consumer_done = done + channel._consumer_feed = feed + channel._consumer_close_inbox = close_inbox + + def stop() -> None: + # thread-safe: end the task's inbox from any thread (local close) + with suppress(trio.RunFinishedError, trio.ClosedResourceError): + self.host.portal.post(inbox_send.close) + + channel._consumer_stop = stop + channel._has_consumer = True + + # Guarantee the raw channel routes to us right now (idempotent with + # the bind posted at newchannel()): otherwise, if that bind has not + # run yet, inbound payloads would buffer unread in the raw channel + # and the consumer task would wait forever. + self._install_sync_consumer(weakref.ref(channel), channel.id) + + if saw_end: + close_inbox() + self.host.start_soon( + self._run_consumer, channel, inbox_recv, callback, endmarker, done + ) + + self.run_on_loop(switch) + + async def _run_consumer( + self, + channel: Any, + inbox: trio.MemoryReceiveChannel[bytes], + callback: Callable[[Any], Any], + endmarker: object, + done: Flag, + ) -> None: + """Drain ``inbox`` into ``callback`` (each call off the loop thread). + + Runs on the host loop; ``channel`` is held for the task's lifetime so + the channel stays alive while consuming. Items are delivered in order + and each callback runs in a threadpool thread. On completion (EOF, + local close, or a raising callback) the endmarker fires and ``done`` + is set -- which is what ``waitclose()`` waits on. + """ + limiter = self.host.callback_limiter + try: + async for data in inbox: + try: + await trio.to_thread.run_sync( + functools.partial(_run_callback, callback, data, channel), + limiter=limiter, + ) + except Exception as exc: + # trio.Cancelled is a BaseException and propagates past + # here (host shutdown); only a real callback/deserialize + # failure closes the channel with the error. + self._consumer_failed(channel, exc) + break + finally: + # Fire the endmarker and signal done even while the host is being + # torn down, but never let a stuck callback hang shutdown forever. + with trio.CancelScope(shield=True): + if endmarker is not NO_ENDMARKER_WANTED: + with ( + trio.move_on_after(CONSUMER_ENDMARKER_GRACE), + suppress(BaseException), + ): + await trio.to_thread.run_sync( + functools.partial(callback, endmarker), limiter=limiter + ) + done.set() + + def _consumer_failed(self, channel: Any, exc: BaseException) -> None: + """A callback (or its deserialization) raised: close with the error.""" + gateway = self.sync_gateway + gateway._trace("exception during callback: %s" % exc) + errortext = gateway._geterrortext(exc) + with suppress(OSError): + gateway._send( + Message.CHANNEL_CLOSE_ERROR, channel.id, dumps_internal(errortext) + ) + channel._close_from_remote(RemoteError(errortext), sendonly=False) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: """Blocking host call that parks correctly for the wait backend. @@ -406,6 +562,7 @@ def __init__(self, name: str = "execnet-trio-host") -> None: self._ready = threading.Event() self._shutdown: trio.Event | None = None self._started = False + self._callback_limiter: trio.CapacityLimiter | None = None def start(self) -> None: if self._started: @@ -422,6 +579,13 @@ def portal(self) -> LoopPortal: raise RuntimeError("TrioHost is not running") return self._portal + @property + def callback_limiter(self) -> trio.CapacityLimiter: + """Bound on concurrent threadpool threads running receiver callbacks.""" + if self._callback_limiter is None: + raise RuntimeError("TrioHost is not running") + return self._callback_limiter + def is_host_thread(self) -> bool: return self._portal is not None and self._portal.is_loop_thread() @@ -431,6 +595,7 @@ def _run(self) -> None: async def _main(self) -> None: self._portal = LoopPortal() self._shutdown = trio.Event() + self._callback_limiter = trio.CapacityLimiter(DEFAULT_CALLBACK_THREADS) try: async with trio.open_nursery() as nursery: self._nursery = nursery diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2b183c1e..8eaa770e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -13,7 +13,6 @@ import builtins import os -import queue as _queue import struct import sys import threading @@ -371,6 +370,23 @@ class Channel: TimeoutError = TimeoutError _INTERNALWAKEUP = 1000 _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; gateway_base stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None def __init__(self, gateway: BaseGateway, id: int) -> None: """:private:""" @@ -380,10 +396,8 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - # serialized payloads (or ENDMARKER); None once a callback is set + # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) - self._callback: Callable[[Any], Any] | None = None - self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] @@ -398,37 +412,19 @@ def setcallback( ) -> None: """Set a callback function for receiving items. - All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver (loop) thread - for each received data item and calls to ``receive()`` will - raise an error. - If an endmarker is specified the callback will eventually - be called with the endmarker when the channel closes. + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. """ - - def switch() -> None: - # Runs on the loop thread (inline without a session), so the - # switch-over cannot interleave with payload delivery. - mailbox = self._mailbox - if mailbox is None: - raise OSError(f"{self!r} has callback already registered") - self._mailbox = None - while 1: - try: - olditem = mailbox.get_nowait() - except _queue.Empty: - if not (self._closed or self._receiveclosed.is_set()): - self._callback = callback - self._endmarker = endmarker - self.gateway._channelfactory._register_callback_channel(self) - break - if olditem is ENDMARKER: - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - callback(loads_internal(olditem, self)) - - self.gateway._run_on_loop(switch) + self.gateway._start_channel_consumer(self, callback, endmarker) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -454,17 +450,16 @@ def __del__(self) -> None: # in which case the process will go away and we probably # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. if Message is not None: - if self._mailbox is None: # has_callback - msgcode = Message.CHANNEL_LAST_MESSAGE - else: - msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending # Never wait during GC: post the close best-effort. send = getattr( self.gateway, "_send_nonblocking", self.gateway._send ) - send(msgcode, self.id) + send(Message.CHANNEL_CLOSE, self.id) with suppress(Exception): self.gateway._release_channel(self.id) @@ -482,46 +477,43 @@ def _getremoteerror(self): # loop-side delivery (called by the session's raw-channel consumer) # def _deliver_payload(self, data: bytes) -> None: - """Route one inbound serialized payload (loop thread).""" + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ if self._closed: return # late data for a locally closed channel: drop - callback = self._callback - if callback is None: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(data) - # no mailbox and no callback: closed for receiving -- drop - else: - try: - callback(loads_internal(data, self)) - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) - ) - self._close_from_remote(RemoteError(errortext)) + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: - """Close initiated by the peer or session shutdown (loop thread).""" + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ if remoteerror: self._remoteerrors.append(remoteerror) - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) if not sendonly: # otherwise #--> "sendonly" self._closed = True # --> "closed" self._receiveclosed.set() - def _fire_endmarker(self) -> None: - callback = self._callback - if callback is not None: - self._callback = None - if self._endmarker is not NO_ENDMARKER_WANTED: - callback(self._endmarker) - # # public API for channel objects # @@ -588,10 +580,16 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) self.gateway._release_channel(self.id) @@ -611,9 +609,16 @@ def waitclose(self, timeout: float | None = None) -> None: self.TimeoutError is raised after the specified number of seconds (default is None, i.e. wait indefinitely). """ - # wait for non-"opened" state - self._receiveclosed.wait(timeout=timeout) - if not self._receiveclosed.is_set(): + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): raise self.TimeoutError("Timeout after %r seconds" % timeout) error = self._getremoteerror() if error: @@ -693,16 +698,14 @@ class ChannelFactory: Message routing lives in the Trio session (the sync channel binds a consumer on the session's raw channel); the factory only tracks live channels -- weakly, so dropping the last user reference triggers - ``Channel.__del__``'s close message -- and keeps channels with a - registered callback strongly alive until they close. + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. """ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # channels kept strongly alive while their callback is registered - self._callback_channels: dict[int, Channel] = {} self._writelock = threading.Lock() self.gateway = gateway self.count = startcount @@ -739,12 +742,8 @@ def channels(self) -> list[Channel]: # # internal methods, called from the loop thread (or local close paths) # - def _register_callback_channel(self, channel: Channel) -> None: - self._callback_channels[channel.id] = channel - def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: """Close ``id`` as if the peer had closed it (no message is sent).""" @@ -885,6 +884,22 @@ def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: return sync_fn() return session.run_on_loop(sync_fn) + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + def _terminate_execution(self) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index f06b8632..f1b0b454 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -305,6 +305,50 @@ def f(item): channel.send(1) channel.waitclose() + @needs_early_gc + def test_callback_channel_collected_after_close(self, gw: Gateway) -> None: + # A callback channel is kept alive only by its consumer task: while it + # is consuming it survives with no user reference, and once the stream + # closes and the callback has run the object becomes collectable. + import gc + import weakref + + received: list[int] = [] + channel = gw.remote_exec("channel.send(1); channel.send(2)") + channel.setcallback(received.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.05) + assert received == [1, 2] + assert ref() is None # consumer finished -> no strong refs -> reclaimed + + def test_callbacks_run_off_the_loop_thread(self, gw: Gateway) -> None: + # A slow callback on one channel must not block delivery to another: + # callbacks run in threadpool threads, not inline on the loop. + import threading + + release = threading.Event() + fast_ran = threading.Event() + + def slow(item: object) -> None: + release.wait(TESTTIMEOUT) + + chan_slow = gw.remote_exec("channel.send(1); channel.receive()") + chan_fast = gw.remote_exec("channel.send(1)") + chan_slow.setcallback(slow) + chan_fast.setcallback(lambda item: fast_ran.set()) + + # the fast callback fires even while the slow one is still blocking + assert fast_ran.wait(TESTTIMEOUT) + release.set() + chan_slow.send(0) # let the remote finish and close + chan_slow.waitclose(TESTTIMEOUT) + chan_fast.waitclose(TESTTIMEOUT) + class TestChannelFile: def test_channel_file_write(self, gw: Gateway) -> None: From 34c9953dadc2c1eba3888e23de3e2faf39086772 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:55 +0200 Subject: [PATCH 49/66] fix: serialize ints below the signed int4 minimum via the long opcode _save_integral only bounds-checked the upper int4 limit, so a negative int below -2**31 took the short (INT/LONG) path and overflowed struct.pack('!i', ...) with a struct.error instead of using the arbitrary-precision long opcode. Gate the short path on both ends (FOUR_BYTE_INT_MIN..MAX). Found by the new channel-callback Hypothesis stress tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/gateway_base.py | 5 ++++- testing/test_serializer.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8eaa770e..c59f4437 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1051,6 +1051,7 @@ def bchr(n: int) -> bytes: DUMPFORMAT_VERSION = bchr(2) FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 FLOAT_FORMAT = "!d" FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) @@ -1407,7 +1408,9 @@ def _write_byte_sequence(self, bytes_: bytes) -> None: self._write(bytes_) def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - if i <= FOUR_BYTE_INT_MAX: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: self._write(short_op) self._write_int4(i) else: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index cb2d422c..87adf04c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -126,6 +126,26 @@ def test_long(load, dump) -> None: assert v == really_big +@pytest.mark.parametrize( + "value", + [ + "2147483647", # int4 max: short path + "-2147483648", # int4 min: short path + "2147483648", # just over max: long path + "-2147483649", # just under min: long path (used to crash in struct.pack) + "9223372036854775807324234", + "-9223372036854775807324234", + ], +) +def test_int_boundaries(value, dump, load) -> None: + # regression: negative ints below the signed int4 minimum must take the + # arbitrary-precision long path instead of overflowing the 4-byte pack. + p = dump(value) + tp, v = load(p) + assert tp == "int" + assert v == value + + def test_bytes(dump, load) -> None: p = dump("b'hi'") tp, v = load(p) From 0d36f7fd8a05524e3dbba03e4f669d7921e7b8dc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 50/66] test: hypothesis stress tests for channel callbacks with a --stress knob testing/test_channel_stress.py hammers the setcallback consumer-task path with randomised traffic: ordering/completeness, many-channel isolation, receive-then-switch, endmarker-last, and GC-reclaim-after-close. A --stress=N pytest option scales the number of Hypothesis examples (profiles registered in conftest.pytest_configure; a quick profile runs by default). hypothesis added to the testing extra. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + testing/conftest.py | 37 ++++++++++ testing/test_channel_stress.py | 123 +++++++++++++++++++++++++++++++++ uv.lock | 75 ++++++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 testing/test_channel_stress.py diff --git a/pyproject.toml b/pyproject.toml index 6e12c8da..f29b24d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ testing = [ "pre-commit", "pytest>8.0", "pytest-timeout", + "hypothesis", "tox", "hatch", "uv", diff --git a/testing/conftest.py b/testing/conftest.py index 7c3ed7a6..becf10f2 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -69,6 +69,43 @@ def pytest_addoption(parser: pytest.Parser) -> None: "page on invalid addresses" ), ) + group.addoption( + "--stress", + action="store", + dest="stress", + default=None, + metavar="N", + help=( + "how hard the Hypothesis stress tests try: number of examples " + "per test (e.g. --stress=500). Without it a quick profile runs." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + # Register Hypothesis profiles scaled by --stress. The stress tests reuse + # a function-scoped gateway across examples on purpose (spawning one per + # example would dominate the runtime), and each round-trip can be slow, so + # the health checks for those are suppressed. + try: + from hypothesis import HealthCheck + from hypothesis import settings + except ImportError: + return + common = dict( + deadline=None, + suppress_health_check=[ + HealthCheck.function_scoped_fixture, + HealthCheck.too_slow, + ], + ) + settings.register_profile("execnet-quick", max_examples=15, **common) + stress = config.getoption("stress") + if stress is not None: + settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.load_profile("execnet-stress") + else: + settings.load_profile("execnet-quick") @pytest.fixture diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py new file mode 100644 index 00000000..8090367d --- /dev/null +++ b/testing/test_channel_stress.py @@ -0,0 +1,123 @@ +"""Hypothesis stress tests for the channel callback (consumer-task) machinery. + +These hammer ``setcallback`` -- the loop-task-plus-threadpool consumer -- with +randomised traffic to check the invariants that must hold no matter the timing: + +* every item reaches the callback, exactly once and in send order; +* many callback channels run concurrently without cross-talk; +* switching to a callback after some ``receive()`` calls loses nothing; +* the endmarker is always delivered last; +* a callback channel with no user reference is kept alive by its consumer. + +Use ``--stress=N`` to raise the number of examples per test (default: a quick +profile registered in ``conftest.pytest_configure``). +""" + +from __future__ import annotations + +import gc +import weakref + +import pytest + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from execnet.gateway import Gateway # noqa: E402 + +# High --stress levels replay one test many times; lift the per-test timeout +# well above the default so that only a real hang (bounded by TESTTIMEOUT on +# every blocking call below) fails, not sheer example count. +pytestmark = pytest.mark.timeout(600) + +TESTTIMEOUT = 10.0 + +# Varied payloads: unbounded ints exercise both the short and long serializer +# paths (and their boundaries), on top of the callback machinery itself. +payload_strategy = st.one_of( + st.integers(), + st.text(max_size=20), + st.booleans(), + st.none(), +) +items_strategy = st.lists(payload_strategy, max_size=40) + + +def _echo(channel, items): + """Remote: send every item, in order, then let the channel close.""" + for item in items: + channel.send(item) + + +class TestCallbackStress: + # popen only: fast to spawn and the callback path is transport-independent + gwtype = "popen" + + @given(data=items_strategy) + def test_callback_receives_all_in_order( + self, gw: Gateway, data: list[int] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + channel.waitclose(TESTTIMEOUT) + assert collected == data + + @given(batches=st.lists(items_strategy, min_size=1, max_size=6)) + def test_many_channels_stay_ordered_and_isolated( + self, gw: Gateway, batches: list[list[int]] + ) -> None: + results: list[list[int]] = [[] for _ in batches] + channels = [] + for index, items in enumerate(batches): + channel = gw.remote_exec(_echo, items=items) + channel.setcallback(results[index].append) + channels.append(channel) + for channel in channels: + channel.waitclose(TESTTIMEOUT) + assert results == batches + + @given(data=st.data()) + def test_receive_then_switch_loses_nothing( + self, gw: Gateway, data: st.DataObject + ) -> None: + items = data.draw(items_strategy) + split = data.draw(st.integers(min_value=0, max_value=len(items))) + channel = gw.remote_exec(_echo, items=items) + first = [channel.receive(TESTTIMEOUT) for _ in range(split)] + rest: list[int] = [] + channel.setcallback(rest.append) + channel.waitclose(TESTTIMEOUT) + assert first + rest == items + + @given(data=items_strategy) + def test_endmarker_is_always_last(self, gw: Gateway, data: list[int]) -> None: + endmarker = object() + collected: list[object] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append, endmarker=endmarker) + channel.waitclose(TESTTIMEOUT) + assert collected == [*data, endmarker] + + @pytest.mark.skipif( + "not hasattr(sys, 'getrefcount')", reason="needs refcount GC semantics" + ) + @given(data=st.lists(payload_strategy, min_size=1, max_size=40)) + def test_callback_channel_kept_alive_then_collected( + self, gw: Gateway, data: list[object] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + import time + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.02) + assert collected == data + assert ref() is None # consumer finished -> reclaimed diff --git a/uv.lock b/uv.lock index 3b8bb46e..311eb0d8 100644 --- a/uv.lock +++ b/uv.lock @@ -393,6 +393,7 @@ gevent = [ testing = [ { name = "asyncssh" }, { name = "hatch" }, + { name = "hypothesis" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-timeout" }, @@ -416,6 +417,7 @@ requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, + { name = "hypothesis", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, @@ -680,6 +682,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z" }, ] +[[package]] +name = "hypothesis" +version = "6.161.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/94/d208ced653376e7e0a2f0429ee5be864dd0b59393b98a8b41a35ceb4d035/hypothesis-6.161.5.tar.gz", hash = "sha256:ba73a3c3b68e63a0bee5ea1a8a13efce60bcc7ee5fc7e71df2954db39c225b95", size = 486653, upload-time = "2026-07-25T14:39:34.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/58/e48aa878d119474631fc097d511b5e70807dfe56be4b244cce0275f1805e/hypothesis-6.161.5-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4c89e0c35d7cd70af2a0a5f0b5ad69d1898c369adf15115c6d4271d47bf1b280", size = 766970, upload-time = "2026-07-25T14:38:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ff/c83c1a4d5d3c4b6a4dc1fcb5069245171b7a30ad5dd31f44282e8881e089/hypothesis-6.161.5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1bb987e519a6d00675bab124c925000637e2e59384196b0ded5d108dc1851449", size = 762574, upload-time = "2026-07-25T14:38:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/d2/04/7a5ba16cd14340009d1d8aa46083bf3a3a55c5b0d1ccf553e6f6748ee0e8/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5990e9e2e145ff5369c95ad684bd6eee7ebd2d4de37d37eea4c6ca5e339e2a52", size = 1091754, upload-time = "2026-07-25T14:38:38.562Z" }, + { url = "https://files.pythonhosted.org/packages/70/69/031913f7408ebb41e31a9b20e5bca75c5a64ab151d57ca75e944a09ba6e0/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2a5c6400c58c9c3d616ad7177ada12ae577e5103b3b0df6e79920729250bf100", size = 1120372, upload-time = "2026-07-25T14:38:18.962Z" }, + { url = "https://files.pythonhosted.org/packages/b9/95/360884e86ab99099340fca781531286c26d40ce32c853097e76537cc0726/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71aa718e08bdfbacd1a5d8f86ffd55f1d86e64a57cec6a107144d883b522823e", size = 1141230, upload-time = "2026-07-25T14:39:03.757Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5b/6f3c9fbe9191432f33c9aaf951cf5a5416533848999f2e2c6a20ec00098b/hypothesis-6.161.5-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:66912038467e1af791f76336bc079d669f7c8bbf7aeb85bdd52e83259d885122", size = 1096611, upload-time = "2026-07-25T14:38:34.688Z" }, + { url = "https://files.pythonhosted.org/packages/32/c7/d126b9f66295fed5d5745f98ad92f485c98dee303dd67ea7a53fe4a903aa/hypothesis-6.161.5-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99d8b2380d0bba602df963d6e42d56bb50cf1734376c1b9b14dcab3cec21814e", size = 1133382, upload-time = "2026-07-25T14:38:20.27Z" }, + { url = "https://files.pythonhosted.org/packages/56/de/277a17091687f298079c197cadd806d64908c5a08c9c91bb27b834192503/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:19e57e621c6abd98123a91bd4b022bde7760ba709f1ca121f822d9aa291bd001", size = 1265592, upload-time = "2026-07-25T14:38:11.734Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/3f15b1a43cb70b223427ce3a9a6afbe430bf1754b3346ac546a3df38b96b/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b74cced62995ed2e0096fa63d0b7babd1fa08ce1944cff41134275516848708", size = 1393424, upload-time = "2026-07-25T14:38:46.738Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1d/a3cf1441550dbf5a362dcfa19c831721f3bb6a749eb53ecbdfc983959027/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:fd37a0257647288be8c27d56a78a31da2b65e3b6254511f03164f5e1c786187a", size = 1266197, upload-time = "2026-07-25T14:39:33.052Z" }, + { url = "https://files.pythonhosted.org/packages/57/e5/2616432d6144057b6c8f4409c95d95610bdbdf07fecb6618e98761861c24/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf9dfba054164e481f05774c70b65ea88ba735b986cdc44f5210ef40871a32e9", size = 1308330, upload-time = "2026-07-25T14:39:27.86Z" }, + { url = "https://files.pythonhosted.org/packages/f7/cd/9e9c7a88858f84447e44e45c8b9fd1058120b4d5fa9bab9c3ceb9b9cf96b/hypothesis-6.161.5-cp310-abi3-win32.whl", hash = "sha256:8763938787e6c98e461f8c11139981597d083e6915a9956db1c26cc609bc7b19", size = 652819, upload-time = "2026-07-25T14:38:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/03/23/aae37b5bb2014e0e01e372498f7f5977b9fd61a0179143854b7fb0538b1c/hypothesis-6.161.5-cp310-abi3-win_amd64.whl", hash = "sha256:79b5d069095a726dca5b6b7e4c5d268acc809a6595c46d7eee860708f960cb8f", size = 658970, upload-time = "2026-07-25T14:39:05.239Z" }, + { url = "https://files.pythonhosted.org/packages/70/77/fbda7005f891f534af0a74cb5acd7a67cb1a7fbee1ae170d176c7dcd5e0a/hypothesis-6.161.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c73c86afe87b1bf81af586402d41bff69d93c8c34517a81d3a88a1e28e8662e2", size = 767688, upload-time = "2026-07-25T14:39:13.418Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8d/f884e10029018504504df6100faf3795918e1dd14d4b6f4102c3fcb21acf/hypothesis-6.161.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b289d33f0154850baed802d088564bba83dae58bae6fb9157e498c43d758afd", size = 763399, upload-time = "2026-07-25T14:38:10.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/75d9c1f7ad8a52a14c5981c9ba97b1d4abfd263cd48ce7ffb036ce1f5a16/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60afc424627a4119d89f02425049a97e500e022aef6a06d76e987e266e3d720", size = 1092278, upload-time = "2026-07-25T14:38:51.187Z" }, + { url = "https://files.pythonhosted.org/packages/be/52/efc7c45352636fc71d24590893e52f0e8b4ca42984aea870b47831720fef/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:defa4f24aa62b3a6d07271a4d54f31c7bb07ce1f3fa86cc0705890a071a58301", size = 1141822, upload-time = "2026-07-25T14:38:48.298Z" }, + { url = "https://files.pythonhosted.org/packages/6b/48/697b194412fed03cbf9757771b9c3b51ca062e1d78fe2b5ddf833a25b7f2/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f5e8be93cf0da10b8a2f1044c12f2aa506e80c5071e84dffeec4311fddeb8d7", size = 1266206, upload-time = "2026-07-25T14:39:31.414Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b8/56c7996272a8310737c44378fe181befbe3ab41f72ee3d7180e32269cb7c/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:46aa94fa4107f97760b63c83142cd2fe208f38a6708736494cabfcf219b137cb", size = 1308570, upload-time = "2026-07-25T14:39:09.772Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ba/ed5baa948085a08ed38362e0e88ddad7a4a524b0a2a1581c8138ce7b307a/hypothesis-6.161.5-cp310-cp310-win_amd64.whl", hash = "sha256:44e25c35123a2b77ebd2df356e1a78bf2c9abbf875222ff252b3012801bfb143", size = 658822, upload-time = "2026-07-25T14:38:45.402Z" }, + { url = "https://files.pythonhosted.org/packages/9d/82/60f7213dd5262863646bbf31ca28e1dba68730080c14f39744d110733932/hypothesis-6.161.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6d55821d13890875d5a50a1433448c8a86f7f83e85103a527dad232d18c470e", size = 767446, upload-time = "2026-07-25T14:38:39.904Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/145fe198d155ac394cac067ae852074adbca2c015a7244fdb3df2f655c19/hypothesis-6.161.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecb5878b81beb1dfda4e573375a816b1ffa7931d7899a1a2d7216afa5e1efb4f", size = 763215, upload-time = "2026-07-25T14:38:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c0/0ea73426ba5b2504d4e3f4e6c7589f5808ad6eef4c922f15766c0537bbc2/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bff9062c1af02e391c0e48aa75f989393651c6e07f1b50eddeadaa8ca440f944", size = 1092115, upload-time = "2026-07-25T14:39:18.121Z" }, + { url = "https://files.pythonhosted.org/packages/dc/65/2a52daac1d39d0a1a88f96a602ff997e4665c5a9acc4d029e3168e536cd1/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:570ab5afc34dd7a78e4d1ab1a0ea4e026bbae6caee79dc04245d0347f1d548a7", size = 1141577, upload-time = "2026-07-25T14:38:55.841Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/77693dbb6766345d980fdfa41f7e7d3d0a676469d156c8e9455c1ebda67d/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7541433984c5b5fb4ad433a715808579cba16ebaca6a214a99d2f8c35ed49d", size = 1265879, upload-time = "2026-07-25T14:38:21.759Z" }, + { url = "https://files.pythonhosted.org/packages/ed/af/ea72a466210a43b35f3e9d86350aca94529a00d2aff698b4cdd490238955/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f3118cce6e66366e2c64389a5dd6e9bc49c697c366d62709e6fda0b3a8a7d997", size = 1308593, upload-time = "2026-07-25T14:38:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/50/8c/d0ae6738baed9061ca29b18de506f5bc8dc84649b216b55213474ec7da9b/hypothesis-6.161.5-cp311-cp311-win_amd64.whl", hash = "sha256:d8cbd7c938b191d5f9bf846a79e81484135a239cd45bf194993949645495ee6f", size = 658671, upload-time = "2026-07-25T14:38:54.453Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/71a53545f2732574bdd7a45a6e073043bce3447fec4aa722211ebf742f2e/hypothesis-6.161.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:37016b6b4842993b06ee453dda4dedac5ef328cefa6daa36372d261abf12db43", size = 768565, upload-time = "2026-07-25T14:38:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/e6/56/7f32ba1443d5819b324444e7d5ebcf207ba4a0f9219a1693a7994293fd3f/hypothesis-6.161.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d60a204b86936b29d8914f93a99c59a2ecd4e311be793bf32a3f94774d41e016", size = 760188, upload-time = "2026-07-25T14:38:07.894Z" }, + { url = "https://files.pythonhosted.org/packages/07/0d/699436929d2980103c9ddba153872ebed55cfcc13f0f78c1741ac6128205/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cee9f1a563830a9137b9d1505c9c4fac760bc43dfbfa5873bebefe36c8dca4ec", size = 1090570, upload-time = "2026-07-25T14:39:00.492Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a5/2c28b58d16443fd8ad63e4f287440291a42135073748e5e511084f32a6f3/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7fcf44f9876b7b4fdc40b607342219c634cf6a1acc708dc0ff88e8e48ae8ea2", size = 1140601, upload-time = "2026-07-25T14:39:21.403Z" }, + { url = "https://files.pythonhosted.org/packages/4e/fe/f0f87c38dc741687bace80553478f8c9796ba5b6e68793a80990d79ec5ac/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:59088f6458d2a6ef04724a2a639418c97dd8b8c280bfd222fcc5566854560caf", size = 1263341, upload-time = "2026-07-25T14:38:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/02e30bcdb3434f4eee8fdbebf5cf151fecd37d2a76f2fc19f2745c93ca38/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea8e7e6bed5407ab902026d8cac74d10cac894a496d82e261a0beaca499114dc", size = 1307604, upload-time = "2026-07-25T14:39:19.867Z" }, + { url = "https://files.pythonhosted.org/packages/1d/70/1f6349a244f13c5a383a62ca674d3175018c7fc30fbaad10faa859d5c2cf/hypothesis-6.161.5-cp312-cp312-win_amd64.whl", hash = "sha256:a4ad11f4eafd561a672cb9fa977d0f58ab4ae9cf4b160dd3c12c351089f3e2be", size = 656103, upload-time = "2026-07-25T14:38:57.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/07/86f98ebd945f3f8a821c1e7bf9b530902675145d93ed44fa56a309cc24fd/hypothesis-6.161.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3b4c308ff19741ab9f795cac61cce61227f55a1b9767ad525a34de8b01391d1d", size = 768455, upload-time = "2026-07-25T14:38:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/70/11/93c00ba5c77b886017ea8eaa42de8ca937e319032c4251b0540fc1838ae1/hypothesis-6.161.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef16fac46cd5675504a4d3824f443f6842b80fae8b6d15ecffa9c90e0fcf4522", size = 760099, upload-time = "2026-07-25T14:38:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/86/13/c8787cfae81c816976c1b3aaa43294c3abd6fd055a0147122edd7357a349/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4c6aab8dbabb98b9c5ca8da8fa9294d2e5173432a12abcb447e83e666fa430", size = 1090486, upload-time = "2026-07-25T14:39:26.243Z" }, + { url = "https://files.pythonhosted.org/packages/f8/be/48f4f56bfb24787aa27e5ae851d1aee5214ab6e95311cf8c88a56707941f/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f485e3c7ead1f76a8c060b16c6b50e5615264819e95b794c6fc6882a9cbfdc4", size = 1140433, upload-time = "2026-07-25T14:38:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/50/c119bfec24d267e4af5bdefda8fd490f7b3a7ee36a52d5ac0b88e07c10b3/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:75bb899b2db5c45dbfa6ef704c26259bfd27fb3418179c3bd5788b7f2ecce72d", size = 1263296, upload-time = "2026-07-25T14:38:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/81/89/eeb97a9684e3eaae801dd538058202b0b65c2aaecea0211e17d79f731170/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:894e5d9e2cd97798c3dc2731699098c7f800a7905572db1677e0d27d3b41f541", size = 1307324, upload-time = "2026-07-25T14:38:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/15/9d/e0c64a64721ba169dff5a6f12236d5102f8c76f661598d7837591ec7b375/hypothesis-6.161.5-cp313-cp313-win_amd64.whl", hash = "sha256:62452bb19d73496a74919d82afc6306bf9bd42b8cf1dfce21da3802c596a59b1", size = 656067, upload-time = "2026-07-25T14:38:37.157Z" }, + { url = "https://files.pythonhosted.org/packages/31/27/0c6884785ab6afb41305296b2a5fac2c5d1d26dba40e85c70909b8692fce/hypothesis-6.161.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b41648b547a233dd89bed37a3ce8c8298a454fd7133b27b0a37ced135eddfe", size = 768668, upload-time = "2026-07-25T14:39:15.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/17c63bba85201d5fa9db89841e8548c4fa6b240b2437c80d606a65889eb6/hypothesis-6.161.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:429d2179b01787a54f8ccd150c02077a6feb973f14ef31664227148c6a1fadff", size = 760240, upload-time = "2026-07-25T14:38:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/a2/df/1084fd695a1516120a5ea26c026d9f0a071fdf9efcba774c94c09332b372/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b23fc5441ef297d56797dd372889dec38ddf3662468772b347db2fd8fd43eced", size = 1090987, upload-time = "2026-07-25T14:39:06.646Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a3/bef99daeaf6d1d2db09790cd4ec9ce0cdcac52faaf1fd80b4eaba5d5ea5e/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f42dcc7fee0d4d56d011218b8c4d5afa6cdee5dbc690652d2a1a598d21969a3", size = 1140613, upload-time = "2026-07-25T14:38:26.284Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ce1f6e29d897c7dcb7de1665a9e30d1e00500933caec697a4338db9e7bd0/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2825fac0d0428cb5e7826347aa4e60106d28f32948cd58c837097f3bd96dbf6c", size = 1263802, upload-time = "2026-07-25T14:38:33.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/e5/7b33aae590bc457fffba70c4f19cc150e3ca356f8e15322f9224c5372a64/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e9f859ecfd3e00ebb8a36fbb36a8951513fd8e09103a56c645b102e0a51dd1bc", size = 1307642, upload-time = "2026-07-25T14:39:29.729Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8b/4156a9e70bf8c69f54954539ca805e7311048516b665c5f87bee2c1955b7/hypothesis-6.161.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b2647c2d5da341467c1ffaad0dd86d8612e27da2419e76b3b6ee79a33bba7d86", size = 600146, upload-time = "2026-07-25T14:38:17.815Z" }, + { url = "https://files.pythonhosted.org/packages/f9/db/48a518f3f2facd7b280592f81dbb0ba09109656be4ad3dbf0d95a02b4c02/hypothesis-6.161.5-cp314-cp314-win_amd64.whl", hash = "sha256:7c40dd1a3e99497d48a3cfd4c5d71bdb0cd70402a9e09eceb160f045edc92b3a", size = 655981, upload-time = "2026-07-25T14:39:16.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/92/1e9f9f44ef75fc052cc0e3700cfc7d52fd4af1ed9e8b2945d19956bb83cb/hypothesis-6.161.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6c1b7e1d508a3cc5c10ea7a7a4a7d3b0b56fe6291e8936bb9d052af530380b15", size = 767251, upload-time = "2026-07-25T14:38:52.608Z" }, + { url = "https://files.pythonhosted.org/packages/48/31/0c3f824bbb1fccc0338930c8ef855880065d9d58dddf9547a1b9ae4e664d/hypothesis-6.161.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b0926ba51452c24b92fbcbb30d28cc43fd6cd9e636ea2134fbc1cc2c9da109de", size = 758710, upload-time = "2026-07-25T14:38:09.156Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/5811dbb4c1dc0a772519fb9af9c4089128bceeb5deb4c36a887e1e731920/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba8d0346308b558bfbb49f965ed1d9d5bb65758f4530cb30ce1f9bd911a130b4", size = 1089583, upload-time = "2026-07-25T14:38:49.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/29/f9cbfd9d0aaea5370a7ceb966f6c8d47e9101ee9a6623606e9f564a59c6e/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c64079ef4633ab7b540f532107188d75999ee5b342116636312e09ada15783e", size = 1139525, upload-time = "2026-07-25T14:38:29.262Z" }, + { url = "https://files.pythonhosted.org/packages/99/27/f56a03be4ee21f25828e8fba8a502d6840826aea4ff784ff54cbd1f51175/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed3eede1c23689edb4773b4c2303ee91518e82056a0a4893fc8415b18e5c3e8b", size = 1261963, upload-time = "2026-07-25T14:38:31.931Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ea/72e11fe6832a68674271655be5cb6f035502dd328ea629b3ee510fb130a5/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82b1088ebcad5ab6d146caf9a1552019472a8a9302727f1be5939e30460a101b", size = 1306382, upload-time = "2026-07-25T14:38:13.011Z" }, + { url = "https://files.pythonhosted.org/packages/df/8b/bb039d11a805db0977904c245fa3dcf6d266808482d9dfc6dad3e5f1b256/hypothesis-6.161.5-cp314-cp314t-win_amd64.whl", hash = "sha256:bbe8705ff394a573624cd53f2192dad6255c86346704adf9873cb3acb9b940e5", size = 656131, upload-time = "2026-07-25T14:38:16.712Z" }, + { url = "https://files.pythonhosted.org/packages/21/71/0b28e9ac10f692d9b85a0df74615d308ec1e77140c61773ab107070b82d3/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:64167d94bcd4a15c1b0aa44c3176f019614898ff10907a2065c142ed34acf828", size = 768375, upload-time = "2026-07-25T14:39:23Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/f518bdc85ee5a9b9153aa01b3ed6a77304b8f573a461da1a6694a6992001/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:22bc641a290428cfd01c62b2b7fe7686007b104dfb5e8ed5420fd3d3a281ebb9", size = 764276, upload-time = "2026-07-25T14:39:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/af/3b/6c93b03adb5804b01854d2801a1f74705ea73d685cef7861484ad8804856/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35527a795992396b82293fec2294ba5b9c0768350c8cb89085e3f5f76ed7d08e", size = 1093089, upload-time = "2026-07-25T14:39:11.442Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/6ebfbd76534a71c7ffdd48e0d42373a14152e45714feabf0c25bd7d6b10a/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae81035628d2b80435320c43cb09f5b421bac4941bd8ea9b2778d278ae65996", size = 1142876, upload-time = "2026-07-25T14:39:24.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/b57cfd34f55c14a2e823284a796d4228c356e338e1195affc0546bca567b/hypothesis-6.161.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:20e3ab3b0dc300a84b58f41ddd7f5190757521282aad58701c4c76f28a3d981a", size = 659768, upload-time = "2026-07-25T14:39:08.166Z" }, +] + [[package]] name = "identify" version = "2.6.12" From f911ce7589d7e748725c8a5ff3639a4cb0c5cea9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 51/66] refactor: name the main_thread_only deadlock-admission timeout Extract MAIN_THREAD_ONLY_ADMIT_TIMEOUT (unchanged, 1s) so the deadlock window is documented in one place. It stays small: a genuine second concurrent remote_exec never returns, so this only bounds how fast that is reported. main_thread_only is an xdist-only transitional profile; the whole guard goes away with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/_trio_worker.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index d8c6cc0e..4f3a65f2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -59,13 +59,23 @@ def trigger_shutdown(self) -> None: pass +#: How long admission waits for the previous main-thread exec to finish +#: before declaring a deadlock. Kept small: a genuine second concurrent +#: remote_exec never returns, so this only affects how fast that is +#: reported. (Under heavy CPU contention a merely slow predecessor can be +#: misread as a deadlock, but main_thread_only is an xdist-only transitional +#: profile slated for removal, so the whole guard goes away with it.) +MAIN_THREAD_ONLY_ADMIT_TIMEOUT = 1.0 + + class MainExec: """Exec strategy: serialize each request onto the process main thread. The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist runs this way). Admission waits for the previous request to finish and - closes the channel with the deadlock text when it cannot within a - second (a second concurrent remote_exec would deadlock the requester). + closes the channel with the deadlock text when it cannot within + ``MAIN_THREAD_ONLY_ADMIT_TIMEOUT`` (a second concurrent remote_exec would + deadlock the requester). """ needs_primary_thread = True @@ -79,7 +89,9 @@ def __init__(self, gateway: WorkerGateway) -> None: async def admit(self, channel: Channel, item: ExecItem) -> bool: complete = self.gateway._executetask_complete assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) + wait_slot = functools.partial( + complete.wait, timeout=MAIN_THREAD_ONLY_ADMIT_TIMEOUT + ) if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) return False From 8eee047f5e4b87f0dbce30a9c63d555bb4189804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 52/66] feat!: hide the standalone serializer from the public API dumps/loads/dump/load are no longer re-exported on any public namespace (execnet, execnet.sync, execnet.trio, execnet.aio); they remain internal to execnet.gateway_base. A channel carries only simple builtin data (plus channel references) -- encoding rich objects is the caller's job, and execnet deliberately does not provide or manage a codec. The serializer error types (DataFormatError / DumpError / LoadError) stay exposed on every surface for `except` and documentation, and now spell out that hitting one is a caller error to resolve (encode the value to simple data first), pointing at pydantic model_dump/model_validate and pytest's report to/from-serializable hooks as the mechanisms to use. Docs: the stale "Cross-interpreter serialization" section (which documented the removed execnet.dumps, with an autofunction + doctest that would now break the build) becomes "Sending objects over a channel" guidance; the dumps/loads anchors are kept so existing refs resolve. BREAKING CHANGE: execnet.dumps, execnet.loads, execnet.dump and execnet.load are removed from the public API (use your own encoding, e.g. pydantic or the pytest report hooks). Shipping with the trio backend as a major release. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/basics.rst | 56 ++++++++++++++++++++++++++++--------- doc/index.rst | 8 +++--- src/execnet/__init__.py | 8 ------ src/execnet/aio.py | 14 +++------- src/execnet/gateway_base.py | 33 ++++++++++++++++++++-- src/execnet/sync.py | 8 ------ src/execnet/trio.py | 14 +++------- testing/test_basics.py | 24 ++++++++++------ testing/test_namespaces.py | 5 ++-- testing/test_serializer.py | 2 +- 10 files changed, 105 insertions(+), 67 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index a6b256b7..94f43ad8 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -216,23 +216,53 @@ configure a tracing mechanism: .. _`dumps/loads`: .. _`dumps/loads API`: +.. _`serialization`: -Cross-interpreter serialization of Python objects +Sending objects over a channel ======================================================= -.. versionadded:: 1.1 +A channel carries only **simple builtin data**: ``None``, ``bool``, +``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily nested +``list`` / ``tuple`` / ``set`` / ``frozenset`` / ``dict`` of those -- plus +**channel references**, which arrive as channels on the peer. That is the +entire contract. -Execnet exposes a function pair which you can safely use to -store and load values from different Python interpreters -(e.g. Python2 and Python3, PyPy and Jython). Here is -a basic example:: +execnet does **not** pickle and does **not** encode rich objects for you: +arbitrary instances, functions, ``datetime``, dataclasses, pydantic models, +numpy arrays, enums, etc. have no wire representation. This is deliberate; +encoded / rich-object channels are out of scope for execnet. - >>> import execnet - >>> dump = execnet.dumps([1,2,3]) - >>> execnet.loads(dump) - [1,2,3] +Sending an unsupported value raises ``DumpError`` (a subclass of +``DataFormatError``); a corrupt or protocol-mismatched payload on receive +raises ``LoadError``. These signal a **caller error to resolve** -- reduce +the value to simple data before sending -- not a transport failure. The +standalone serializer itself is an internal implementation detail +(``execnet.gateway_base``) and is not part of the public API. -For more examples see :ref:`dumps/loads examples`. +Encode rich objects yourself +------------------------------------------------------- -.. autofunction:: execnet.dumps(spec) -.. autofunction:: execnet.loads(spec) +Turning a rich object into simple data (and back) is the caller's job. Use +an established encoding mechanism rather than expecting the channel to do it: + +- **pydantic**: ``model.model_dump(mode="json")`` reduces a model to simple + data (``datetime`` -> ISO string, ``UUID`` / ``Enum`` / ``Decimal`` -> + primitives); ``Model.model_validate(...)`` rebuilds it on the other side. + ``TypeAdapter`` covers non-model types. + + :: + + channel.send(model.model_dump(mode="json")) + # peer: + model = MyModel.model_validate(channel.receive()) + +- **pytest** does exactly this above execnet: pytest-xdist ships + ``TestReport`` objects with the ``pytest_report_to_serializable`` / + ``pytest_report_from_serializable`` hooks (rich report <-> simple dict) + around ``channel.send`` / ``channel.receive``. + +- **stdlib**: ``dataclasses.asdict(obj)``, ``dt.isoformat()`` / + ``datetime.fromisoformat``, or ``json`` with a ``default=`` hook. + +Channels are the one non-builtin you *can* send: nested channel references +pass through intact, so callbacks and sub-streams need no encoding. diff --git a/doc/index.rst b/doc/index.rst index 19a20eb8..ce161b47 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,10 +31,10 @@ Features * Automatic bootstrapping: no manual remote installation. * Safe and simple serialization of Python builtin - types for sending/receiving structured data messages. - (New in 1.1) execnet offers a new :ref:`dumps/loads ` - API which allows cross-interpreter compatible serialization - of Python builtin types. + types for sending/receiving structured data messages; + see :ref:`sending objects over a channel `. + Encoding rich objects is the caller's job (execnet stays + builtin-types-only). * Flexible communication: synchronous send/receive as well as callback/queue mechanisms supported diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 66c61821..b81aaedd 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -34,10 +34,6 @@ from .sync import TimeoutError from .sync import XSpec from .sync import default_group -from .sync import dump -from .sync import dumps -from .sync import load -from .sync import loads from .sync import makegateway from .sync import set_execmodel @@ -56,10 +52,6 @@ "XSpec", "__version__", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/aio.py b/src/execnet/aio.py index bbfe06e3..7151ab5d 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -23,8 +23,10 @@ async def main(): side only -- the host-side task runs to completion (a cancelled ``receive`` may still consume the next item). -The serialization helpers and error types are shared with -:mod:`execnet.sync` and :mod:`execnet.trio`. +The error types are shared with :mod:`execnet.sync` and +:mod:`execnet.trio`. Items you send must already be simple builtin data +(plus channels); the standalone serializer is intentionally not part of +the public API -- see ``DumpError``. """ from __future__ import annotations @@ -53,10 +55,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec if TYPE_CHECKING: @@ -73,10 +71,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", ] diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c59f4437..75e5895a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1033,15 +1033,42 @@ def executetask( class DataFormatError(Exception): - pass + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ class DumpError(DataFormatError): - """Error while serializing an object.""" + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ class LoadError(DataFormatError): - """Error while unserializing an object.""" + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ def bchr(n: int) -> bytes: diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 0d3c835f..e5e026c7 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -14,10 +14,6 @@ from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .multi import Group from .multi import MultiChannel from .multi import default_group @@ -40,10 +36,6 @@ "TimeoutError", "XSpec", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/trio.py b/src/execnet/trio.py index df0ab493..4b712c0a 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -14,8 +14,10 @@ async def main(): trio.run(main) -The serialization helpers and error types are shared with the blocking -API in :mod:`execnet.sync`. +The error types are shared with the blocking API in :mod:`execnet.sync`. +Items you send must already be simple builtin data (plus channels); the +standalone serializer is intentionally not part of the public API -- see +``DumpError``. """ from ._trio_gateway import AsyncChannel @@ -32,10 +34,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec __all__ = [ @@ -52,10 +50,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", "serve_gateway", ] diff --git a/testing/test_basics.py b/testing/test_basics.py index 470bec6d..d5606610 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -28,37 +28,45 @@ ) +# The standalone serializer is an internal detail (execnet.gateway_base), +# not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = execnet.dumps(val) - val2 = execnet.loads(dumped) + dumped = gateway_base.dumps(val) + val2 = gateway_base.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(execnet.dumps(val)) + p.write_bytes(gateway_base.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = execnet.load(m) + val2 = gateway_base.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - execnet.dump(f, val) + gateway_base.dump(f, val) read = BytesIO(f.getvalue()) - val2 = execnet.load(read) + val2 = gateway_base.load(read) assert val == val2 +def test_serializer_not_public() -> None: + # dumps/loads/dump/load are internal to gateway_base only. + for name in ("dumps", "loads", "dump", "load"): + assert not hasattr(execnet, name), name + + def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: bchr = gateway_base.bchr monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = execnet.dumps(42) + dumped = gateway_base.dumps(42) monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) + pytest.raises(execnet.DataFormatError, lambda: gateway_base.loads(dumped)) def test_errors_on_execnet() -> None: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index a8bfef0c..0317bc2b 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -32,9 +32,10 @@ def test_trio_namespace_exposes_async_core() -> None: assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway - # serialization + errors are shared with the sync surface + # error types are shared with the sync surface; the standalone serializer + # is intentionally not exposed on any public namespace assert execnet.trio.RemoteError is execnet.RemoteError - assert execnet.trio.dumps is execnet.dumps + assert not hasattr(execnet.trio, "dumps") def test_portal_namespace() -> None: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 87adf04c..49332c97 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -189,4 +189,4 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: def test_py2_string_loads() -> None: """Regression test for #267.""" - assert execnet.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" + assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" From 6a3371673d492d7a17cb835cd6c29105e6295067 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 53/66] test: type-check the callback stress tests under mypy Add hypothesis to the mypy pre-commit hook's dependencies so @given is typed, and pass Hypothesis settings.register_profile keywords explicitly (a homogeneous **dict tripped the arg-type check). Also a ruff-format tidy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .pre-commit-config.yaml | 1 + testing/conftest.py | 17 +++++++++-------- testing/test_channel_stress.py | 4 +--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7a9f86c..91db1f29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,5 @@ repos: additional_dependencies: - pytest - trio + - hypothesis - types-pywin32 diff --git a/testing/conftest.py b/testing/conftest.py index becf10f2..04dbe631 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -92,17 +92,18 @@ def pytest_configure(config: pytest.Config) -> None: from hypothesis import settings except ImportError: return - common = dict( - deadline=None, - suppress_health_check=[ - HealthCheck.function_scoped_fixture, - HealthCheck.too_slow, - ], + suppress = [HealthCheck.function_scoped_fixture, HealthCheck.too_slow] + settings.register_profile( + "execnet-quick", max_examples=15, deadline=None, suppress_health_check=suppress ) - settings.register_profile("execnet-quick", max_examples=15, **common) stress = config.getoption("stress") if stress is not None: - settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.register_profile( + "execnet-stress", + max_examples=int(stress), + deadline=None, + suppress_health_check=suppress, + ) settings.load_profile("execnet-stress") else: settings.load_profile("execnet-quick") diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py index 8090367d..12f69d47 100644 --- a/testing/test_channel_stress.py +++ b/testing/test_channel_stress.py @@ -55,9 +55,7 @@ class TestCallbackStress: gwtype = "popen" @given(data=items_strategy) - def test_callback_receives_all_in_order( - self, gw: Gateway, data: list[int] - ) -> None: + def test_callback_receives_all_in_order(self, gw: Gateway, data: list[int]) -> None: collected: list[int] = [] channel = gw.remote_exec(_echo, items=data) channel.setcallback(collected.append) From 12fdbbec13564e6576f4c8ab4f284b0b90aea983 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:57:26 +0200 Subject: [PATCH 54/66] refactor!: drop the injected remote shell script execnet.script.shell connected to a socket server, sent repr() of its own source as the first line, and expected the server to exec it with clientsock bound in globals. That handshake belonged to the pre-Trio socket server; the Trio one speaks the execnet frame protocol from the first byte and never execs an incoming line, so the module has been inert since that rewrite. Nothing imports it, no test covers it, and a remote prompt is better served as a remote_exec example than as a socket-protocol side channel. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 ++ src/execnet/script/shell.py | 91 ------------------------------------- 2 files changed, 3 insertions(+), 91 deletions(-) delete mode 100644 src/execnet/script/shell.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eac3c8d6..3e99f1d5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types and greenlet execmodels keep the legacy thread path. +* Removed ``execnet.script.shell``, an interactive remote prompt that injected its own + source into the pre-Trio socket server. The Trio socket server never execs an incoming + source line, so the module could no longer work against it. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/shell.py b/src/execnet/script/shell.py deleted file mode 100644 index de6f8ded..00000000 --- a/src/execnet/script/shell.py +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env python -""" -a remote python shell - -for injection into startserver.py -""" - -import os -import select -import socket -import sys -from threading import Thread -from traceback import print_exc -from typing import NoReturn - - -def clientside() -> NoReturn: - print("client side starting") - host, portstr = sys.argv[1].split(":") - port = int(portstr) - with open(os.path.abspath(sys.argv[0])) as fd: - myself = fd.read() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.sendall((repr(myself) + "\n").encode()) - print("send boot string") - inputlist = [sock, sys.stdin] - try: - while 1: - r, _w, _e = select.select(inputlist, [], []) - if sys.stdin in r: - line = input() - sock.sendall((line + "\n").encode()) - if sock in r: - line = sock.recv(4096).decode() - sys.stdout.write(line) - sys.stdout.flush() - except BaseException: - import traceback - - traceback.print_exc() - - sys.exit(1) - - -class promptagent(Thread): - def __init__(self, clientsock) -> None: - print("server side starting") - super().__init__() # type: ignore[call-overload] - self.clientsock = clientsock - - def run(self) -> None: - print("Entering thread prompt loop") - clientfile = self.clientsock.makefile("w") - - filein = self.clientsock.makefile("r") - loc = self.clientsock.getsockname() - - while 1: - try: - clientfile.write("{} {} >>> ".format(*loc)) - clientfile.flush() - line = filein.readline() - if not line: - raise EOFError("nothing") - if line.strip(): - oldout, olderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = clientfile, clientfile - try: - try: - exec(compile(line + "\n", "", "single")) - except BaseException: - print_exc() - finally: - sys.stdout = oldout - sys.stderr = olderr - clientfile.flush() - except EOFError: - sys.stderr.write("connection close, prompt thread returns") - break - - self.clientsock.close() - - -sock = globals().get("clientsock") -if sock is not None: - prompter = promptagent(sock) - prompter.start() - print("promptagent - thread started") -else: - clientside() From 6718808a37ff099cbe5f57739f3d1aec6deac821 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:58:56 +0200 Subject: [PATCH 55/66] refactor!: drop the quitserver script execnet.script.quitserver stopped a socket server by connecting and sending '"raise KeyboardInterrupt"' for the server to exec. Like the shell script it depended on the pre-Trio exec-a-source-line handshake; against the Trio socket server it is just a malformed first frame. Stopping the server is now the job of whatever supervises the execnet-socketserver process. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 2 ++ src/execnet/script/quitserver.py | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 src/execnet/script/quitserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e99f1d5..1ac8a0ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ * Removed ``execnet.script.shell``, an interactive remote prompt that injected its own source into the pre-Trio socket server. The Trio socket server never execs an incoming source line, so the module could no longer work against it. +* Removed ``execnet.script.quitserver``, which shut a socket server down by sending it + ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/quitserver.py b/src/execnet/script/quitserver.py deleted file mode 100644 index 4c94c383..00000000 --- a/src/execnet/script/quitserver.py +++ /dev/null @@ -1,17 +0,0 @@ -""" - -send a "quit" signal to a remote server - -""" - -from __future__ import annotations - -import socket -import sys - -host, port = sys.argv[1].split(":") -hostport = (host, int(port)) - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(hostport) -sock.sendall(b'"raise KeyboardInterrupt"\n') From 7124fc0cfeb51a65bc0d5454296be1486cd46a4c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:00:37 +0200 Subject: [PATCH 56/66] refactor!: drop the socketserver restart loop script loop_socketserver re-Popen'd "python /socketserver.py" in a while loop, from the days when a single connection ended the server process. The Trio socket server accepts connections in a loop via trio.serve_listeners and only exits after one connection when asked with --once, so the wrapper has nothing left to restart. Its sibling-file lookup also only ever worked from a source checkout, not from an installed execnet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 4 ++++ src/execnet/script/loop_socketserver.py | 14 -------------- 2 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 src/execnet/script/loop_socketserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1ac8a0ef..7bde0c0f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ source line, so the module could no longer work against it. * Removed ``execnet.script.quitserver``, which shut a socket server down by sending it ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. +* Removed ``execnet.script.loop_socketserver``, a restart loop around a sibling + ``socketserver.py`` file. The socket server serves connections in a loop itself + (``--once`` opts out), and the sibling-file path never resolved for an installed + execnet anyway. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/loop_socketserver.py b/src/execnet/script/loop_socketserver.py deleted file mode 100644 index a4688a80..00000000 --- a/src/execnet/script/loop_socketserver.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess -import sys - -if __name__ == "__main__": - directory = os.path.dirname(os.path.abspath(sys.argv[0])) - script = os.path.join(directory, "socketserver.py") - while 1: - cmdlist = ["python", script] - cmdlist.extend(sys.argv[1:]) - text = "starting subcommand: " + " ".join(cmdlist) - print(text) - process = subprocess.Popen(cmdlist) - process.wait() From 47e5b78ead6be7c9d96c9649747c18254f0cb2c4 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:05:36 +0200 Subject: [PATCH 57/66] refactor!: drop the pywin32 windows service wrapper socketserverservice.py registered the socket server as a Windows service via pywin32. pywin32 was never a dependency, nothing imported the module, no test ever ran it, and no entry point installed it -- it was a recipe living in the package. Now that the server is a real console command, any service host can supervise it. Document that instead: an NSSM (or WinSW) wrapper on Windows, a systemd unit on Linux, plus why sc.exe alone will not start a plain console program. While rewriting that section, replace the stale instruction to download socketserver.py from raw.githubusercontent and run it standalone. The Trio server imports execnet and trio, so it is no longer a version-independent single file; "uvx --from execnet execnet-socketserver" is the install-free equivalent. Also document --once, port 0, and that the port is unauthenticated. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 + doc/example/test_info.rst | 65 ++++++++++++++-- src/execnet/script/socketserverservice.py | 90 ----------------------- 3 files changed, 61 insertions(+), 97 deletions(-) delete mode 100644 src/execnet/script/socketserverservice.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7bde0c0f..0b66b38a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,9 @@ ``socketserver.py`` file. The socket server serves connections in a loop itself (``--once`` opts out), and the sibling-file path never resolved for an installed execnet anyway. +* Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. + Wrap the ``execnet-socketserver`` console command with a service host such as NSSM + instead; the socket gateway example documents how. 2.1.2 (2025-11-11) diff --git a/doc/example/test_info.rst b/doc/example/test_info.rst index eefe91b3..7a5ead4f 100644 --- a/doc/example/test_info.rst +++ b/doc/example/test_info.rst @@ -172,15 +172,23 @@ sends back the results. Instantiate gateways through sockets ----------------------------------------------------- -.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/main/src/execnet/script/socketserver.py +In cases where you do not have SSH-access to a machine you need a +bootstrapping-point that listens on a socket. execnet ships one as the +``execnet-socketserver`` console command; run it on the target machine:: -In cases where you do not have SSH-access to a machine -you need to download a small version-independent standalone -`socketserver.py`_ script to provide a remote bootstrapping-point. -You do not need to install the execnet package remotely. -Simply run the script like this:: + execnet-socketserver :8888 # bind to all IPs, port 8888 - python socketserver.py :8888 # bind to all IPs, port 8888 +If execnet is not installed there, uv_ can fetch and run it in one step +without leaving anything behind:: + + uvx --from execnet execnet-socketserver :8888 + +.. _uv: https://docs.astral.sh/uv/ + +The server accepts connections in a loop and serves each one in its own +worker subprocess; pass ``--once`` to serve a single connection and exit. +Passing port ``0`` binds an ephemeral port -- the bound address is printed +on the first line of output either way. You can then instruct execnet on your local machine to bootstrap itself into the remote socket endpoint:: @@ -191,4 +199,47 @@ itself into the remote socket endpoint:: That's it, you can now use the gateway object just like a popen- or SSH-based one. +.. warning:: + + The socket server performs no authentication and its traffic is not + encrypted -- anyone who can reach the port can execute code on that + machine. Bind it to a trusted network only, or tunnel it over SSH. + +Keeping the socket server running ++++++++++++++++++++++++++++++++++ + +``execnet-socketserver`` is an ordinary long-running process, so restarts and +boot-time startup are the job of your platform's service manager. + +On Linux, a systemd unit does it:: + + # /etc/systemd/system/execnet-socketserver.service + [Unit] + Description=execnet socket server + + [Service] + ExecStart=/usr/local/bin/execnet-socketserver :8888 + Restart=always + + [Install] + WantedBy=multi-user.target + +On Windows, wrap the console command with a service host such as NSSM_ or +WinSW_. Use ``where execnet-socketserver`` to find the installed +``execnet-socketserver.exe`` (it lives in the ``Scripts`` directory of the +environment it was installed into), then:: + + nssm install ExecNetSocketServer C:\path\to\Scripts\execnet-socketserver.exe :8888 + nssm set ExecNetSocketServer Start SERVICE_AUTO_START + net start ExecNetSocketServer + +NSSM restarts the process if it exits. Note that ``sc.exe create`` on its own +is not enough: it expects a binary that implements the Windows service control +protocol, which a plain console program does not. If you would rather not +install a service host, a Task Scheduler task with an *At startup* trigger +works too. + +.. _NSSM: https://nssm.cc/ +.. _WinSW: https://github.com/winsw/winsw + .. include:: test_ssh_fileserver.rst diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py deleted file mode 100644 index 1c991307..00000000 --- a/src/execnet/script/socketserverservice.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -A windows service wrapper for the py.execnet socketserver. - -To use, run: - python socketserverservice.py register - net start ExecNetSocketServer -""" - -import sys -import threading - -import servicemanager -import win32event -import win32evtlogutil -import win32service -import win32serviceutil - -from . import socketserver - -appname = "ExecNetSocketServer" - - -class SocketServerService(win32serviceutil.ServiceFramework): - _svc_name_ = appname - _svc_display_name_ = "%s" % appname - _svc_deps_ = ["EventLog"] - - def __init__(self, args) -> None: - # The exe-file has messages for the Event Log Viewer. - # Register the exe-file as event source. - # - # Probably it would be better if this is done at installation time, - # so that it also could be removed if the service is uninstalled. - # Unfortunately it cannot be done in the 'if __name__ == "__main__"' - # block below, because the 'frozen' exe-file does not run this code. - # - win32evtlogutil.AddSourceToRegistry( - self._svc_display_name_, servicemanager.__file__, "Application" - ) - super().__init__(args) - self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) - self.WAIT_TIME = 1000 # in milliseconds - - def SvcStop(self) -> None: - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self.hWaitStop) - - def SvcDoRun(self) -> None: - # Redirect stdout and stderr to prevent "IOError: [Errno 9] - # Bad file descriptor". Windows services don't have functional - # output streams. - sys.stdout = sys.stderr = open("nul", "w") - - # Write a 'started' event to the event log... - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STARTED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("Begin: %s" % self._svc_display_name_) - - hostport = ":8888" - print("Starting py.execnet SocketServer on %s" % hostport) - thread = threading.Thread( - target=socketserver.main, args=([hostport],), daemon=True - ) - thread.start() - - # wait to be stopped or self.WAIT_TIME to pass - while True: - result = win32event.WaitForSingleObject(self.hWaitStop, self.WAIT_TIME) - if result == win32event.WAIT_OBJECT_0: - break - - # write a 'stopped' event to the event log. - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STOPPED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("End: %s" % appname) - - -if __name__ == "__main__": - # Note that this code will not be run in the 'frozen' exe-file!!! - win32serviceutil.HandleCommandLine(SocketServerService) From fa0d7a8b7b94f563372a0c51e393b808f8033c74 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:13:39 +0200 Subject: [PATCH 58/66] refactor: move the socket server to execnet._socketserver The Trio rewrite turned socketserver.py from a copy-me standalone script into ordinary package code: it imports trio and execnet._trio_host, and reaches users as the execnet-socketserver console command. Sitting in an execnet.script package still advertised the old download-and-run story. Move it next to the other private implementation modules (_trio_host, _trio_worker, _provision) and drop the execnet.script package, which is now empty. The console command and its behaviour are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 +++ pyproject.toml | 2 +- src/execnet/{script/socketserver.py => _socketserver.py} | 6 ++++-- src/execnet/script/__init__.py | 1 - 4 files changed, 8 insertions(+), 4 deletions(-) rename src/execnet/{script/socketserver.py => _socketserver.py} (91%) delete mode 100644 src/execnet/script/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b66b38a..3a1e9422 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,9 @@ * Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. Wrap the ``execnet-socketserver`` console command with a service host such as NSSM instead; the socket gateway example documents how. +* Moved the socket server to ``execnet._socketserver`` and removed the now empty + ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged + and remains the supported way to run it. 2.1.2 (2025-11-11) diff --git a/pyproject.toml b/pyproject.toml index f29b24d0..68029e90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ classifiers = [ ] [project.scripts] -execnet-socketserver = "execnet.script.socketserver:main" +execnet-socketserver = "execnet._socketserver:main" [project.optional-dependencies] gevent = [ diff --git a/src/execnet/script/socketserver.py b/src/execnet/_socketserver.py similarity index 91% rename from src/execnet/script/socketserver.py rename to src/execnet/_socketserver.py index eb2a86ae..c27609f6 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/_socketserver.py @@ -1,9 +1,11 @@ -#! /usr/bin/env python """Trio socket server for execnet gateways. Listens on a TCP port and hands each accepted connection (by fd) to a fresh ``python -m execnet._trio_worker`` subprocess that serves the gateway over it. -No code is executed inline. Run directly, e.g. provisioned anywhere with +No code is executed inline. + +This module implements the ``execnet-socketserver`` console command, which is +the supported entry point -- run it on the target host, or install-free with ``uvx --from execnet execnet-socketserver``. """ diff --git a/src/execnet/script/__init__.py b/src/execnet/script/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/src/execnet/script/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# From 871dfd730b3bd80407bdf271844359fed4f04a83 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 16:14:08 +0200 Subject: [PATCH 59/66] refactor!: drop the py2/py3 string coercion machinery reconfigure() configured how str/bytes cross a Python2 <-> Python3 gateway. execnet cannot have such a peer anymore, so remove it: * Gateway.reconfigure, Channel.reconfigure and AsyncChannel.reconfigure, plus the _strconfig attribute they fed. * the py2str_as_py3str / py3str_as_py2str arguments of gateway_base.loads and .load, and the strconfig argument of loads_internal. py2str_as_py3str was already unreachable on Python3 -- it only gates the PY2STRING opcode, which no Python3 serializer emits. * the retired PY2STRING and UNICODE opcodes; PY3STRING becomes STRING. Opcode bytes are unchanged for every surviving type, and M/S stay unassigned. Values dumped by execnet on Python2 no longer load. * the no-op reconfigure() call in rsync, which cost a round-trip per channel to set what was already the default. * doc/example/hybridpython.rst and py3topy2.py, along with the remaining Python2 references in basics.rst and index.rst. The RECONFIGURE message code stays reserved, but is no longer sent or handled. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 9 ++ doc/basics.rst | 8 +- doc/example/hybridpython.rst | 153 ---------------------- doc/example/py3topy2.py | 19 --- doc/examples.rst | 1 - doc/index.rst | 4 - src/execnet/_trio_gateway.py | 32 +---- src/execnet/_trio_host.py | 7 - src/execnet/gateway.py | 12 -- src/execnet/gateway_base.py | 89 ++----------- src/execnet/rsync.py | 1 - testing/test_compatibility_regressions.py | 6 +- testing/test_serializer.py | 7 +- testing/test_trio_gateway.py | 13 -- 14 files changed, 35 insertions(+), 326 deletions(-) delete mode 100644 doc/example/hybridpython.rst delete mode 100644 doc/example/py3topy2.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a1e9422..9f2d3f38 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,15 @@ * Moved the socket server to ``execnet._socketserver`` and removed the now empty ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged and remains the supported way to run it. +* Removed ``Gateway.reconfigure`` and ``Channel.reconfigure`` along with the + ``py2str_as_py3str`` / ``py3str_as_py2str`` arguments of ``gateway_base.loads`` and + ``gateway_base.load``. They configured string coercion between Python2 and Python3 + peers, which execnet can no longer have: ``py2str_as_py3str`` was already unreachable + because only a Python2 serializer emits the opcode it gates. The ``RECONFIGURE`` + message code stays reserved but is no longer sent or handled. +* The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed + ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer + load. Opcode bytes are unchanged for every type that survives. 2.1.2 (2025-11-11) diff --git a/doc/basics.rst b/doc/basics.rst index 94f43ad8..a22ad7de 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -43,8 +43,8 @@ Examples for valid gateway specifications ``default`` via SSH through Vagrant's ``vagrant ssh`` command. It supports the same additional parameters as regular SSH connections. -* ``popen//python=python2.7//nice=20`` specification of - a python subprocess using the ``python2.7`` executable which must be +* ``popen//python=python3.13//nice=20`` specification of + a python subprocess using the ``python3.13`` executable which must be discoverable through the system ``PATH``; running with the lowest CPU priority ("nice" level). By default current dir will be the current dir of the instantiator. @@ -86,10 +86,6 @@ a channel object whose symmetric counterpart channel is available to the remotely executing source. -.. method:: Gateway.reconfigure([py2str_as_py3str=True, py3str_as_py2str=False]) - - Reconfigures the string-coercion behaviour of the gateway - .. _`Channel`: .. _`channel-api`: diff --git a/doc/example/hybridpython.rst b/doc/example/hybridpython.rst deleted file mode 100644 index f4ad465a..00000000 --- a/doc/example/hybridpython.rst +++ /dev/null @@ -1,153 +0,0 @@ -Connecting different Python interpreters -========================================== - -.. _`dumps/loads examples`: - -Dumping and loading values across interpreter versions ----------------------------------------------------------- - -.. versionadded:: 1.1 - -Execnet offers a new safe and fast :ref:`dumps/loads API` which you -can use to dump builtin python data structures and load them -later with the same or a different python interpreter (including -between Python2 and Python3). The standard library offers -the pickle and marshal modules but they do not work safely -between different interpreter versions. Using xml/json -requires a mapping of Python objects and is not easy to -get right. Moreover, execnet allows to control handling -of bytecode/strings/unicode types. Here is an example:: - - # using python2 - import execnet - with open("data.py23", "wb") as f: - f.write(execnet.dumps(["hello", "world"])) - - # using Python3 - import execnet - with open("data.py23", "rb") as f: - val = execnet.loads(f.read(), py2str_as_py3str=True) - assert val == ["hello", "world"] - -See the :ref:`dumps/loads API` for more details on string -conversion options. Please note, that you can not dump -user-level instances, only builtin python types. - -Connect to Python2/Numpy from Python3 ----------------------------------------- - -Here we run a Python3 interpreter to connect to a Python2.7 interpreter -that has numpy installed. We send items to be added to an array and -receive back the remote "repr" of the array:: - - import execnet - gw = execnet.makegateway("popen//python=python2.7") - channel = gw.remote_exec(""" - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) - """) - for x in range(10): - channel.send(x) - channel.send(None) - print (channel.receive()) - -will print on the CPython3.1 side:: - - array([1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - -A more refined real-life example of python3/python2 interaction -is the anyvc_ project which uses version-control bindings in -a Python2 subprocess in order to offer Python3-based library -functionality. - -.. _anyvc: http://bitbucket.org/RonnyPfannschmidt/anyvc/overview/ - - -Reconfiguring the string coercion between python2 and python3 -------------------------------------------------------------- - -Sometimes the default configuration of string coercion (2str to 3str, 3str to 2unicode) -is inconvient, thus it can be reconfigured via `gw.reconfigure` and `channel.reconfigure`. Here is an example session on a Python2 interpreter:: - - - >>> import execnet - >>> execnet.makegateway("popen//python=python3.2") - - >>> gw=execnet.makegateway("popen//python=python3.2") - >>> gw.remote_exec("channel.send('hello')").receive() - u'hello' - >>> gw.reconfigure(py3str_as_py2str=True) - >>> gw.remote_exec("channel.send('hello')").receive() - 'hello' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.send('a') - >>> ch.receive() - 'str' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.reconfigure(py2str_as_py3str=False) - >>> ch.send('a') - >>> ch.receive() - u'bytes' - - -Work with Java objects from CPython ----------------------------------------- - -Use your CPython interpreter to connect to a `Jython 2.5.1`_ interpreter -and work with Java types:: - - import execnet - gw = execnet.makegateway("popen//python=jython") - channel = gw.remote_exec(""" - from java.util import Vector - v = Vector() - v.add('aaa') - v.add('bbb') - for val in v: - channel.send(val) - """) - - for item in channel: - print (item) - -will print on the CPython side:: - - aaa - bbb - -.. _`Jython 2.5.1`: http://www.jython.org - -Work with C# objects from CPython ----------------------------------------- - -(Experimental) use your CPython interpreter to connect to a IronPython_ interpreter -which can work with C# classes. Here is an example for instantiating -a CLR Array instance and sending back its representation:: - - import execnet - gw = execnet.makegateway("popen//python=ipy") - - channel = gw.remote_exec(""" - import clr - clr.AddReference("System") - from System import Array - array = Array[float]([1,2]) - channel.send(str(array)) - """) - print (channel.receive()) - -using Mono 2.0 and IronPython-1.1 this will print on the CPython side:: - - System.Double[](1.0, 2.0) - -.. note:: - Using IronPython needs more testing, likely newer versions - will work better. please feedback if you have information. - -.. _IronPython: http://ironpython.net diff --git a/doc/example/py3topy2.py b/doc/example/py3topy2.py deleted file mode 100644 index 3dcf0437..00000000 --- a/doc/example/py3topy2.py +++ /dev/null @@ -1,19 +0,0 @@ -import execnet - -gw = execnet.makegateway("popen//python=python2") -channel = gw.remote_exec( - """ - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) -""" -) -for x in range(10): - channel.send(x) -channel.send(None) -print(channel.receive()) diff --git a/doc/examples.rst b/doc/examples.rst index 767fa4fc..3f09197a 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -14,7 +14,6 @@ Note: all examples with `>>>` prompts are automatically tested. example/test_group example/test_proxy example/test_multi - example/hybridpython example/test_debug .. toctree:: diff --git a/doc/index.rst b/doc/index.rst index ce161b47..47a918cc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -59,9 +59,6 @@ Known uses project to launch computation processes through ssh. He also compares `disco and execnet`_ in a subsequent post. -* Ronny Pfannschmidt uses it for his `anyvc`_ VCS-abstraction project - to bridge the Python2/Python3 version gap. - * Sysadmins and developers are using it for ad-hoc custom scripting .. _`quora`: http://quora.com @@ -71,7 +68,6 @@ Known uses .. _`distributed testing`: https://pypi.python.org/pypi/pytest-xdist .. _`Distributed NTLK with execnet`: http://streamhacker.com/2009/11/29/distributed-nltk-execnet/ .. _`disco and execnet`: http://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ -.. _`anyvc`: http://bitbucket.org/RonnyPfannschmidt/anyvc/ Project status -------------------------- diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 81ea76d0..3972db4f 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -8,7 +8,7 @@ Two-level channel model: * :class:`RawChannel` (this module) -- id-routed raw byte payload streams - over the gateway: no serialization, no strconfig, no callbacks. + over the gateway: no serialization and no callbacks. ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top decides what the bytes mean. * ``AsyncChannel`` -- the serialized object API layered on a RawChannel. @@ -44,7 +44,6 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import gateway_info from .gateway_base import loads_internal @@ -149,8 +148,6 @@ class RawChannel: the peer drains, hits EOF, and may keep sending to us. """ - _strconfig: tuple[bool, bool] | None = None - def __init__(self, gateway: AsyncGateway, id: int) -> None: self.gateway = gateway self.id = id @@ -405,16 +402,6 @@ async def wait_closed(self) -> None: if error is not None: raise error - async def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for both ends of this channel.""" - strconfig = (py2str_as_py3str, py3str_as_py2str) - self._raw._strconfig = strconfig - await self.gateway._send( - Message.RECONFIGURE, self.id, dumps_internal(strconfig) - ) - def __aiter__(self) -> AsyncChannel: return self @@ -424,12 +411,8 @@ async def __anext__(self) -> Any: except EOFError: raise StopAsyncIteration from None - # Unserializer duck-type: loads_internal(data, self) reads _strconfig - # and _channelfactory off the object to resolve CHANNEL opcodes. - - @property - def _strconfig(self) -> tuple[bool, bool]: - return self._raw._strconfig or self.gateway._strconfig + # Unserializer duck-type: loads_internal(data, self) reads + # _channelfactory off the object to resolve CHANNEL opcodes. @property def _channelfactory(self) -> _AsyncChannelFactory: @@ -475,7 +458,6 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._async_channels: dict[int, AsyncChannel] = {} self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[ tuple[bytes, Callable[[BaseException | None], None] | None] ](math.inf) @@ -693,14 +675,6 @@ def _dispatch(self, message: Message) -> None: Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) ) self._send_nowait(Message.CHANNEL_CLOSE, channelid) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data) - assert isinstance(data, tuple) - if channelid == 0: - self._strconfig = data - else: - # picked up by the serialized channel layer - self._channel_for(channelid)._strconfig = data else: # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core self._trace("rejecting unsupported message", message) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e119ca68..1d9149dd 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -204,13 +204,6 @@ def _dispatch(self, message: Message) -> None: elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - if message.channelid == 0: - gateway._strconfig = data - else: - gateway._channelfactory.new(message.channelid)._strconfig = data elif code == Message.GATEWAY_START_SOCKET: handle_start_socket(gateway, message.channelid, message.data) elif code == Message.GATEWAY_START_SUB: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 672a0830..ac41705c 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -80,18 +80,6 @@ def exit(self) -> None: self._trace("io-error: could not send termination sequence") self._trace(" exception: %r" % exc) - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this gateway. - - The default is to try to convert py2 str as py3 str, but not to try and - convert py3 str to py2 str. - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = gateway_base.dumps_internal(self._strconfig) - self._send(Message.RECONFIGURE, data=data) - def _rinfo(self, update: bool = False) -> RInfo: """Return some sys/env information from remote. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 75e5895a..ce19eb0f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -186,6 +186,8 @@ class only carries the framing and the code constants. """ STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. RECONFIGURE = 1 GATEWAY_TERMINATE = 2 CHANNEL_EXEC = 3 @@ -393,8 +395,6 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: assert isinstance(id, int) assert not isinstance(gateway, type) self.gateway = gateway - # XXX: defaults copied from Unserializer - self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) @@ -672,18 +672,6 @@ def next(self) -> Any: __next__ = next - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this channel. - - The default is to try to convert py2 str as py3 str, - but not to try and convert py3 str to py2 str - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = dumps_internal(self._strconfig) - self.gateway._send(Message.RECONFIGURE, self.id, data=data) - ENDMARKER = object() INTERRUPT_TEXT = "keyboard-interrupted" @@ -838,7 +826,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel self._io = io self.id = id - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) # globals may be NONE at process-termination self.__trace = trace @@ -1106,35 +1093,26 @@ class opcode: NEWDICT = b"J" NEWLIST = b"K" NONE = b"L" - PY2STRING = b"M" - PY3STRING = b"N" + STRING = b"N" SET = b"O" SETITEM = b"P" STOP = b"Q" TRUE = b"R" - UNICODE = b"S" COMPLEX = b"T" class Unserializer: num2func: dict[bytes, Callable[[Unserializer], None]] = {} - py2str_as_py3str = True # True - py3str_as_py2str = False # false means py2 will get unicode def __init__( self, stream: ReadIO, channel_or_gateway: Channel | BaseGateway | None = None, - strconfig: tuple[bool, bool] | None = None, ) -> None: if isinstance(channel_or_gateway, Channel): gw: BaseGateway | None = channel_or_gateway.gateway else: gw = channel_or_gateway - if channel_or_gateway is not None: - strconfig = channel_or_gateway._strconfig - if strconfig: - self.py2str_as_py3str, self.py3str_as_py2str = strconfig self.stream = stream if gw is None: self.channelfactory = None @@ -1219,25 +1197,10 @@ def _read_byte_string(self) -> bytes: as_bytes = self.stream.read(length) return as_bytes - def load_py3string(self) -> None: - as_bytes = self._read_byte_string() - if self.py3str_as_py2str: - # XXX Should we try to decode into latin-1? - self.stack.append(as_bytes) - else: - self.stack.append(as_bytes.decode("utf-8")) - - num2func[opcode.PY3STRING] = load_py3string - - def load_py2string(self) -> None: - as_bytes = self._read_byte_string() - if self.py2str_as_py3str: - s: bytes | str = as_bytes.decode("latin-1") - else: - s = as_bytes - self.stack.append(s) + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) - num2func[opcode.PY2STRING] = load_py2string + num2func[opcode.STRING] = load_string def load_bytes(self) -> None: s = self._read_byte_string() @@ -1245,11 +1208,6 @@ def load_bytes(self) -> None: num2func[opcode.BYTES] = load_bytes - def load_unicode(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.UNICODE] = load_unicode - def load_newlist(self) -> None: length = self._read_int4() self.stack.append([None] * length) @@ -1323,46 +1281,27 @@ def dump(byteio, obj: object) -> None: _Serializer(write=byteio.write).save(obj, versioned=True) -def loads( - bytestring: bytes, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def loads(bytestring: bytes) -> Any: """Deserialize the given bytestring to an object. - py2str_as_py3str: If true then string (str) objects previously - dumped on Python2 will be loaded as Python3 - strings which really are text objects. - py3str_as_py2str: If true then string (str) objects previously - dumped on Python3 will be loaded as Python2 - strings instead of unicode objects. - If the bytestring was dumped with an incompatible protocol version or if the bytestring is corrupted, the ``execnet.DataFormatError`` will be raised. """ - io = BytesIO(bytestring) - return load( - io, py2str_as_py3str=py2str_as_py3str, py3str_as_py2str=py3str_as_py2str - ) + return load(BytesIO(bytestring)) -def load( - io: ReadIO, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def load(io: ReadIO) -> Any: """Derserialize an object form the specified stream. - Behaviour and parameters are otherwise the same as with ``loads`` + Behaviour is otherwise the same as with ``loads`` """ - strconfig = (py2str_as_py3str, py3str_as_py2str) - return Unserializer(io, strconfig=strconfig).load(versioned=True) + return Unserializer(io).load(versioned=True) -def loads_internal( - bytestring: bytes, - channelfactory=None, - strconfig: tuple[bool, bool] | None = None, -) -> Any: +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: io = BytesIO(bytestring) - return Unserializer(io, channelfactory, strconfig).load() + return Unserializer(io, channelfactory).load() def dumps_internal(obj: object) -> bytes: @@ -1420,7 +1359,7 @@ def save_bytes(self, bytes_: bytes) -> None: self._write_byte_sequence(bytes_) def save_str(self, s: str) -> None: - self._write(opcode.PY3STRING) + self._write(opcode.STRING) self._write_unicode_string(s) def _write_unicode_string(self, s: str) -> None: diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index f92bc37a..876e38fc 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -173,7 +173,6 @@ def itemcallback(req) -> None: self._receivequeue.put((channel, req)) channel = gateway.remote_exec(execnet.rsync_remote) - channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) channel.setcallback(itemcallback, endmarker=None) channel.send((str(destdir), options)) self._channels[channel] = finishedcallback diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 5343e7d5..837c0c6d 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -18,13 +18,13 @@ def test_opcodes() -> None: "NEWDICT": b"J", "NEWLIST": b"K", "NONE": b"L", - "PY2STRING": b"M", - "PY3STRING": b"N", + # b"M" (py2 str) and b"S" (py2 unicode) are retired along with + # Python2 support -- the bytes stay unused rather than reassigned. + "STRING": b"N", "SET": b"O", "SETITEM": b"P", "STOP": b"Q", "TRUE": b"R", - "UNICODE": b"S", # added in 1.4 # causes a regression since it was ordered in # between CHANNEL and FALSE as "C" moving the other items diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 49332c97..ec8e86ae 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -187,6 +187,7 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: assert s == "(1, (), 3)" -def test_py2_string_loads() -> None: - """Regression test for #267.""" - assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" +def test_py2_string_opcode_is_retired() -> None: + """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" + with pytest.raises(execnet.DataFormatError, match="unknown opcode"): + execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index a6f27981..ef52e232 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -403,16 +403,3 @@ async def main() -> None: await group.makegateway("id=notype") trio.run(main) - - -def test_channel_reconfigure_string_coercion() -> None: - async def main() -> None: - async with gateway_pair() as (left, right): - sender = left.open_channel() - receiver = right.open_channel(sender.id) - await sender.reconfigure(py3str_as_py2str=True) - await receiver.send("text") - # our side now loads py3 strings as bytes - assert await sender.receive() == b"text" - - trio.run(main) From 740b11c5d85d516d6a58d27e6ea4a881a018c4ac Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 21:29:26 +0200 Subject: [PATCH 60/66] refactor!: make the accidentally-public modules private execnet grew four deliberate public namespaces during the Trio port -- execnet (aliases of execnet.sync), execnet.trio, execnet.aio and execnet.portal -- and every module added since is underscore-prefixed. Six pre-Trio modules never got that treatment and stayed importable as execnet.gateway_base, .gateway, .multi, .rsync, .rsync_remote and .xspec, reachable only because `import execnet` pulled them in transitively. The docs pointed Sphinx at three of them, so they read as public API. Each is now an underscore-prefixed module plus a thin shim under the old name that warns and forwards, removable in execnet 3.0. Both `import execnet.gateway_base` and `execnet.gateway_base.X` after a plain `import execnet` keep working (pytest-xdist does the latter). gateway_base.py held six unrelated concerns in 1449 lines and is split by concern, each layer importing only from the ones above it: _trace EXECNET_DEBUG tracing _errors every exception, plus sysex and the error texts _execmodel the deprecated ExecModel presets _message IO protocols, Message, FrameDecoder _serialize the wire serializer _channel Channel, ChannelFactory, ChannelFile adapters _gateway_base BaseGateway, WorkerGateway Two cycles fell out of that. Unserializer resolved its channel argument with isinstance(Channel); it now duck-types on .gateway, which also subsumes the AsyncChannel._channelfactory property the async core needed. INTERRUPT_TEXT and MAIN_THREAD_ONLY_DEADLOCK_TEXT moved to _errors rather than _channel, since RemoteError.warn() reads the former. Surface changes: * execnet.trio no longer exports ByteStream, RawChannel, RawChannelStream or serve_gateway -- internal routing detail with no caller outside _trio_host. serve_gateway had none at all. * execnet._gateway no longer lists _find_non_builtin_globals and _source_of_function in __all__; they were there for tests only. * Added execnet.can_send(obj), the supported way to ask whether a value can cross a channel. It sits on execnet alone -- the wire contract does not vary by namespace -- so it is not mirrored onto sync/trio/aio. * Removed execnet.loads/dump/load and dropped execnet.dumps from the surface. dumps stays reachable behind a warning, out of __all__ and dir(), purely so released pytest-xdist keeps working; see the FOLLOW-UP note on execnet._XDIST_COMPAT. test_namespaces.py now enforces the boundary: every namespace __all__ resolves, no unexpected non-underscore module exists, and each shim attribute warns and forwards to the same object. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 28 + doc/basics.rst | 23 +- src/execnet/__init__.py | 50 +- src/execnet/_channel.py | 483 +++++++ src/execnet/_errors.py | 109 ++ src/execnet/_execmodel.py | 92 ++ src/execnet/_gateway.py | 158 +++ src/execnet/_gateway_base.py | 237 ++++ src/execnet/_message.py | 174 +++ src/execnet/_multi.py | 419 ++++++ src/execnet/_rsync.py | 248 ++++ src/execnet/_rsync_remote.py | 126 ++ src/execnet/_serialize.py | 439 ++++++ src/execnet/_shim.py | 47 + src/execnet/_trace.py | 47 + src/execnet/_trio_gateway.py | 25 +- src/execnet/_trio_host.py | 28 +- src/execnet/_trio_worker.py | 16 +- src/execnet/_xspec.py | 74 + src/execnet/aio.py | 17 +- src/execnet/gateway.py | 166 +-- src/execnet/gateway_base.py | 1518 +-------------------- src/execnet/multi.py | 427 +----- src/execnet/rsync.py | 246 +--- src/execnet/rsync_remote.py | 127 +- src/execnet/sync.py | 30 +- src/execnet/trio.py | 26 +- src/execnet/xspec.py | 75 +- testing/conftest.py | 6 +- testing/test_basics.py | 113 +- testing/test_channel.py | 4 +- testing/test_channel_stress.py | 2 +- testing/test_compatibility_regressions.py | 4 +- testing/test_execmodel_trio.py | 2 +- testing/test_gateway.py | 25 +- testing/test_gevent.py | 2 +- testing/test_multi.py | 10 +- testing/test_namespaces.py | 120 +- testing/test_rsync.py | 2 +- testing/test_serializer.py | 10 +- testing/test_termination.py | 4 +- testing/test_trio_gateway.py | 12 +- testing/test_xspec.py | 2 +- 43 files changed, 3148 insertions(+), 2625 deletions(-) create mode 100644 src/execnet/_channel.py create mode 100644 src/execnet/_errors.py create mode 100644 src/execnet/_execmodel.py create mode 100644 src/execnet/_gateway.py create mode 100644 src/execnet/_gateway_base.py create mode 100644 src/execnet/_message.py create mode 100644 src/execnet/_multi.py create mode 100644 src/execnet/_rsync.py create mode 100644 src/execnet/_rsync_remote.py create mode 100644 src/execnet/_serialize.py create mode 100644 src/execnet/_shim.py create mode 100644 src/execnet/_trace.py create mode 100644 src/execnet/_xspec.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9f2d3f38..0a966560 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,34 @@ * The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer load. Opcode bytes are unchanged for every type that survives. +* The supported API is now exactly five namespaces: ``execnet`` (aliases of + ``execnet.sync``), ``execnet.sync``, ``execnet.trio``, ``execnet.aio`` and + ``execnet.portal``. The pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, + ``execnet.multi``, ``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` + were only ever reachable because ``import execnet`` pulled them in transitively; they + are now deprecated forwarding shims that warn on attribute access and will be removed + in execnet 3.0. Both ``import execnet.gateway_base`` and ``execnet.gateway_base.X`` + after a plain ``import execnet`` keep working for now. Every name they exposed is + available from a public namespace, except the internals listed below. +* ``gateway_base`` was split into private modules grouped by concern: ``_trace``, + ``_errors``, ``_execmodel``, ``_message`` (IO protocols, ``Message``, + ``FrameDecoder``), ``_serialize``, ``_channel`` (``Channel``, ``ChannelFactory``, + the ``ChannelFile`` adapters) and ``_gateway_base`` (``BaseGateway``, + ``WorkerGateway``). +* Removed ``execnet.loads``, ``execnet.dump`` and ``execnet.load``, and dropped + ``execnet.dumps`` from the public surface. The standalone serializer is internal. + Added ``execnet.can_send(obj)``, which answers whether a value can cross a channel, + for callers that previously probed with ``try: execnet.dumps(x) / except DumpError``. + It lives on ``execnet`` only -- the wire contract does not vary by namespace. + + ``execnet.dumps`` itself stays *reachable* for now, warning on access, purely so + released ``pytest-xdist`` keeps working; it is absent from ``__all__`` and from + ``dir(execnet)``. It is scheduled for removal once xdist ports its probe to + ``can_send``. +* ``execnet.trio`` no longer exports ``ByteStream``, ``RawChannel``, + ``RawChannelStream`` or ``serve_gateway``; the raw-channel layer is internal routing + detail. No names were added to ``execnet.sync``, ``execnet.aio`` or + ``execnet.portal``. 2.1.2 (2025-11-11) diff --git a/doc/basics.rst b/doc/basics.rst index a22ad7de..8378dbc3 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -72,7 +72,7 @@ Examples for valid gateway specifications remote_exec: execute source code remotely =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to execute source code in the instantiated subprocess-interpreter: @@ -94,7 +94,7 @@ is available to the remotely executing source. Channels: exchanging data with remote code ======================================================= -.. currentmodule:: execnet.gateway_base +.. currentmodule:: execnet A channel object allows to send and receive data between two asynchronously running programs. @@ -114,7 +114,7 @@ two asynchronously running programs. Grouped Gateways and robust termination =============================================== -.. currentmodule:: execnet.multi +.. currentmodule:: execnet All created gateway instances are part of a group. If you call ``execnet.makegateway`` it actually is forwarded to @@ -169,7 +169,7 @@ executions concurrently, they will run in non-main threads. remote_status: get low-level execution info =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to obtain some status information from the remote side. @@ -232,8 +232,19 @@ Sending an unsupported value raises ``DumpError`` (a subclass of ``DataFormatError``); a corrupt or protocol-mismatched payload on receive raises ``LoadError``. These signal a **caller error to resolve** -- reduce the value to simple data before sending -- not a transport failure. The -standalone serializer itself is an internal implementation detail -(``execnet.gateway_base``) and is not part of the public API. +standalone serializer itself is an internal implementation detail and is not +part of the public API. + +To branch *before* sending rather than handling the error, ask: + +.. autofunction:: can_send + +:: + + channel.send(value if execnet.can_send(value) else repr(value)) + +It lives on ``execnet`` itself rather than on any one namespace: the wire +contract is the same whichever surface you drive a gateway from. Encode rich objects yourself ------------------------------------------------------- diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index b81aaedd..5912ae9a 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -15,11 +15,15 @@ * :mod:`execnet.portal` — cross-thread / cross-loop communication primitives shared by all of them. +``can_send`` sits here rather than on any one of them: the wire-format +contract is the same whichever surface you drive a gateway from. + (c) 2012, Holger Krekel and others """ from typing import Any +from ._serialize import can_send from ._version import version as __version__ from .sync import Channel from .sync import DataFormatError @@ -51,17 +55,57 @@ "TimeoutError", "XSpec", "__version__", + "can_send", "default_group", "makegateway", "set_execmodel", ] +#: resolved lazily so ``import execnet`` does not load the trio event loop +#: machinery, plus the deprecated pre-Trio module names -- those used to be +#: reachable here only because the import chain pulled them in, and callers +#: that still do ``execnet.gateway_base.X`` must reach the warning shim. +_LAZY_MODULES = ( + "aio", + "portal", + "trio", + "gateway", + "gateway_base", + "multi", + "rsync", + "rsync_remote", + "xspec", +) + + +#: TEMPORARY pytest-xdist compatibility. ``xdist/remote.py`` probes +#: serializability with ``try: execnet.dumps(x) / except execnet.DumpError`` +#: before shipping warning args and report attrs. The standalone serializer +#: is internal and :func:`can_send` replaces that probe, but dropping the name +#: outright breaks every released xdist, so it stays reachable -- warning, and +#: deliberately absent from ``__all__``. +#: +#: FOLLOW-UP (after the execnet release): port xdist to ``execnet.can_send``, +#: then delete this and its test. Nothing else may be added here. +_XDIST_COMPAT = ("dumps",) + + def __getattr__(name: str) -> Any: - # Lazy namespace modules: keep ``import execnet`` from loading the - # trio event loop machinery until it is actually used. - if name in ("aio", "portal", "trio"): + if name in _LAZY_MODULES: import importlib return importlib.import_module(f".{name}", __name__) + if name in _XDIST_COMPAT: + import importlib + import warnings + + warnings.warn( + f"execnet.{name} is a temporary pytest-xdist compatibility shim and" + " will be removed; the standalone serializer is not public. Use" + " execnet.can_send(obj) to test whether a value can cross a channel.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py new file mode 100644 index 00000000..e1f5449b --- /dev/null +++ b/src/execnet/_channel.py @@ -0,0 +1,483 @@ +"""The blocking :class:`Channel` and its registry and file adapters. + +A ``Channel`` is a facade over the async core: the gateway's Trio session +diverts inbound payloads for a channel id into a :class:`~execnet._boundary.Mailbox` +(or a registered callback), and ``receive()`` deserializes at the call site, +off the loop thread. +""" + +from __future__ import annotations + +import builtins +import threading +import weakref +from collections.abc import Callable +from collections.abc import Iterator +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import cast +from typing import overload + +from ._boundary import Flag +from ._boundary import Mailbox +from ._errors import RemoteError +from ._errors import TimeoutError +from ._message import Message +from ._serialize import dumps_internal +from ._serialize import loads_internal + +if TYPE_CHECKING: + from ._gateway_base import BaseGateway + +NO_ENDMARKER_WANTED = object() + + +class Channel: + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + _INTERNALWAKEUP = 1000 + _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; this module stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None + + def __init__(self, gateway: BaseGateway, id: int) -> None: + """:private:""" + assert isinstance(id, int) + assert not isinstance(gateway, type) + self.gateway = gateway + self.id = id + # serialized payloads (or ENDMARKER); None once a consumer is attached + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) + self._closed = False + self._receiveclosed = Flag(gateway._new_wakener()) + self._remoteerrors: list[RemoteError] = [] + + def _trace(self, *msg: object) -> None: + self.gateway._trace(self.id, *msg) + + def setcallback( + self, + callback: Callable[[Any], Any], + endmarker: object = NO_ENDMARKER_WANTED, + ) -> None: + """Set a callback function for receiving items. + + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. + """ + self.gateway._start_channel_consumer(self, callback, endmarker) + + def __repr__(self) -> str: + flag = (self.isclosed() and "closed") or "open" + return "" % (self.id, flag) + + def __del__(self) -> None: + if self.gateway is None: # can be None in tests + return # type: ignore[unreachable] + + self._trace("channel.__del__") + # no multithreading issues here, because we have the last ref to 'self' + if self._closed: + # state transition "closed" --> "deleted" + for error in self._remoteerrors: + error.warn() + elif self._receiveclosed.is_set(): + # state transition "sendonly" --> "deleted" + # the remote channel is already in "deleted" state, nothing to do + pass + else: + # state transition "opened" --> "deleted" + # check if we are in the middle of interpreter shutdown + # in which case the process will go away and we probably + # don't need to try to send a closing or last message + # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. + if Message is not None: + with suppress(OSError, ValueError): # ignore problems with sending + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(Message.CHANNEL_CLOSE, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) + + def _getremoteerror(self): + try: + return self._remoteerrors.pop(0) + except IndexError: + try: + return self.gateway._error + except AttributeError: + pass + return None + + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ + if self._closed: + return # late data for a locally closed channel: drop + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ + if remoteerror: + self._remoteerrors.append(remoteerror) + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + # + # public API for channel objects + # + def isclosed(self) -> bool: + """Return True if the channel is closed. + + A closed channel may still hold items. + """ + return self._closed + + @overload + def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: + pass + + @overload + def makefile( + self, + mode: Literal["w"] = ..., + proxyclose: bool = ..., + ) -> ChannelFileWrite: + pass + + def makefile( + self, + mode: Literal["r", "w"] = "w", + proxyclose: bool = False, + ) -> ChannelFileWrite | ChannelFileRead: + """Return a file-like object. + + mode can be 'w' or 'r' for writeable/readable files. + If proxyclose is true, file.close() will also close the channel. + """ + if mode == "w": + return ChannelFileWrite(channel=self, proxyclose=proxyclose) + elif mode == "r": + return ChannelFileRead(channel=self, proxyclose=proxyclose) + raise ValueError(f"mode {mode!r} not available") + + def close(self, error=None) -> None: + """Close down this channel with an optional error message. + + Note that closing of a channel tied to remote_exec happens + automatically at the end of execution and cannot + be done explicitly. + """ + if self._executing: + raise OSError("cannot explicitly close channel within remote_exec") + if self._closed: + self.gateway._trace(self, "ignoring redundant call to close()") + if not self._closed: + # state transition "opened/sendonly" --> "closed" + # threads warning: the channel might be closed under our feet, + # but it's never damaging to send too many CHANNEL_CLOSE messages + # however, if the other side triggered a close already, we + # do not send back a closed message. + if not self._receiveclosed.is_set(): + put = self.gateway._send + if error is not None: + put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) + else: + put(Message.CHANNEL_CLOSE, self.id) + self._trace("sent channel close message") + if isinstance(error, RemoteError): + self._remoteerrors.append(error) + self._closed = True # --> "closed" + self._receiveclosed.set() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) + + def waitclose(self, timeout: float | None = None) -> None: + """Wait until this channel is closed (or the remote side + otherwise signalled that no more data was being sent). + + The channel may still hold receiveable items, but not receive + any more after waitclose() has returned. + + Exceptions from executing code on the other side are reraised as local + channel.RemoteErrors. + + EOFError is raised if the reading-connection was prematurely closed, + which often indicates a dying process. + + self.TimeoutError is raised after the specified number of seconds + (default is None, i.e. wait indefinitely). + """ + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): + raise self.TimeoutError("Timeout after %r seconds" % timeout) + error = self._getremoteerror() + if error: + raise error + + def send(self, item: object) -> None: + """Sends the given item to the other side of the channel, + possibly blocking if the sender queue is full. + + The item must be a simple Python type and will be + copied to the other side by value. + + OSError is raised if the write pipe was prematurely closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) + + def receive(self, timeout: float | None = None) -> Any: + """Receive a data item that was sent from the other side. + + timeout: None [default] blocked waiting. A positive number + indicates the number of seconds after which a channel.TimeoutError + exception will be raised if no item was received. + + Note that exceptions from the remotely executing code will be + reraised as channel.RemoteError exceptions containing + a textual representation of the remote traceback. + """ + mailbox = self._mailbox + if mailbox is None: + raise OSError("cannot receive(), channel has receiver callback") + try: + x = mailbox.get(timeout) + except builtins.TimeoutError: + raise self.TimeoutError("no item after %r seconds" % timeout) from None + if x is ENDMARKER: + mailbox.put(x) # for other receivers + raise self._getremoteerror() or EOFError() + else: + return loads_internal(x, self) + + def __iter__(self) -> Iterator[Any]: + return self + + def next(self) -> Any: + try: + return self.receive() + except EOFError: + raise StopIteration from None + + __next__ = next + + +ENDMARKER = object() + + +class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. + """ + + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: + self._channels: weakref.WeakValueDictionary[int, Channel] = ( + weakref.WeakValueDictionary() + ) + self._writelock = threading.Lock() + self.gateway = gateway + self.count = startcount + self.finished = False + self._list = list # needed during interp-shutdown + + def new(self, id: int | None = None) -> Channel: + """Create a new Channel with 'id' (or create new id if None).""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + if id is None: + id = self.count + self.count += 2 + try: + channel = self._channels[id] + except KeyError: + channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) + return channel + + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def channels(self) -> list[Channel]: + return self._list(self._channels.values()) + + # + # internal methods, called from the loop thread (or local close paths) + # + def _no_longer_opened(self, id: int) -> None: + self._channels.pop(id, None) + + def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + """Close ``id`` as if the peer had closed it (no message is sent).""" + channel = self._channels.get(id) + if channel is None: + # channel already in "deleted" state + if remoteerror: + remoteerror.warn() + self._no_longer_opened(id) + else: + channel._close_from_remote(remoteerror, sendonly=sendonly) + + def _finished_receiving(self) -> None: + with self._writelock: + self.finished = True + for id in self._list(self._channels): + self._local_close(id, sendonly=True) + + +class ChannelFile: + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + self.channel = channel + self._proxyclose = proxyclose + + def isatty(self) -> bool: + return False + + def close(self) -> None: + if self._proxyclose: + self.channel.close() + + def __repr__(self) -> str: + state = (self.channel.isclosed() and "closed") or "open" + return "" % (self.channel.id, state) + + +class ChannelFileWrite(ChannelFile): + def write(self, out: bytes) -> None: + self.channel.send(out) + + def flush(self) -> None: + pass + + +class ChannelFileRead(ChannelFile): + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + super().__init__(channel, proxyclose) + self._buffer: str | None = None + + def read(self, n: int) -> str: + try: + if self._buffer is None: + self._buffer = cast(str, self.channel.receive()) + while len(self._buffer) < n: + self._buffer += cast(str, self.channel.receive()) + except EOFError: + self.close() + if self._buffer is None: + ret = "" + else: + ret = self._buffer[:n] + self._buffer = self._buffer[n:] + return ret + + def readline(self) -> str: + if self._buffer is not None: + i = self._buffer.find("\n") + if i != -1: + return self.read(i + 1) + line = self.read(len(self._buffer) + 1) + else: + line = self.read(1) + while line and line[-1] != "\n": + c = self.read(1) + if not c: + break + line += c + return line diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py new file mode 100644 index 00000000..e90ee2d0 --- /dev/null +++ b/src/execnet/_errors.py @@ -0,0 +1,109 @@ +"""Exception types and the error texts that cross the wire. + +The channel-facing errors (:class:`RemoteError`, :class:`TimeoutError`, +:class:`HostNotFound`, and the :class:`DataFormatError` family) are the one +part of this module that is public: every namespace -- :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` -- re-exports them, because the same +errors are raised whichever surface you drive a gateway from. +""" + +from __future__ import annotations + +import os +import sys +import traceback + +#: exceptions that must never be swallowed by a broad ``except`` +sysex = (KeyboardInterrupt, SystemExit) + +INTERRUPT_TEXT = "keyboard-interrupted" +MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( + "concurrent remote_exec would cause deadlock for main_thread_only execmodel" +) + + +class GatewayReceivedTerminate(Exception): + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" + + +def geterrortext( + exc: BaseException, + format_exception=traceback.format_exception, + sysex: tuple[type[BaseException], ...] = sysex, +) -> str: + try: + # In py310, can change this to: + # l = format_exception(exc) + l = format_exception(type(exc), exc, exc.__traceback__) + errortext = "".join(l) + except sysex: + raise + except BaseException: + errortext = f"{type(exc).__name__}: {exc}" + return errortext + + +class RemoteError(Exception): + """Exception containing a stringified error from the other side.""" + + def __init__(self, formatted: str) -> None: + super().__init__() + self.formatted = formatted + + def __str__(self) -> str: + return self.formatted + + def __repr__(self) -> str: + return f"{self.__class__.__name__}: {self.formatted}" + + def warn(self) -> None: + if self.formatted != INTERRUPT_TEXT: + # XXX do this better + sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") + + +class TimeoutError(IOError): + """Exception indicating that a timeout was reached.""" + + +class DataFormatError(Exception): + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ + + +class DumpError(DataFormatError): + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ + + +class LoadError(DataFormatError): + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py new file mode 100644 index 00000000..9cb1d75c --- /dev/null +++ b/src/execnet/_execmodel.py @@ -0,0 +1,92 @@ +"""Deprecated execution-model presets. + +The machinery behind execution models was retired during the Trio port; what +survives is the preset *name*, which maps onto the worker config axes +(``loop=`` / ``exec=`` / ``wait=``). Kept because pytest-xdist's remote worker +still builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. +""" + +from __future__ import annotations + +import os +import threading + + +class ExecModel: + """Deprecated preset name for an execution model. + + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ + + def __init__(self, backend: str) -> None: + self.backend = backend + + def __repr__(self) -> str: + return "" % self.backend + + @property + def queue(self): + import queue + + return queue + + @property + def subprocess(self): + import subprocess + + return subprocess + + @property + def socket(self): + import socket + + return socket + + def get_ident(self) -> int: + import _thread + + return _thread.get_ident() + + def sleep(self, delay: float) -> None: + import time + + time.sleep(delay) + + def start(self, func, args=()) -> None: + import _thread + + _thread.start_new_thread(func, args) + + def fdopen(self, fd, mode, bufsize=1, closefd=True): + return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) + + def Lock(self): + return threading.RLock() + + def RLock(self): + return threading.RLock() + + def Event(self) -> threading.Event: + return threading.Event() + + +#: worker profiles: where exec'd code runs relative to the protocol loop +EXECMODEL_PROFILES = ( + "thread", # hybrid: primary on the main thread, overflow on pool threads + "main_thread_only", # exec serialized on the main thread (GUI/pytest) + "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec +) + + +def get_execmodel(backend: str | ExecModel) -> ExecModel: + if isinstance(backend, ExecModel): + return backend + if backend in EXECMODEL_PROFILES: + return ExecModel(backend) + raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py new file mode 100644 index 00000000..facbdeca --- /dev/null +++ b/src/execnet/_gateway.py @@ -0,0 +1,158 @@ +"""Gateway code for initiating popen, socket and ssh connections. + +(c) 2004-2013, Holger Krekel and others +""" + +from __future__ import annotations + +import types +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any + +from ._channel import Channel +from ._exec_source import normalize_exec_source +from ._gateway_base import BaseGateway +from ._message import IO +from ._message import Message +from ._multi import Group +from ._serialize import dumps_internal +from ._xspec import XSpec + +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", +] + + +class Gateway(BaseGateway): + """Gateway to a local or remote Python Interpreter.""" + + _group: Group + + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ + super().__init__(io=io, id=spec.id, _startcount=1) + self.spec = spec + if spec.wait: + self._wait_backend = spec.wait + + @property + def remoteaddress(self) -> str: + # Only defined for remote IO types. + return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] + + def __repr__(self) -> str: + """A string representing gateway type and status.""" + try: + r: str = (self.hasreceiver() and "receive-live") or "not-receiving" + i = str(len(self._channelfactory.channels())) + except AttributeError: + r = "uninitialized" + i = "no" + return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" + + def exit(self) -> None: + """Trigger gateway exit. + + Defer waiting for finishing of receiver-thread and subprocess activity + to when group.terminate() is called. + """ + self._trace("gateway.exit() called") + if self not in self._group: + self._trace("gateway already unregistered with group") + return + self._group._unregister(self) + try: + self._trace("--> sending GATEWAY_TERMINATE") + self._send(Message.GATEWAY_TERMINATE) + self._trace("--> io.close_write") + self._io.close_write() + except (ValueError, EOFError, OSError) as exc: + self._trace("io-error: could not send termination sequence") + self._trace(" exception: %r" % exc) + + def _rinfo(self, update: bool = False) -> RInfo: + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ + if update or not hasattr(self, "_cache_rinfo"): + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return self._cache_rinfo + + def hasreceiver(self) -> bool: + """Whether gateway is able to receive data.""" + session = self._trio_session + return session is not None and bool(session.is_alive()) + + def remote_status(self) -> RemoteStatus: + """Obtain information about the remote execution status.""" + channel = self.newchannel() + self._send(Message.STATUS, channel.id) + statusdict = channel.receive() + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return RemoteStatus(statusdict) + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Return channel object and connect it to a remote + execution thread where the given ``source`` executes. + + * ``source`` is a string: execute source string remotely + with a ``channel`` put into the global namespace. + * ``source`` is a pure function: serialize source and + call function with ``**kwargs``, adding a + ``channel`` object to the keyword arguments. + * ``source`` is a pure module: execute source of module + with a ``channel`` in its global namespace. + + In all cases the binding ``__name__='__channelexec__'`` + will be available in the global namespace of the remotely + executing code. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.newchannel() + self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + + def remote_init_threads(self, num: int | None = None) -> None: + """DEPRECATED. Is currently a NO-OPERATION already.""" + print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") + + +class RInfo: + def __init__(self, kwargs) -> None: + self.__dict__.update(kwargs) + + def __repr__(self) -> str: + info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) + return "" % info + + if TYPE_CHECKING: + + def __getattr__(self, name: str) -> Any: ... + + +RemoteStatus = RInfo diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py new file mode 100644 index 00000000..16b21602 --- /dev/null +++ b/src/execnet/_gateway_base.py @@ -0,0 +1,237 @@ +"""The gateway base classes shared by coordinator and worker. + +:class:`BaseGateway` owns the channel factory, the send path and the handle on +the Trio session that does the actual Message IO; :class:`WorkerGateway` adds +the worker-side ``remote_exec`` scheduling and shutdown. + +:copyright: 2004-2015 +:authors: + - Holger Krekel + - Armin Rigo + - Benjamin Peterson + - Ronny Pfannschmidt + - many others +""" + +from __future__ import annotations + +import os +import sys +import threading +from _thread import interrupt_main +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +from ._boundary import Wakener +from ._boundary import make_wakener +from ._channel import Channel +from ._channel import ChannelFactory +from ._errors import INTERRUPT_TEXT +from ._errors import geterrortext +from ._errors import sysex +from ._message import IO +from ._message import Message +from ._trace import trace + + +class BaseGateway: + _sysex = sysex + id = "" + _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None + #: wait= axis: which wakener backend this gateway's blocking waits + #: park on (channels, write-acks, join) + _wait_backend: str = "thread" + + def __init__(self, io: IO, id, _startcount: int = 2) -> None: + self.execmodel = io.execmodel + self._io = io + self.id = id + self._channelfactory = ChannelFactory(self, _startcount) + # globals may be NONE at process-termination + self.__trace = trace + self._geterrortext = geterrortext + self._trio_session = None + + def _trace(self, *msg: object) -> None: + self.__trace(self.id, *msg) + + def _attach_trio_session(self, session: Any) -> None: + """Attach the Trio bridge session doing the Message IO.""" + self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier (wait= axis).""" + return make_wakener(self._wait_backend) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) + + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + + def _terminate_execution(self) -> None: + pass + + def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return + try: + message.to_io(self._io) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + # ValueError might be because the IO is already closed + raise OSError("cannot send (already closed?)") from e + + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + channel.close("execution disallowed") + + # _____________________________________________________________________ + # + # High Level Interface + # _____________________________________________________________________ + # + def newchannel(self) -> Channel: + """Return a new independent channel.""" + return self._channelfactory.new() + + def join(self, timeout: float | None = None) -> None: + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + + +class WorkerGateway(BaseGateway): + _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + _executetask_complete: threading.Event | None = None + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") + return + trio_exec.schedule(channel, sourcetask) + + def _terminate_execution(self) -> None: + # called from receiverthread + self._trace("shutting down execution pool") + self._execpool.trigger_shutdown() + if not self._execpool.waitall(5.0): + self._trace("execution ongoing after 5 secs, trying interrupt_main") + # We try hard to terminate execution based on the assumption + # that there is only one gateway object running per-process. + if sys.platform != "win32": + self._trace("sending ourselves a SIGINT") + os.kill(os.getpid(), 2) # send ourselves a SIGINT + elif interrupt_main is not None: + self._trace("calling interrupt_main()") + interrupt_main() + if not self._execpool.waitall(10.0): + self._trace( + "execution did not finish in another 10 secs, calling os._exit()" + ) + os._exit(1) + + def executetask( + self, + item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], + ) -> None: + try: + channel, (source, file_name, call_name, kwargs) = item + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") + channel._executing = True + try: + co = compile(source + "\n", file_name or "", "exec") + exec(co, loc) + if call_name: + self._trace("calling %s(**%60r)" % (call_name, kwargs)) + function = loc[call_name] + function(channel, **kwargs) + finally: + channel._executing = False + self._trace("execution finished") + except KeyboardInterrupt: + channel.close(INTERRUPT_TEXT) + raise + except EOFError: + self._trace("ignoring EOFError because receiving finished") + + except BaseException as exc: + if not channel.gateway._channelfactory.finished: + self._trace(f"got exception: {exc!r}") + errortext = self._geterrortext(exc) + channel.close(errortext) + return + channel.close() + if self._executetask_complete is not None: + # Indicate that this task has finished executing, meaning + # that there is no possibility of it triggering a deadlock + # for the next spawn call. + self._executetask_complete.set() diff --git a/src/execnet/_message.py b/src/execnet/_message.py new file mode 100644 index 00000000..4d31339f --- /dev/null +++ b/src/execnet/_message.py @@ -0,0 +1,174 @@ +"""The wire protocol: IO protocols, Message framing and its decoder. + +A frame is a 9-byte header (``!bii``: message code, channel id, payload +length) followed by the payload. Nothing here dispatches -- routing lives in +``AsyncGateway._dispatch`` and the sync bridge session; this module only packs, +unpacks and buffers. +""" + +from __future__ import annotations + +import os +import struct +import sys +from collections.abc import Iterator +from typing import TYPE_CHECKING +from typing import Protocol + +if TYPE_CHECKING: + from ._execmodel import ExecModel + + +class WriteIO(Protocol): + def write(self, data: bytes, /) -> None: ... + + +class ReadIO(Protocol): + def read(self, numbytes: int, /) -> bytes: ... + + +class IO(Protocol): + execmodel: ExecModel + + def read(self, numbytes: int, /) -> bytes: ... + + def write(self, data: bytes, /) -> None: ... + + def close_read(self) -> None: ... + + def close_write(self) -> None: ... + + def wait(self) -> int | None: ... + + def kill(self) -> None: ... + + +class Message: + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 + + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", + } + + def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + self.msgcode = msgcode + self.channelid = channelid + self.data = data + + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + + @staticmethod + def from_io(io: ReadIO) -> Message: + try: + header = io.read(9) # type 1, channel 4, payload 4 + if not header: + raise EOFError("empty read") + except EOFError as e: + raise EOFError("couldn't load message header, " + e.args[0]) from None + msgtype, channel, payload = Message.from_header(header) + return Message(msgtype, channel, io.read(payload)) + + def to_io(self, io: WriteIO) -> None: + io.write(self.pack()) + + def __repr__(self) -> str: + name = self._types[self.msgcode] + return f"" + + +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py new file mode 100644 index 00000000..429103f7 --- /dev/null +++ b/src/execnet/_multi.py @@ -0,0 +1,419 @@ +""" +Managing Gateway Groups and interactions with multiple channels. + +(c) 2008-2014, Holger Krekel and others +""" + +from __future__ import annotations + +import atexit +import queue +import threading +import time +import types +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import suppress +from threading import Lock +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import TypeAlias +from typing import overload + +from ._boundary import wakener_names +from ._channel import Channel +from ._execmodel import EXECMODEL_PROFILES +from ._execmodel import ExecModel +from ._execmodel import get_execmodel +from ._trace import trace +from ._xspec import XSpec + +if TYPE_CHECKING: + from ._gateway import Gateway + + +NO_ENDMARKER_WANTED = object() + + +class Group: + """Gateway Group.""" + + defaultspec = "popen" + + def __init__( + self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" + ) -> None: + """Initialize a group and make gateways as specified. + + execmodel can be one of the supported execution models. + """ + self._gateways: list[Gateway] = [] + self._autoidcounter = 0 + self._autoidlock = Lock() + self._gateways_to_join: list[Gateway] = [] + self._trio_host: Any = None + self._async_group: Any = None + # we use the same execmodel for all of the Gateway objects + # we spawn on our side. Probably we should not allow different + # execmodels between different groups but not clear. + # Note that "other side" execmodels may differ and is typically + # specified by the spec passed to makegateway. + self.set_execmodel(execmodel) + for xspec in xspecs: + self.makegateway(xspec) + atexit.register(self._cleanup_atexit) + + def _ensure_trio_host(self) -> Any: + if self._trio_host is None: + from . import _trio_host + + self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") + self._trio_host.start() + return self._trio_host + + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + + @property + def execmodel(self) -> ExecModel: + return self._execmodel + + @property + def remote_execmodel(self) -> ExecModel: + return self._remote_execmodel + + def set_execmodel( + self, execmodel: str, remote_execmodel: str | None = None + ) -> None: + """Set the execution model for local and remote site. + + execmodel can be one of the supported execution models. + It determines the execution model for any newly created gateway. + If remote_execmodel is not specified it takes on the value of execmodel. + + NOTE: Execution models can only be set before any gateway is created. + """ + if self._gateways: + raise ValueError( + "can not set execution models if gateways have been created already" + ) + if remote_execmodel is None: + remote_execmodel = execmodel + self._execmodel = get_execmodel(execmodel) + self._remote_execmodel = get_execmodel(remote_execmodel) + + def __repr__(self) -> str: + idgateways = [gw.id for gw in self] + return "" % idgateways + + def __getitem__(self, key: int | str | Gateway) -> Gateway: + if isinstance(key, int): + return self._gateways[key] + for gw in self._gateways: + if gw == key or gw.id == key: + return gw + raise KeyError(key) + + def __contains__(self, key: str) -> bool: + try: + self[key] + return True + except KeyError: + return False + + def __len__(self) -> int: + return len(self._gateways) + + def __iter__(self) -> Iterator[Gateway]: + return iter(list(self._gateways)) + + def makegateway(self, spec: XSpec | str | None = None) -> Gateway: + """Create and configure a gateway to a Python interpreter. + + The ``spec`` string encodes the target gateway type + and configuration information. The general format is:: + + key1=value1//key2=value2//... + + If you leave out the ``=value`` part a True value is assumed. + Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. + Valid configuration:: + + id= specifies the gateway id + python= specifies which python interpreter to execute + execmodel=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' (pool + threads) or 'main_thread_only' (serialized on + the worker main thread, GUI/signal-safe). + wait=backend wakener for blocking waits ('thread' default) + chdir= specifies to which directory to change + nice= specifies process priority of new process + env:NAME=value specifies a remote environment variable setting. + + If no spec is given, self.defaultspec is used. + """ + if not spec: + spec = self.defaultspec + if not isinstance(spec, XSpec): + spec = XSpec(spec) + self.allocate_id(spec) + if spec.execmodel is None: + spec.execmodel = self.remote_execmodel.backend + elif spec.execmodel not in EXECMODEL_PROFILES: + raise ValueError( + f"unknown execmodel {spec.execmodel!r}" + f" (known profiles: {list(EXECMODEL_PROFILES)})" + ) + if spec.wait is not None and spec.wait not in wakener_names(): + raise ValueError( + f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" + ) + from . import _trio_host + + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): + raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) + gw.spec = spec + self._register(gw) + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. + return gw + + def allocate_id(self, spec: XSpec) -> None: + """(re-entrant) allocate id for the given xspec object.""" + if spec.id is None: + with self._autoidlock: + id = "gw" + str(self._autoidcounter) + self._autoidcounter += 1 + if id in self: + raise ValueError(f"already have gateway with id {id!r}") + spec.id = id + + def _register(self, gateway: Gateway) -> None: + assert not hasattr(gateway, "_group") + assert gateway.id + assert gateway.id not in self + self._gateways.append(gateway) + gateway._group = self + + def _unregister(self, gateway: Gateway) -> None: + self._gateways.remove(gateway) + self._gateways_to_join.append(gateway) + + def _cleanup_atexit(self) -> None: + trace(f"=== atexit cleanup {self!r} ===") + self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._trio_host.call_sync(self._async_group.shutdown.set) + self._async_group = None + if self._trio_host is not None: + self._trio_host.stop(timeout=1.0) + self._trio_host = None + + def terminate(self, timeout: float | None = None) -> None: + """Trigger exit of member gateways and wait for termination + of member gateways and associated subprocesses. + + After waiting timeout seconds try to to kill local sub processes of + popen- and ssh-gateways. + + Timeout defaults to None meaning open-ended waiting and no kill + attempts. + """ + while self or self._gateways_to_join: + vias: set[str] = set() + for gw in self: + if gw.spec.via: + vias.add(gw.spec.via) + for gw in self: + if gw.id not in vias: + gw.exit() + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._host_terminate(timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: + gw.join() + self._gateways_to_join[:] = [] + + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for wait backends. + + A member gateway created with a non-thread ``wait=`` implies the + caller may be a greenlet: wait on a OneShot instead of blocking + the OS thread (which would stall the hub for the whole grace). + """ + backends = {gw._wait_backend for gw in self._gateways_to_join} | { + gw._wait_backend for gw in self + } + backends.discard("thread") + if backends: + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(backends.pop()), + ).wait() + else: + self._trio_host.call(self._async_group.terminate, timeout) + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs, + ) -> MultiChannel: + """remote_exec source on all member gateways and return + a MultiChannel connecting to all sub processes.""" + channels = [] + for gw in self: + channels.append(gw.remote_exec(source, **kwargs)) + return MultiChannel(channels) + + +class MultiChannel: + def __init__(self, channels: Sequence[Channel]) -> None: + self._channels = channels + + def __len__(self) -> int: + return len(self._channels) + + def __iter__(self) -> Iterator[Channel]: + return iter(self._channels) + + def __getitem__(self, key: int) -> Channel: + return self._channels[key] + + def __contains__(self, chan: Channel) -> bool: + return chan in self._channels + + def send_each(self, item: object) -> None: + for ch in self._channels: + ch.send(item) + + @overload + def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: + pass + + @overload + def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: + pass + + def receive_each( + self, withchannel: bool = False + ) -> list[tuple[Channel, Any]] | list[Any]: + assert not hasattr(self, "_queue") + l: list[object] = [] + for ch in self._channels: + obj = ch.receive() + if withchannel: + l.append((ch, obj)) + else: + l.append(obj) + return l + + def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): + try: + return self._queue # type: ignore[has-type] + except AttributeError: + self._queue: queue.Queue[tuple[Channel, Any]] | None = None + for ch in self._channels: + if self._queue is None: + self._queue = queue.Queue() + + def putreceived(obj, channel: Channel = ch) -> None: + self._queue.put((channel, obj)) # type: ignore[union-attr] + + if endmarker is NO_ENDMARKER_WANTED: + ch.setcallback(putreceived) + else: + ch.setcallback(putreceived, endmarker=endmarker) + return self._queue + + def waitclose(self) -> None: + first = None + for ch in self._channels: + try: + ch.waitclose() + except ch.RemoteError as exc: + if first is None: + first = exc + if first: + raise first + + +TermKillFunc: TypeAlias = Callable[[], object] +TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] + + +def safe_terminate( + execmodel: ExecModel, + timeout: float | None, + list_of_paired_functions: Sequence[TermKillPair], +) -> None: + """Run terminate/kill pairs in parallel with a hard wait bound. + + Each termfunc is given ``timeout``. If it does not finish, killfunc runs. + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). + """ + errors: list[BaseException] = [] + + def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): + killfunc() + return + if term_errors: + errors.append(term_errors[0]) + + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions + ] + for thread in threads: + thread.start() + # Allow term timeout plus a kill attempt; never block indefinitely. + wait_timeout = None if timeout is None else timeout * 2 + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] + + +default_group = Group() +makegateway = default_group.makegateway +set_execmodel = default_group.set_execmodel diff --git a/src/execnet/_rsync.py b/src/execnet/_rsync.py new file mode 100644 index 00000000..d14825f6 --- /dev/null +++ b/src/execnet/_rsync.py @@ -0,0 +1,248 @@ +""" +1:N rsync implementation on top of execnet. + +(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Callable +from hashlib import md5 +from queue import Queue +from typing import Literal + +from execnet import _rsync_remote +from execnet._channel import Channel +from execnet._gateway import Gateway +from execnet._gateway_base import BaseGateway + + +class RSync: + """This class allows to send a directory structure (recursively) + to one or multiple remote filesystems. + + There is limited support for symlinks, which means that symlinks + pointing to the sourcetree will be send "as is" while external + symlinks will be just copied (regardless of existence of such + a path on remote side). + """ + + def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: + self._sourcedir = str(sourcedir) + self._verbose = verbose + assert callback is None or callable(callback) + self._callback = callback + self._channels: dict[Channel, Callable[[], None] | None] = {} + self._receivequeue: Queue[ + tuple[ + Channel, + ( + None + | tuple[Literal["send"], tuple[list[str], bytes]] + | tuple[Literal["list_done"], None] + | tuple[Literal["ack"], str] + | tuple[Literal["links"], None] + | tuple[Literal["done"], None] + ), + ] + ] = Queue() + self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] + + def filter(self, path: str) -> bool: + return True + + def _end_of_channel(self, channel: Channel) -> None: + if channel in self._channels: + # too early! we must have got an error + channel.waitclose() + # or else we raise one + raise OSError(f"connection unexpectedly closed: {channel.gateway} ") + + def _process_link(self, channel: Channel) -> None: + for link in self._links: + channel.send(link) + # completion marker, this host is done + channel.send(42) + + def _done(self, channel: Channel) -> None: + """Call all callbacks.""" + finishedcallback = self._channels.pop(channel) + if finishedcallback: + finishedcallback() + channel.waitclose() + + def _list_done(self, channel: Channel) -> None: + # sum up all to send + if self._callback: + s = sum([self._paths[i] for i in self._to_send[channel]]) + self._callback("list", s, channel) + + def _send_item( + self, + channel: Channel, + modified_rel_path_components: list[str], + checksum: bytes, + ) -> None: + """Send one item.""" + modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) + try: + with open(modified_path, "rb") as fp: + data = fp.read() + except OSError: + data = None + + # provide info to progress callback function + modified_rel_path = "/".join(modified_rel_path_components) + if data is not None: + self._paths[modified_rel_path] = len(data) + else: + self._paths[modified_rel_path] = 0 + if channel not in self._to_send: + self._to_send[channel] = [] + self._to_send[channel].append(modified_rel_path) + # print "sending", modified_rel_path, data and len(data) or 0, checksum + + if data is not None: + if checksum is not None and checksum == md5(data).digest(): + data = None # not really modified + else: + self._report_send_file(channel.gateway, modified_rel_path) + channel.send(data) + + def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: + if self._verbose: + print(f"{gateway} <= {modified_rel_path}") + + def send(self, raises: bool = True) -> None: + """Sends a sourcedir to all added targets. + + raises indicates whether to raise an error or return in case of lack of + targets. + """ + if not self._channels: + if raises: + raise OSError( + "no targets available, maybe you are trying call send() twice?" + ) + return + # normalize a trailing '/' away + self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) + # send directory structure and file timestamps/sizes + self._send_directory_structure(self._sourcedir) + + # paths and to_send are only used for doing + # progress-related callbacks + self._paths: dict[str, int] = {} + self._to_send: dict[Channel, list[str]] = {} + + # send modified file to clients + while self._channels: + channel, req = self._receivequeue.get() + if req is None: + self._end_of_channel(channel) + else: + if req[0] == "links": + self._process_link(channel) + elif req[0] == "done": + self._done(channel) + elif req[0] == "ack": + if self._callback: + self._callback("ack", self._paths[req[1]], channel) + elif req[0] == "list_done": + self._list_done(channel) + elif req[0] == "send": + self._send_item(channel, req[1][0], req[1][1]) + else: + assert "Unknown command %s" % req[0] # type: ignore[unreachable] + + def add_target( + self, + gateway: Gateway, + destdir: str | os.PathLike[str], + finishedcallback: Callable[[], None] | None = None, + **options, + ) -> None: + """Add a remote target specified via a gateway and a remote destination + directory.""" + for name in options: + assert name in ("delete",) + + def itemcallback(req) -> None: + self._receivequeue.put((channel, req)) + + channel = gateway.remote_exec(_rsync_remote) + channel.setcallback(itemcallback, endmarker=None) + channel.send((str(destdir), options)) + self._channels[channel] = finishedcallback + + def _broadcast(self, msg: object) -> None: + for channel in self._channels: + channel.send(msg) + + def _send_link( + self, + linktype: Literal["linkbase", "link"], + basename: str, + linkpoint: str, + ) -> None: + self._links.append((linktype, basename, linkpoint)) + + def _send_directory(self, path: str) -> None: + # dir: send a list of entries + names = [] + subpaths = [] + for name in os.listdir(path): + p = os.path.join(path, name) + if self.filter(p): + names.append(name) + subpaths.append(p) + mode = os.lstat(path).st_mode + self._broadcast([mode, *names]) + for p in subpaths: + self._send_directory_structure(p) + + def _send_link_structure(self, path: str) -> None: + sourcedir = self._sourcedir + basename = path[len(self._sourcedir) + 1 :] + linkpoint = os.readlink(path) + # On Windows, readlink returns an extended path (//?/) for + # absolute links, but relpath doesn't like mixing extended + # and non-extended paths. So fix it up ourselves. + if ( + os.path.__name__ == "ntpath" + and linkpoint.startswith("\\\\?\\") + and not self._sourcedir.startswith("\\\\?\\") + ): + sourcedir = "\\\\?\\" + self._sourcedir + try: + relpath = os.path.relpath(linkpoint, sourcedir) + except ValueError: + relpath = None + if ( + relpath is not None + and relpath not in (os.curdir, os.pardir) + and not relpath.startswith(os.pardir + os.sep) + ): + self._send_link("linkbase", basename, relpath) + else: + # relative or absolute link, just send it + self._send_link("link", basename, linkpoint) + self._broadcast(None) + + def _send_directory_structure(self, path: str) -> None: + try: + st = os.lstat(path) + except OSError: + self._broadcast((None, 0, 0)) + return + if stat.S_ISREG(st.st_mode): + # regular file: send a mode/timestamp/size pair + self._broadcast((st.st_mode, st.st_mtime, st.st_size)) + elif stat.S_ISDIR(st.st_mode): + self._send_directory(path) + elif stat.S_ISLNK(st.st_mode): + self._send_link_structure(path) + else: + raise ValueError(f"cannot sync {path!r}") diff --git a/src/execnet/_rsync_remote.py b/src/execnet/_rsync_remote.py new file mode 100644 index 00000000..bc2e4f41 --- /dev/null +++ b/src/execnet/_rsync_remote.py @@ -0,0 +1,126 @@ +""" +(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Literal +from typing import cast + +if TYPE_CHECKING: + from execnet._channel import Channel + + +def serve_rsync(channel: Channel) -> None: + import os + import shutil + import stat + from hashlib import md5 + + destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) + modifiedfiles = [] + + def remove(path: str) -> None: + assert path.startswith(destdir) + try: + os.unlink(path) + except OSError: + # assume it's a dir + shutil.rmtree(path, True) + + def receive_directory_structure(path: str, relcomponents: list[str]) -> None: + try: + st = os.lstat(path) + except OSError: + st = None + msg = channel.receive() + if isinstance(msg, list): + if st and not stat.S_ISDIR(st.st_mode): + os.unlink(path) + st = None + if not st: + os.makedirs(path) + mode = msg.pop(0) + if mode: + # Ensure directories are writable, otherwise a + # permission denied error (EACCES) would be raised + # when attempting to receive read-only directory + # structures. + os.chmod(path, mode | 0o700) + entrynames = {} + for entryname in msg: + destpath = os.path.join(path, entryname) + receive_directory_structure(destpath, [*relcomponents, entryname]) + entrynames[entryname] = True + if options.get("delete"): + for othername in os.listdir(path): + if othername not in entrynames: + otherpath = os.path.join(path, othername) + remove(otherpath) + elif msg is not None: + assert isinstance(msg, tuple) + checksum = None + if st: + if stat.S_ISREG(st.st_mode): + msg_mode, msg_mtime, msg_size = msg + if msg_size != st.st_size: + pass + elif msg_mtime != st.st_mtime: + with open(path, "rb") as fp: + checksum = md5(fp.read()).digest() + elif msg_mode and msg_mode != st.st_mode: + os.chmod(path, msg_mode | 0o700) + return + else: + return # already fine + else: + remove(path) + channel.send(("send", (relcomponents, checksum))) + modifiedfiles.append((path, msg)) + + receive_directory_structure(destdir, []) + + STRICT_CHECK = False # seems most useful this way for py.test + channel.send(("list_done", None)) + + for path, (mode, time, size) in modifiedfiles: + data = cast(bytes, channel.receive()) + channel.send(("ack", path[len(destdir) + 1 :])) + if data is not None: + if STRICT_CHECK and len(data) != size: + raise OSError(f"file modified during rsync: {path!r}") + with open(path, "wb") as fp: + fp.write(data) + try: + if mode: + os.chmod(path, mode) + os.utime(path, (time, time)) + except OSError: + pass + del data + channel.send(("links", None)) + + msg = channel.receive() + while msg != 42: + # we get symlink + _type, relpath, linkpoint = cast( + "tuple[Literal['linkbase', 'link'], str, str]", msg + ) + path = os.path.join(destdir, relpath) + with suppress(OSError): + remove(path) + + if _type == "linkbase": + src = os.path.join(destdir, linkpoint) + else: + assert _type == "link", _type + src = linkpoint + os.symlink(src, path) + msg = channel.receive() + channel.send(("done", None)) + + +if __name__ == "__channelexec__": + serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/_serialize.py b/src/execnet/_serialize.py new file mode 100644 index 00000000..3151efe3 --- /dev/null +++ b/src/execnet/_serialize.py @@ -0,0 +1,439 @@ +"""The wire serializer for execnet's simple builtin data format. + +Internal: no public namespace re-exports ``dumps``/``loads``. Callers that +need to know whether a value can cross a channel use +:func:`execnet.can_send`, which is exported from every public namespace. + +The channel layer is *not* imported here -- ``Unserializer`` resolves a +channel or gateway argument by duck-typing -- so ``_channel`` may depend on +this module and not the other way round. +""" + +from __future__ import annotations + +import struct +from collections.abc import Callable +from io import BytesIO +from typing import TYPE_CHECKING +from typing import Any + +from ._errors import DumpError +from ._errors import LoadError + +if TYPE_CHECKING: + from ._channel import Channel + from ._gateway_base import BaseGateway + from ._message import ReadIO + + +def bchr(n: int) -> bytes: + return bytes([n]) + + +DUMPFORMAT_VERSION = bchr(2) + +FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 + +FLOAT_FORMAT = "!d" +FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) +COMPLEX_FORMAT = "!dd" +COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) + + +class _Stop(Exception): + pass + + +class opcode: + """Container for name -> num mappings.""" + + BUILDTUPLE = b"@" + BYTES = b"A" + CHANNEL = b"B" + FALSE = b"C" + FLOAT = b"D" + FROZENSET = b"E" + INT = b"F" + LONG = b"G" + LONGINT = b"H" + LONGLONG = b"I" + NEWDICT = b"J" + NEWLIST = b"K" + NONE = b"L" + STRING = b"N" + SET = b"O" + SETITEM = b"P" + STOP = b"Q" + TRUE = b"R" + COMPLEX = b"T" + + +class Unserializer: + num2func: dict[bytes, Callable[[Unserializer], None]] = {} + + def __init__( + self, + stream: ReadIO, + channel_or_gateway: Channel | BaseGateway | None = None, + ) -> None: + # A channel -- sync or trio-native -- resolves through its gateway; a + # gateway is already the right object. Duck-typed so the serializer + # stays independent of the channel layer. + gw: Any = getattr(channel_or_gateway, "gateway", channel_or_gateway) + self.stream = stream + if gw is None: + self.channelfactory = None + else: + self.channelfactory = gw._channelfactory + + def load(self, versioned: bool = False) -> Any: + if versioned: + ver = self.stream.read(1) + if ver != DUMPFORMAT_VERSION: + raise LoadError("wrong dumpformat version %r" % ver) + self.stack: list[object] = [] + try: + while True: + opcode = self.stream.read(1) + if not opcode: + raise EOFError + try: + loader = self.num2func[opcode] + except KeyError: + raise LoadError( + f"unknown opcode {opcode!r} - wire protocol corruption?" + ) from None + loader(self) + except _Stop: + if len(self.stack) != 1: + raise LoadError("internal unserialization error") from None + return self.stack.pop(0) + else: + raise LoadError("didn't get STOP") + + def load_none(self) -> None: + self.stack.append(None) + + num2func[opcode.NONE] = load_none + + def load_true(self) -> None: + self.stack.append(True) + + num2func[opcode.TRUE] = load_true + + def load_false(self) -> None: + self.stack.append(False) + + num2func[opcode.FALSE] = load_false + + def load_int(self) -> None: + i = self._read_int4() + self.stack.append(i) + + num2func[opcode.INT] = load_int + + def load_longint(self) -> None: + s = self._read_byte_string() + self.stack.append(int(s)) + + num2func[opcode.LONGINT] = load_longint + + load_long = load_int + num2func[opcode.LONG] = load_long + load_longlong = load_longint + num2func[opcode.LONGLONG] = load_longlong + + def load_float(self) -> None: + binary = self.stream.read(FLOAT_FORMAT_SIZE) + self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) + + num2func[opcode.FLOAT] = load_float + + def load_complex(self) -> None: + binary = self.stream.read(COMPLEX_FORMAT_SIZE) + self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) + + num2func[opcode.COMPLEX] = load_complex + + def _read_int4(self) -> int: + value: int = struct.unpack("!i", self.stream.read(4))[0] + return value + + def _read_byte_string(self) -> bytes: + length = self._read_int4() + as_bytes = self.stream.read(length) + return as_bytes + + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) + + num2func[opcode.STRING] = load_string + + def load_bytes(self) -> None: + s = self._read_byte_string() + self.stack.append(s) + + num2func[opcode.BYTES] = load_bytes + + def load_newlist(self) -> None: + length = self._read_int4() + self.stack.append([None] * length) + + num2func[opcode.NEWLIST] = load_newlist + + def load_setitem(self) -> None: + if len(self.stack) < 3: + raise LoadError("not enough items for setitem") + value = self.stack.pop() + key = self.stack.pop() + self.stack[-1][key] = value # type: ignore[index] + + num2func[opcode.SETITEM] = load_setitem + + def load_newdict(self) -> None: + self.stack.append({}) + + num2func[opcode.NEWDICT] = load_newdict + + def _load_collection(self, type_: type) -> None: + length = self._read_int4() + if length: + res = type_(self.stack[-length:]) + del self.stack[-length:] + self.stack.append(res) + else: + self.stack.append(type_()) + + def load_buildtuple(self) -> None: + self._load_collection(tuple) + + num2func[opcode.BUILDTUPLE] = load_buildtuple + + def load_set(self) -> None: + self._load_collection(set) + + num2func[opcode.SET] = load_set + + def load_frozenset(self) -> None: + self._load_collection(frozenset) + + num2func[opcode.FROZENSET] = load_frozenset + + def load_stop(self) -> None: + raise _Stop + + num2func[opcode.STOP] = load_stop + + def load_channel(self) -> None: + id = self._read_int4() + assert self.channelfactory is not None + newchannel = self.channelfactory.new(id) + self.stack.append(newchannel) + + num2func[opcode.CHANNEL] = load_channel + + +def dumps(obj: object) -> bytes: + """Serialize the given obj to a bytestring. + + The obj and all contained objects must be of a builtin + Python type (so nested dicts, sets, etc. are all OK but + not user-level instances). + """ + return _Serializer().save(obj, versioned=True) # type: ignore[return-value] + + +def dump(byteio, obj: object) -> None: + """write a serialized bytestring of the given obj to the given stream.""" + _Serializer(write=byteio.write).save(obj, versioned=True) + + +def loads(bytestring: bytes) -> Any: + """Deserialize the given bytestring to an object. + + If the bytestring was dumped with an incompatible protocol + version or if the bytestring is corrupted, the + ``execnet.DataFormatError`` will be raised. + """ + return load(BytesIO(bytestring)) + + +def load(io: ReadIO) -> Any: + """Derserialize an object form the specified stream. + + Behaviour is otherwise the same as with ``loads`` + """ + return Unserializer(io).load(versioned=True) + + +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: + io = BytesIO(bytestring) + return Unserializer(io, channelfactory).load() + + +def dumps_internal(obj: object) -> bytes: + return _Serializer().save(obj) # type: ignore[return-value] + + +def can_send(obj: object) -> bool: + """Whether ``obj`` can cross a channel as-is. + + True for execnet's simple builtin wire data -- ``None``, ``bool``, + ``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily + nested ``list``/``tuple``/``set``/``frozenset``/``dict`` of those -- + and for channel references. False for anything execnet has no wire + representation for, which ``channel.send`` would reject with + :class:`~execnet.DumpError`. + + Use it to branch *before* sending, instead of sending and handling the + error:: + + channel.send(value if execnet.can_send(value) else repr(value)) + """ + try: + _Serializer().save(obj) + except DumpError: + return False + return True + + +class _Serializer: + _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} + + def __init__(self, write: Callable[[bytes], None] | None = None) -> None: + if write is None: + self._streamlist: list[bytes] = [] + write = self._streamlist.append + self._write = write + + def save(self, obj: object, versioned: bool = False) -> bytes | None: + # calling here is not re-entrant but multiple instances + # may write to the same stream because of the common platform + # atomic-write guarantee (concurrent writes each happen atomically) + if versioned: + self._write(DUMPFORMAT_VERSION) + self._save(obj) + self._write(opcode.STOP) + try: + streamlist = self._streamlist + except AttributeError: + return None + return b"".join(streamlist) + + def _save(self, obj: object) -> None: + tp = type(obj) + try: + dispatch = self._dispatch[tp] + except KeyError: + methodname = "save_" + tp.__name__ + meth: Callable[[_Serializer, object], None] | None = getattr( + self.__class__, methodname, None + ) + if meth is None: + raise DumpError(f"can't serialize {tp}") from None + dispatch = self._dispatch[tp] = meth + dispatch(self, obj) + + def save_NoneType(self, non: None) -> None: + self._write(opcode.NONE) + + def save_bool(self, boolean: bool) -> None: + if boolean: + self._write(opcode.TRUE) + else: + self._write(opcode.FALSE) + + def save_bytes(self, bytes_: bytes) -> None: + self._write(opcode.BYTES) + self._write_byte_sequence(bytes_) + + def save_str(self, s: str) -> None: + self._write(opcode.STRING) + self._write_unicode_string(s) + + def _write_unicode_string(self, s: str) -> None: + try: + as_bytes = s.encode("utf-8") + except UnicodeEncodeError as e: + raise DumpError("strings must be utf-8 encodable") from e + self._write_byte_sequence(as_bytes) + + def _write_byte_sequence(self, bytes_: bytes) -> None: + self._write_int4(len(bytes_), "string is too long") + self._write(bytes_) + + def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: + self._write(short_op) + self._write_int4(i) + else: + self._write(long_op) + self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) + + def save_int(self, i: int) -> None: + self._save_integral(i, opcode.INT, opcode.LONGINT) + + def save_long(self, l: int) -> None: + self._save_integral(l, opcode.LONG, opcode.LONGLONG) + + def save_float(self, flt: float) -> None: + self._write(opcode.FLOAT) + self._write(struct.pack(FLOAT_FORMAT, flt)) + + def save_complex(self, cpx: complex) -> None: + self._write(opcode.COMPLEX) + self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) + + def _write_int4( + self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) + ) -> None: + if i > FOUR_BYTE_INT_MAX: + raise DumpError(error) + self._write(struct.pack("!i", i)) + + def save_list(self, L: list[object]) -> None: + self._write(opcode.NEWLIST) + self._write_int4(len(L), "list is too long") + for i, item in enumerate(L): + self._write_setitem(i, item) + + def _write_setitem(self, key: object, value: object) -> None: + self._save(key) + self._save(value) + self._write(opcode.SETITEM) + + def save_dict(self, d: dict[object, object]) -> None: + self._write(opcode.NEWDICT) + for key, value in d.items(): + self._write_setitem(key, value) + + def save_tuple(self, tup: tuple[object, ...]) -> None: + for item in tup: + self._save(item) + self._write(opcode.BUILDTUPLE) + self._write_int4(len(tup), "tuple is too long") + + def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: + for item in s: + self._save(item) + self._write(op) + self._write_int4(len(s), "set is too long") + + def save_set(self, s: set[object]) -> None: + self._write_set(s, opcode.SET) + + def save_frozenset(self, s: frozenset[object]) -> None: + self._write_set(s, opcode.FROZENSET) + + def save_Channel(self, channel: Channel) -> None: + self._write(opcode.CHANNEL) + self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py new file mode 100644 index 00000000..35e7682c --- /dev/null +++ b/src/execnet/_shim.py @@ -0,0 +1,47 @@ +"""Machinery for the deprecated pre-Trio module names. + +``execnet.gateway_base``, ``execnet.gateway``, ``execnet.multi``, +``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` were never +part of a documented public API -- they were importable only because +``import execnet`` pulled them in transitively. Their contents now live in +private modules grouped by concern; each old name survives as a thin module +whose ``__getattr__`` warns and forwards. + +The supported surfaces are :mod:`execnet` / :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.portal`. +""" + +from __future__ import annotations + +import importlib +import warnings +from typing import Any + +#: shims are scheduled for removal in this release +REMOVED_IN = "execnet 3.0" + + +def forwarder(shim: str, moved: dict[str, str]) -> Any: + """Build the ``__getattr__`` for a deprecated module. + + ``moved`` maps each previously-reachable name to the private module that + now defines it (a relative name such as ``"._channel"``). + """ + + def __getattr__(name: str) -> Any: + try: + module = moved[name] + except KeyError: + raise AttributeError( + f"module 'execnet.{shim}' has no attribute {name!r}" + ) from None + warnings.warn( + f"execnet.{shim} is private and will be removed in {REMOVED_IN}; " + f"{name} now lives in execnet{module}. The supported surfaces are " + f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.portal.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module(module, __package__), name) + + return __getattr__ diff --git a/src/execnet/_trace.py b/src/execnet/_trace.py new file mode 100644 index 00000000..edb650d8 --- /dev/null +++ b/src/execnet/_trace.py @@ -0,0 +1,47 @@ +"""Debug tracing, configured once from ``EXECNET_DEBUG``. + +:EXECNET_DEBUG=1: write per-process trace files to ``execnet-debug-PID`` +:EXECNET_DEBUG=2: trace to stderr (popen workers forward this to their + instantiator) + +Unset, ``trace`` is a no-op lambda so tracing costs a call and nothing else. +""" + +from __future__ import annotations + +import os +import sys + +DEBUG = os.environ.get("EXECNET_DEBUG") +pid = os.getpid() + +if DEBUG == "2": + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + sys.stderr.write(f"[{pid}] {line}\n") + sys.stderr.flush() + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +elif DEBUG: + import tempfile + + fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) + # sys.stderr.write("execnet-debug at %r" % (fn,)) + debugfile = open(fn, "w") + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + debugfile.write(line + "\n") + debugfile.flush() + except Exception as exc: + try: + sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +else: + notrace = trace = lambda *msg: None diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 3972db4f..90246fb7 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1,6 +1,7 @@ """Trio-native gateway core: async dispatch loop and low-level raw channels. -Async-first counterpart of the sync machinery in ``gateway_base``: an +Async-first counterpart of the sync machinery in ``_channel`` / +``_gateway_base``: an :class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline on that task, so there is no receiver thread and no receive lock. @@ -37,17 +38,17 @@ if TYPE_CHECKING: from typing_extensions import Self +from ._errors import GatewayReceivedTerminate +from ._errors import HostNotFound +from ._errors import RemoteError +from ._errors import TimeoutError from ._exec_source import normalize_exec_source -from .gateway_base import FrameDecoder -from .gateway_base import GatewayReceivedTerminate -from .gateway_base import HostNotFound -from .gateway_base import Message -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dumps_internal -from .gateway_base import gateway_info -from .gateway_base import loads_internal -from .gateway_base import trace +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace RECEIVE_CHUNK = 65536 @@ -898,7 +899,7 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: ``installvia=``), and ``via=`` sub-gateways relayed through a group member. """ - from .xspec import XSpec + from ._xspec import XSpec if self._nursery is None: raise RuntimeError(f"{self!r} is not entered") diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1d9149dd..a1ea4acb 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -26,6 +26,17 @@ import trio from ._boundary import Flag +from ._channel import ENDMARKER +from ._channel import NO_ENDMARKER_WANTED +from ._errors import GatewayReceivedTerminate +from ._errors import RemoteError +from ._execmodel import ExecModel +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup @@ -34,17 +45,6 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from .gateway_base import ENDMARKER -from .gateway_base import NO_ENDMARKER_WANTED -from .gateway_base import ExecModel -from .gateway_base import FrameDecoder -from .gateway_base import GatewayReceivedTerminate -from .gateway_base import Message -from .gateway_base import RemoteError -from .gateway_base import dumps_internal -from .gateway_base import gateway_info -from .gateway_base import loads_internal -from .gateway_base import trace from .portal import LoopPortal from .portal import OneShot @@ -60,9 +60,9 @@ def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> if TYPE_CHECKING: - from .gateway import Gateway - from .gateway_base import BaseGateway - from .multi import Group + from ._gateway import Gateway + from ._gateway_base import BaseGateway + from ._multi import Group T = TypeVar("T") diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 4f3a65f2..633b7d94 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -12,18 +12,18 @@ import trio -from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT -from .gateway_base import WorkerGateway -from .gateway_base import get_execmodel -from .gateway_base import geterrortext -from .gateway_base import loads_internal -from .gateway_base import trace +from ._errors import MAIN_THREAD_ONLY_DEADLOCK_TEXT +from ._errors import geterrortext +from ._execmodel import get_execmodel +from ._gateway_base import WorkerGateway +from ._serialize import loads_internal +from ._trace import trace from .portal import Mailbox if TYPE_CHECKING: from . import _trio_host - from .gateway_base import Channel - from .gateway_base import ExecModel + from ._channel import Channel + from ._execmodel import ExecModel ExecItem = tuple[Any, ...] diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py new file mode 100644 index 00000000..afc58f4c --- /dev/null +++ b/src/execnet/_xspec.py @@ -0,0 +1,74 @@ +""" +(c) 2008-2013, holger krekel +""" + +from __future__ import annotations + + +class XSpec: + """Execution Specification: key1=value1//key2=value2 ... + + * Keys need to be unique within the specification scope + * Neither key nor value are allowed to contain "//" + * Keys are not allowed to contain "=" + * Keys are not allowed to start with underscore + * If no "=value" is given, assume a boolean True value + """ + + # XXX allow customization, for only allow specific key names + chdir: str | None = None + dont_write_bytecode: bool | None = None + execmodel: str | None = None + id: str | None = None + installvia: str | None = None + nice: str | None = None + popen: bool | None = None + python: str | None = None + socket: str | None = None + ssh: str | None = None + ssh_config: str | None = None + vagrant_ssh: str | None = None + via: str | None = None + wait: str | None = None + + def __init__(self, string: str) -> None: + self._spec = string + self.env = {} + for keyvalue in string.split("//"): + i = keyvalue.find("=") + value: str | bool + if i == -1: + key, value = keyvalue, True + else: + key, value = keyvalue[:i], keyvalue[i + 1 :] + if key[0] == "_": + raise AttributeError("%r not a valid XSpec key" % key) + if key in self.__dict__: + raise ValueError(f"duplicate key: {key!r} in {string!r}") + if key.startswith("env:"): + self.env[key[4:]] = value + else: + setattr(self, key, value) + + def __getattr__(self, name: str) -> None | bool | str: + if name[0] == "_": + raise AttributeError(name) + return None + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self._spec + + def __hash__(self) -> int: + return hash(self._spec) + + def __eq__(self, other: object) -> bool: + return self._spec == getattr(other, "_spec", None) + + def __ne__(self, other: object) -> bool: + return self._spec != getattr(other, "_spec", None) + + def _samefilesystem(self) -> bool: + return self.popen is not None and self.chdir is None diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 7151ab5d..36bd8751 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -26,7 +26,8 @@ async def main(): The error types are shared with :mod:`execnet.sync` and :mod:`execnet.trio`. Items you send must already be simple builtin data (plus channels); the standalone serializer is intentionally not part of -the public API -- see ``DumpError``. +the public API -- ``execnet.can_send`` checks a value before you send it; +see ``DumpError``. """ from __future__ import annotations @@ -45,17 +46,17 @@ async def main(): import trio +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup from ._trio_host import TrioHost -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .xspec import XSpec +from ._xspec import XSpec if TYPE_CHECKING: from typing_extensions import Self diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index ac41705c..d15263ca 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -1,161 +1,19 @@ -"""Gateway code for initiating popen, socket and ssh connections. +"""Deprecated alias for :mod:`execnet._gateway`. -(c) 2004-2013, Holger Krekel and others +``Gateway`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import types -from collections.abc import Callable -from typing import TYPE_CHECKING -from typing import Any +from ._shim import forwarder -from . import gateway_base -from ._exec_source import _find_non_builtin_globals -from ._exec_source import _source_of_function -from ._exec_source import normalize_exec_source -from .gateway_base import IO -from .gateway_base import Channel -from .gateway_base import Message -from .multi import Group -from .xspec import XSpec +_MOVED = { + "Gateway": "._gateway", + "RInfo": "._gateway", + "RemoteStatus": "._gateway", + "normalize_exec_source": "._exec_source", + "_find_non_builtin_globals": "._exec_source", + "_source_of_function": "._exec_source", +} -__all__ = [ - "Gateway", - "RInfo", - "RemoteStatus", - "_find_non_builtin_globals", - "_source_of_function", -] - - -class Gateway(gateway_base.BaseGateway): - """Gateway to a local or remote Python Interpreter.""" - - _group: Group - - def __init__(self, io: IO, spec: XSpec) -> None: - """:private: - - The Trio session doing the Message IO is attached separately via - ``_attach_trio_session`` once the connection is established. - """ - super().__init__(io=io, id=spec.id, _startcount=1) - self.spec = spec - if spec.wait: - self._wait_backend = spec.wait - - @property - def remoteaddress(self) -> str: - # Only defined for remote IO types. - return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] - - def __repr__(self) -> str: - """A string representing gateway type and status.""" - try: - r: str = (self.hasreceiver() and "receive-live") or "not-receiving" - i = str(len(self._channelfactory.channels())) - except AttributeError: - r = "uninitialized" - i = "no" - return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" - - def exit(self) -> None: - """Trigger gateway exit. - - Defer waiting for finishing of receiver-thread and subprocess activity - to when group.terminate() is called. - """ - self._trace("gateway.exit() called") - if self not in self._group: - self._trace("gateway already unregistered with group") - return - self._group._unregister(self) - try: - self._trace("--> sending GATEWAY_TERMINATE") - self._send(Message.GATEWAY_TERMINATE) - self._trace("--> io.close_write") - self._io.close_write() - except (ValueError, EOFError, OSError) as exc: - self._trace("io-error: could not send termination sequence") - self._trace(" exception: %r" % exc) - - def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote. - - A native protocol request (like ``remote_status``): it never - touches the exec machinery, so it cannot claim an exec slot on - main-thread-shaped workers. - """ - if update or not hasattr(self, "_cache_rinfo"): - channel = self.newchannel() - self._send(Message.GATEWAY_INFO, channel.id) - self._cache_rinfo = RInfo(channel.receive()) - # the other side didn't actually instantiate a channel - # so we just delete the internal id/channel mapping - self._channelfactory._local_close(channel.id) - return self._cache_rinfo - - def hasreceiver(self) -> bool: - """Whether gateway is able to receive data.""" - session = self._trio_session - return session is not None and bool(session.is_alive()) - - def remote_status(self) -> RemoteStatus: - """Obtain information about the remote execution status.""" - channel = self.newchannel() - self._send(Message.STATUS, channel.id) - statusdict = channel.receive() - # the other side didn't actually instantiate a channel - # so we just delete the internal id/channel mapping - self._channelfactory._local_close(channel.id) - return RemoteStatus(statusdict) - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs: object, - ) -> Channel: - """Return channel object and connect it to a remote - execution thread where the given ``source`` executes. - - * ``source`` is a string: execute source string remotely - with a ``channel`` put into the global namespace. - * ``source`` is a pure function: serialize source and - call function with ``**kwargs``, adding a - ``channel`` object to the keyword arguments. - * ``source`` is a pure module: execute source of module - with a ``channel`` in its global namespace. - - In all cases the binding ``__name__='__channelexec__'`` - will be available in the global namespace of the remotely - executing code. - """ - source, file_name, call_name = normalize_exec_source(source, kwargs) - channel = self.newchannel() - self._send( - Message.CHANNEL_EXEC, - channel.id, - gateway_base.dumps_internal((source, file_name, call_name, kwargs)), - ) - return channel - - def remote_init_threads(self, num: int | None = None) -> None: - """DEPRECATED. Is currently a NO-OPERATION already.""" - print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") - - -class RInfo: - def __init__(self, kwargs) -> None: - self.__dict__.update(kwargs) - - def __repr__(self) -> str: - info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) - return "" % info - - if TYPE_CHECKING: - - def __getattr__(self, name: str) -> Any: ... - - -RemoteStatus = RInfo +__getattr__ = forwarder("gateway", _MOVED) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index ce19eb0f..b54c68bf 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,1449 +1,79 @@ -"""Core gateway, channel and serialization code shared by coordinator and worker. +"""Deprecated: the pre-Trio core module, now split by concern. -:copyright: 2004-2015 -:authors: - - Holger Krekel - - Armin Rigo - - Benjamin Peterson - - Ronny Pfannschmidt - - many others +``execnet.gateway_base`` was never a supported public API. Its contents live +in private modules -- ``_trace``, ``_errors``, ``_execmodel``, ``_message``, +``_serialize``, ``_channel`` and ``_gateway_base`` -- and this shim forwards +to them with a :class:`DeprecationWarning`. + +Use :mod:`execnet` / :mod:`execnet.sync`, :mod:`execnet.trio`, +:mod:`execnet.aio` or :mod:`execnet.portal` instead. The standalone +serializer stays internal; :func:`execnet.can_send` answers "can this value +cross a channel?" without it. """ from __future__ import annotations -import builtins -import os -import struct -import sys -import threading -import traceback -import weakref -from _thread import interrupt_main -from collections.abc import Callable -from collections.abc import Iterator -from contextlib import suppress -from io import BytesIO -from typing import Any -from typing import Literal -from typing import Protocol -from typing import cast -from typing import overload - -from ._boundary import Flag -from ._boundary import Mailbox -from ._boundary import Wakener -from ._boundary import make_wakener - - -class WriteIO(Protocol): - def write(self, data: bytes, /) -> None: ... - - -class ReadIO(Protocol): - def read(self, numbytes: int, /) -> bytes: ... - - -class IO(Protocol): - execmodel: ExecModel - - def read(self, numbytes: int, /) -> bytes: ... - - def write(self, data: bytes, /) -> None: ... - - def close_read(self) -> None: ... - - def close_write(self) -> None: ... - - def wait(self) -> int | None: ... - - def kill(self) -> None: ... - - -class ExecModel: - """Deprecated preset name for an execution model. - - The machinery behind execution models was retired: protocol IO always - runs on the Trio host and blocking waits go through the boundary kit's - wakeners (``execnet._boundary``); the name maps onto the worker config - axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating - members stay for API compatibility (pytest-xdist builds its test queue - on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. - """ - - def __init__(self, backend: str) -> None: - self.backend = backend - - def __repr__(self) -> str: - return "" % self.backend - - @property - def queue(self): - import queue - - return queue - - @property - def subprocess(self): - import subprocess - - return subprocess - - @property - def socket(self): - import socket - - return socket - - def get_ident(self) -> int: - import _thread - - return _thread.get_ident() - - def sleep(self, delay: float) -> None: - import time - - time.sleep(delay) - - def start(self, func, args=()) -> None: - import _thread - - _thread.start_new_thread(func, args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) - - def Lock(self): - return threading.RLock() - - def RLock(self): - return threading.RLock() - - def Event(self) -> threading.Event: - return threading.Event() - - -#: worker profiles: where exec'd code runs relative to the protocol loop -EXECMODEL_PROFILES = ( - "thread", # hybrid: primary on the main thread, overflow on pool threads - "main_thread_only", # exec serialized on the main thread (GUI/pytest) - "trio", # pure async: loop owns the main thread, async sources as tasks - "gevent", # greenlets on a main-thread hub, one per remote_exec -) - - -def get_execmodel(backend: str | ExecModel) -> ExecModel: - if isinstance(backend, ExecModel): - return backend - if backend in EXECMODEL_PROFILES: - return ExecModel(backend) - raise ValueError(f"unknown execmodel {backend!r}") - - -sysex = (KeyboardInterrupt, SystemExit) - - -DEBUG = os.environ.get("EXECNET_DEBUG") -pid = os.getpid() -if DEBUG == "2": - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - sys.stderr.write(f"[{pid}] {line}\n") - sys.stderr.flush() - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -elif DEBUG: - import os - import tempfile - - fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) - # sys.stderr.write("execnet-debug at %r" % (fn,)) - debugfile = open(fn, "w") - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - debugfile.write(line + "\n") - debugfile.flush() - except Exception as exc: - try: - sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -else: - notrace = trace = lambda *msg: None - - -class Message: - """Encapsulates Messages and their wire protocol. - - Dispatch lives in the async core and the sync bridge session - (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this - class only carries the framing and the code constants. - """ - - STATUS = 0 - #: retired: the py2/py3 string coercion switch. Nothing sends or - #: handles it anymore, the code stays reserved for reuse. - RECONFIGURE = 1 - GATEWAY_TERMINATE = 2 - CHANNEL_EXEC = 3 - CHANNEL_DATA = 4 - CHANNEL_CLOSE = 5 - CHANNEL_CLOSE_ERROR = 6 - CHANNEL_LAST_MESSAGE = 7 - GATEWAY_START_SOCKET = 8 - GATEWAY_START_SUB = 9 - GATEWAY_INFO = 10 - - # message code -> name - _types: dict[int, str] = { - STATUS: "STATUS", - RECONFIGURE: "RECONFIGURE", - GATEWAY_TERMINATE: "GATEWAY_TERMINATE", - CHANNEL_EXEC: "CHANNEL_EXEC", - CHANNEL_DATA: "CHANNEL_DATA", - CHANNEL_CLOSE: "CHANNEL_CLOSE", - CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", - CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", - GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", - GATEWAY_START_SUB: "GATEWAY_START_SUB", - GATEWAY_INFO: "GATEWAY_INFO", - } - - def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - self.msgcode = msgcode - self.channelid = channelid - self.data = data - - def pack(self) -> bytes: - """Return the full wire frame (9-byte header + payload).""" - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - return header + self.data - - @staticmethod - def from_header(header: bytes) -> tuple[int, int, int]: - """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" - if len(header) != 9: - raise EOFError("couldn't load message header, short read") - msgtype, channel, payload = struct.unpack("!bii", header) - return msgtype, channel, payload - - @staticmethod - def from_parts(msgtype: int, channel: int, data: bytes) -> Message: - return Message(msgtype, channel, data) - - @staticmethod - def from_io(io: ReadIO) -> Message: - try: - header = io.read(9) # type 1, channel 4, payload 4 - if not header: - raise EOFError("empty read") - except EOFError as e: - raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = Message.from_header(header) - return Message(msgtype, channel, io.read(payload)) - - def to_io(self, io: WriteIO) -> None: - io.write(self.pack()) - - def __repr__(self) -> str: - name = self._types[self.msgcode] - return f"" - - -def gateway_info() -> dict[str, object]: - """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. - - Answered natively by the dispatch loop -- an info request never - touches the exec machinery, so it cannot claim an exec slot (with - main-thread profiles, an info call stealing the primary slot used to - push the real workload onto a worker thread). - """ - return { - "executable": sys.executable, - "version_info": tuple(sys.version_info[:5]), - "platform": sys.platform, - "cwd": os.getcwd(), - "pid": os.getpid(), - } - - -class FrameDecoder: - """Incremental decoder for the 9-byte-header Message framing. - - ``feed(data)`` accepts arbitrary byte chunks and yields every complete - Message; partial frames buffer internally until more bytes arrive. - Pure computation — no IO, no awaits, no knowledge of streams — so - receivers only ever stream bytes in (``receive_some`` loops) and the - decoder owns framing. - """ - - def __init__(self) -> None: - self._buffer = bytearray() - - def feed(self, data: bytes) -> Iterator[Message]: - self._buffer += data - return self._parse() - - def _parse(self) -> Iterator[Message]: - while len(self._buffer) >= 9: - msgtype, channelid, payload_len = Message.from_header( - bytes(self._buffer[:9]) - ) - if len(self._buffer) < 9 + payload_len: - return - payload = bytes(self._buffer[9 : 9 + payload_len]) - del self._buffer[: 9 + payload_len] - yield Message(msgtype, channelid, payload) - - def close(self) -> None: - """Signal EOF; raises EOFError if the stream ended mid-frame.""" - if self._buffer: - raise EOFError( - "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) - ) - - -class GatewayReceivedTerminate(Exception): - """Receiver got a gateway termination message.""" - - -class HostNotFound(ConnectionError): - """The remote side of a gateway could not be reached.""" - - -def geterrortext( - exc: BaseException, - format_exception=traceback.format_exception, - sysex: tuple[type[BaseException], ...] = sysex, -) -> str: - try: - # In py310, can change this to: - # l = format_exception(exc) - l = format_exception(type(exc), exc, exc.__traceback__) - errortext = "".join(l) - except sysex: - raise - except BaseException: - errortext = f"{type(exc).__name__}: {exc}" - return errortext - - -class RemoteError(Exception): - """Exception containing a stringified error from the other side.""" - - def __init__(self, formatted: str) -> None: - super().__init__() - self.formatted = formatted - - def __str__(self) -> str: - return self.formatted - - def __repr__(self) -> str: - return f"{self.__class__.__name__}: {self.formatted}" - - def warn(self) -> None: - if self.formatted != INTERRUPT_TEXT: - # XXX do this better - sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") - - -class TimeoutError(IOError): - """Exception indicating that a timeout was reached.""" - - -NO_ENDMARKER_WANTED = object() - - -class Channel: - """Communication channel between two Python Interpreter execution points. - - A facade over the async core: the gateway's Trio session diverts - inbound payloads for this id into a :class:`Mailbox` (or a registered - callback, invoked on the loop thread); ``receive()`` deserializes at - the call site. Sends go through ``gateway._send``. - """ - - RemoteError = RemoteError - TimeoutError = TimeoutError - _INTERNALWAKEUP = 1000 - _executing = False - #: set once a receiver callback is attached. A consumer *task* on the - #: loop drains this channel and runs the callback in a threadpool thread; - #: the task holds the channel alive, so a callback channel's lifecycle is - #: bound to consumption (and to GC once the stream closes) rather than to - #: a strong registry. - _has_consumer = False - #: loop-side hooks installed while a consumer is attached: divert one - #: payload into / close the consumer task's inbox (set by the Trio session, - #: so they encapsulate the trio memory channel; gateway_base stays trio-free). - _consumer_feed: Callable[[bytes], None] | None = None - _consumer_close_inbox: Callable[[], None] | None = None - #: thread-safe "stop the consumer" hook -- ends the task's inbox. - _consumer_stop: Callable[[], None] | None = None - #: set by the consumer task once it has drained every item and fired the - #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) - #: so it still guarantees "all callbacks have run" before returning. - _consumer_done: Flag | None = None - - def __init__(self, gateway: BaseGateway, id: int) -> None: - """:private:""" - assert isinstance(id, int) - assert not isinstance(gateway, type) - self.gateway = gateway - self.id = id - # serialized payloads (or ENDMARKER); None once a consumer is attached - self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) - self._closed = False - self._receiveclosed = Flag(gateway._new_wakener()) - self._remoteerrors: list[RemoteError] = [] - - def _trace(self, *msg: object) -> None: - self.gateway._trace(self.id, *msg) - - def setcallback( - self, - callback: Callable[[Any], Any], - endmarker: object = NO_ENDMARKER_WANTED, - ) -> None: - """Set a callback function for receiving items. - - A consumer task on the gateway's loop drains this channel and runs - ``callback`` for each received item in a threadpool thread (so a slow - callback never blocks the loop); items for one channel are delivered - strictly in order. Already-queued items are delivered first. After - this call ``receive()`` raises an error. - - The task keeps the channel alive for as long as it is consuming, so a - callback channel need not be referenced elsewhere. If an endmarker is - specified the callback is eventually called with it when the channel - closes, and ``waitclose()`` does not return until every callback - (including the endmarker) has run. - """ - self.gateway._start_channel_consumer(self, callback, endmarker) - - def __repr__(self) -> str: - flag = (self.isclosed() and "closed") or "open" - return "" % (self.id, flag) - - def __del__(self) -> None: - if self.gateway is None: # can be None in tests - return # type: ignore[unreachable] - - self._trace("channel.__del__") - # no multithreading issues here, because we have the last ref to 'self' - if self._closed: - # state transition "closed" --> "deleted" - for error in self._remoteerrors: - error.warn() - elif self._receiveclosed.is_set(): - # state transition "sendonly" --> "deleted" - # the remote channel is already in "deleted" state, nothing to do - pass - else: - # state transition "opened" --> "deleted" - # check if we are in the middle of interpreter shutdown - # in which case the process will go away and we probably - # don't need to try to send a closing or last message - # (and often it won't work anymore to send things out) - # A callback channel is held by its consumer task until the stream - # closes, so by the time __del__ runs it is never in the "opened" - # state -- this branch only ever fires for a plain receive channel. - if Message is not None: - with suppress(OSError, ValueError): # ignore problems with sending - # Never wait during GC: post the close best-effort. - send = getattr( - self.gateway, "_send_nonblocking", self.gateway._send - ) - send(Message.CHANNEL_CLOSE, self.id) - with suppress(Exception): - self.gateway._release_channel(self.id) - - def _getremoteerror(self): - try: - return self._remoteerrors.pop(0) - except IndexError: - try: - return self.gateway._error - except AttributeError: - pass - return None - - # - # loop-side delivery (called by the session's raw-channel consumer) - # - def _deliver_payload(self, data: bytes) -> None: - """Route one inbound serialized payload (loop thread). - - A callback channel diverts payloads into its consumer task's inbox; a - plain channel queues them for ``receive()``. - """ - if self._closed: - return # late data for a locally closed channel: drop - feed = self._consumer_feed - if feed is not None: - feed(data) - return - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(data) - # no consumer and no mailbox: closed for receiving -- drop - - def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: - """Close initiated by the peer or session shutdown (loop thread). - - For a callback channel the consumer task ends separately (its inbox is - closed) and fires the endmarker; here we only record the state. - """ - if remoteerror: - self._remoteerrors.append(remoteerror) - if self._has_consumer: - close_inbox = self._consumer_close_inbox - if close_inbox is not None: - close_inbox() # ends the consumer task (it fires the endmarker) - else: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self.gateway._channelfactory._no_longer_opened(self.id) - if not sendonly: # otherwise #--> "sendonly" - self._closed = True # --> "closed" - self._receiveclosed.set() - - # - # public API for channel objects - # - def isclosed(self) -> bool: - """Return True if the channel is closed. - - A closed channel may still hold items. - """ - return self._closed - - @overload - def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: - pass - - @overload - def makefile( - self, - mode: Literal["w"] = ..., - proxyclose: bool = ..., - ) -> ChannelFileWrite: - pass - - def makefile( - self, - mode: Literal["r", "w"] = "w", - proxyclose: bool = False, - ) -> ChannelFileWrite | ChannelFileRead: - """Return a file-like object. - - mode can be 'w' or 'r' for writeable/readable files. - If proxyclose is true, file.close() will also close the channel. - """ - if mode == "w": - return ChannelFileWrite(channel=self, proxyclose=proxyclose) - elif mode == "r": - return ChannelFileRead(channel=self, proxyclose=proxyclose) - raise ValueError(f"mode {mode!r} not available") - - def close(self, error=None) -> None: - """Close down this channel with an optional error message. - - Note that closing of a channel tied to remote_exec happens - automatically at the end of execution and cannot - be done explicitly. - """ - if self._executing: - raise OSError("cannot explicitly close channel within remote_exec") - if self._closed: - self.gateway._trace(self, "ignoring redundant call to close()") - if not self._closed: - # state transition "opened/sendonly" --> "closed" - # threads warning: the channel might be closed under our feet, - # but it's never damaging to send too many CHANNEL_CLOSE messages - # however, if the other side triggered a close already, we - # do not send back a closed message. - if not self._receiveclosed.is_set(): - put = self.gateway._send - if error is not None: - put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) - else: - put(Message.CHANNEL_CLOSE, self.id) - self._trace("sent channel close message") - if isinstance(error, RemoteError): - self._remoteerrors.append(error) - self._closed = True # --> "closed" - self._receiveclosed.set() - if self._has_consumer: - # End the consumer task's inbox; it drains any buffered items, - # fires the endmarker, and sets _consumer_done. - stop = self._consumer_stop - if stop is not None: - stop() - else: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self.gateway._channelfactory._no_longer_opened(self.id) - self.gateway._release_channel(self.id) - - def waitclose(self, timeout: float | None = None) -> None: - """Wait until this channel is closed (or the remote side - otherwise signalled that no more data was being sent). - - The channel may still hold receiveable items, but not receive - any more after waitclose() has returned. - - Exceptions from executing code on the other side are reraised as local - channel.RemoteErrors. - - EOFError is raised if the reading-connection was prematurely closed, - which often indicates a dying process. - - self.TimeoutError is raised after the specified number of seconds - (default is None, i.e. wait indefinitely). - """ - # For a callback channel wait on the consumer task finishing (so every - # callback, including the endmarker, has run); otherwise wait for the - # non-"opened" state directly. - signal = ( - self._consumer_done - if self._consumer_done is not None - else (self._receiveclosed) - ) - signal.wait(timeout=timeout) - if not signal.is_set(): - raise self.TimeoutError("Timeout after %r seconds" % timeout) - error = self._getremoteerror() - if error: - raise error - - def send(self, item: object) -> None: - """Sends the given item to the other side of the channel, - possibly blocking if the sender queue is full. - - The item must be a simple Python type and will be - copied to the other side by value. - - OSError is raised if the write pipe was prematurely closed. - """ - if self.isclosed(): - raise OSError(f"cannot send to {self!r}") - self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) - - def receive(self, timeout: float | None = None) -> Any: - """Receive a data item that was sent from the other side. - - timeout: None [default] blocked waiting. A positive number - indicates the number of seconds after which a channel.TimeoutError - exception will be raised if no item was received. - - Note that exceptions from the remotely executing code will be - reraised as channel.RemoteError exceptions containing - a textual representation of the remote traceback. - """ - mailbox = self._mailbox - if mailbox is None: - raise OSError("cannot receive(), channel has receiver callback") - try: - x = mailbox.get(timeout) - except builtins.TimeoutError: - raise self.TimeoutError("no item after %r seconds" % timeout) from None - if x is ENDMARKER: - mailbox.put(x) # for other receivers - raise self._getremoteerror() or EOFError() - else: - return loads_internal(x, self) - - def __iter__(self) -> Iterator[Any]: - return self - - def next(self) -> Any: - try: - return self.receive() - except EOFError: - raise StopIteration from None - - __next__ = next - - -ENDMARKER = object() -INTERRUPT_TEXT = "keyboard-interrupted" -MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( - "concurrent remote_exec would cause deadlock for main_thread_only execmodel" -) - - -class ChannelFactory: - """Registry and id allocator for a gateway's sync channels. - - Message routing lives in the Trio session (the sync channel binds a - consumer on the session's raw channel); the factory only tracks live - channels -- weakly, so dropping the last user reference triggers - ``Channel.__del__``'s close message. A callback channel is kept alive by - its consumer task rather than by any registry here. - """ - - def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: - self._channels: weakref.WeakValueDictionary[int, Channel] = ( - weakref.WeakValueDictionary() - ) - self._writelock = threading.Lock() - self.gateway = gateway - self.count = startcount - self.finished = False - self._list = list # needed during interp-shutdown - - def new(self, id: int | None = None) -> Channel: - """Create a new Channel with 'id' (or create new id if None).""" - with self._writelock: - if self.finished: - raise OSError(f"connection already closed: {self.gateway}") - if id is None: - id = self.count - self.count += 2 - try: - channel = self._channels[id] - except KeyError: - channel = self._channels[id] = Channel(self.gateway, id) - self.gateway._bind_channel(channel) - return channel - - def allocate_id(self) -> int: - """Reserve a fresh channel id without creating a Channel object.""" - with self._writelock: - if self.finished: - raise OSError(f"connection already closed: {self.gateway}") - id = self.count - self.count += 2 - return id - - def channels(self) -> list[Channel]: - return self._list(self._channels.values()) - - # - # internal methods, called from the loop thread (or local close paths) - # - def _no_longer_opened(self, id: int) -> None: - self._channels.pop(id, None) - - def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - """Close ``id`` as if the peer had closed it (no message is sent).""" - channel = self._channels.get(id) - if channel is None: - # channel already in "deleted" state - if remoteerror: - remoteerror.warn() - self._no_longer_opened(id) - else: - channel._close_from_remote(remoteerror, sendonly=sendonly) - - def _finished_receiving(self) -> None: - with self._writelock: - self.finished = True - for id in self._list(self._channels): - self._local_close(id, sendonly=True) - - -class ChannelFile: - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - self.channel = channel - self._proxyclose = proxyclose - - def isatty(self) -> bool: - return False - - def close(self) -> None: - if self._proxyclose: - self.channel.close() - - def __repr__(self) -> str: - state = (self.channel.isclosed() and "closed") or "open" - return "" % (self.channel.id, state) - - -class ChannelFileWrite(ChannelFile): - def write(self, out: bytes) -> None: - self.channel.send(out) - - def flush(self) -> None: - pass - - -class ChannelFileRead(ChannelFile): - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - super().__init__(channel, proxyclose) - self._buffer: str | None = None - - def read(self, n: int) -> str: - try: - if self._buffer is None: - self._buffer = cast(str, self.channel.receive()) - while len(self._buffer) < n: - self._buffer += cast(str, self.channel.receive()) - except EOFError: - self.close() - if self._buffer is None: - ret = "" - else: - ret = self._buffer[:n] - self._buffer = self._buffer[n:] - return ret - - def readline(self) -> str: - if self._buffer is not None: - i = self._buffer.find("\n") - if i != -1: - return self.read(i + 1) - line = self.read(len(self._buffer) + 1) - else: - line = self.read(1) - while line and line[-1] != "\n": - c = self.read(1) - if not c: - break - line += c - return line - - -class BaseGateway: - _sysex = sysex - id = "" - _trio_session: Any = None - # Set by the receiver on EOF without a prior termination message. - _error: BaseException | None = None - #: wait= axis: which wakener backend this gateway's blocking waits - #: park on (channels, write-acks, join) - _wait_backend: str = "thread" - - def __init__(self, io: IO, id, _startcount: int = 2) -> None: - self.execmodel = io.execmodel - self._io = io - self.id = id - self._channelfactory = ChannelFactory(self, _startcount) - # globals may be NONE at process-termination - self.__trace = trace - self._geterrortext = geterrortext - self._trio_session = None - - def _trace(self, *msg: object) -> None: - self.__trace(self.id, *msg) - - def _attach_trio_session(self, session: Any) -> None: - """Attach the Trio bridge session doing the Message IO.""" - self._trio_session = session - # Defensive: channels created before the session existed still - # need their inbound routing diverted to them. - for channel in self._channelfactory.channels(): - session.bind_sync_channel(channel) - - def _new_wakener(self) -> Wakener: - """A fresh wakener for one blocking-wait carrier (wait= axis).""" - return make_wakener(self._wait_backend) - - def _bind_channel(self, channel: Channel) -> None: - """Divert the session's inbound routing for ``channel.id`` to it.""" - session = self._trio_session - if session is not None: - session.bind_sync_channel(channel) - - def _release_channel(self, id: int) -> None: - """Drop the session's loop-side state for ``id`` (best-effort).""" - session = self._trio_session - if session is not None: - with suppress(Exception): - session.release_channel(id) - - def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: - """Run ``sync_fn`` on the session's loop thread (inline without one). - - Payload dispatch happens on the loop thread, so state switches run - there to exclude interleaving with deliveries. - """ - session = self._trio_session - if session is None: - return sync_fn() - return session.run_on_loop(sync_fn) - - def _start_channel_consumer( - self, - channel: Channel, - callback: Callable[[Any], Any], - endmarker: object, - ) -> None: - """Attach a receiver callback: hand the channel to a consumer task. - - The task drains the channel on the loop and runs ``callback`` in a - threadpool thread, holding the channel alive while it consumes. - """ - session = self._trio_session - if session is None: - raise OSError(f"cannot set callback on {channel!r}: no active session") - session.attach_consumer(channel, callback, endmarker) - - def _terminate_execution(self) -> None: - pass - - def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - message = Message(msgcode, channelid, data) - session = self._trio_session - if session is not None: - try: - session.enqueue_message(message) - self._trace("sent", message) - except (OSError, ValueError) as e: - self._trace("failed to send", message, e) - raise OSError("cannot send (already closed?)") from e - return - try: - message.to_io(self._io) - self._trace("sent", message) - except (OSError, ValueError) as e: - self._trace("failed to send", message, e) - # ValueError might be because the IO is already closed - raise OSError("cannot send (already closed?)") from e - - def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: - """Best-effort send that never waits (used during GC). - - Safe to call from any thread, including while the interpreter or - the IO loop is shutting down. - """ - message = Message(msgcode, channelid) - session = self._trio_session - if session is not None: - session.post_message(message) - return - message.to_io(self._io) - - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - channel.close("execution disallowed") - - # _____________________________________________________________________ - # - # High Level Interface - # _____________________________________________________________________ - # - def newchannel(self) -> Channel: - """Return a new independent channel.""" - return self._channelfactory.new() - - def join(self, timeout: float | None = None) -> None: - """Wait for the receiver (Trio session) to terminate.""" - self._trace("waiting for receiver to finish") - session = self._trio_session - if session is not None: - session.wait_done(timeout) - - -class WorkerGateway(BaseGateway): - _trio_exec: Any = None - # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). - _execpool: Any = None - _executetask_complete: threading.Event | None = None - - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - trio_exec = self._trio_exec - if trio_exec is None: - channel.close("execution disallowed") - return - trio_exec.schedule(channel, sourcetask) - - def _terminate_execution(self) -> None: - # called from receiverthread - self._trace("shutting down execution pool") - self._execpool.trigger_shutdown() - if not self._execpool.waitall(5.0): - self._trace("execution ongoing after 5 secs, trying interrupt_main") - # We try hard to terminate execution based on the assumption - # that there is only one gateway object running per-process. - if sys.platform != "win32": - self._trace("sending ourselves a SIGINT") - os.kill(os.getpid(), 2) # send ourselves a SIGINT - elif interrupt_main is not None: - self._trace("calling interrupt_main()") - interrupt_main() - if not self._execpool.waitall(10.0): - self._trace( - "execution did not finish in another 10 secs, calling os._exit()" - ) - os._exit(1) - - def executetask( - self, - item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], - ) -> None: - try: - channel, (source, file_name, call_name, kwargs) = item - loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} - self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") - channel._executing = True - try: - co = compile(source + "\n", file_name or "", "exec") - exec(co, loc) - if call_name: - self._trace("calling %s(**%60r)" % (call_name, kwargs)) - function = loc[call_name] - function(channel, **kwargs) - finally: - channel._executing = False - self._trace("execution finished") - except KeyboardInterrupt: - channel.close(INTERRUPT_TEXT) - raise - except EOFError: - self._trace("ignoring EOFError because receiving finished") - - except BaseException as exc: - if not channel.gateway._channelfactory.finished: - self._trace(f"got exception: {exc!r}") - errortext = self._geterrortext(exc) - channel.close(errortext) - return - channel.close() - if self._executetask_complete is not None: - # Indicate that this task has finished executing, meaning - # that there is no possibility of it triggering a deadlock - # for the next spawn call. - self._executetask_complete.set() - - -# -# Cross-Python pickling code, tested from test_serializer.py -# - - -class DataFormatError(Exception): - """A value could not cross the channel in execnet's simple wire format. - - execnet only moves *simple* builtin data over a channel -- ``None``, - ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and - (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of - those -- plus channel references, which pass through as channels. It does - **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, - pydantic models, numpy arrays, etc. have no wire representation. - - A ``DataFormatError`` therefore signals a caller error to resolve, not a - transport failure: reduce the value to simple data before sending (and - reconstruct it after receiving) with an encoding mechanism of your own -- - e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's - ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` - hooks. See the docs, "Sending objects over a channel". - """ - - -class DumpError(DataFormatError): - """A value being **sent** is not simple wire data; convert it first. - - Raised by ``channel.send`` (and the internal serializer) when an object is - not one of execnet's simple wire types. Fix it at the call site by turning - the rich object into simple data -- e.g. ``dt.isoformat()``, - ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather - than expecting the channel to pickle it. Channels are the one non-builtin - that *is* sendable, so nested channel references are fine. - """ - - -class LoadError(DataFormatError): - """Received bytes could not be turned back into an object. - - Raised while **receiving** (deserializing) -- a corrupted or - protocol-incompatible payload, or data produced by a mismatched peer. - """ - - -def bchr(n: int) -> bytes: - return bytes([n]) - - -DUMPFORMAT_VERSION = bchr(2) - -FOUR_BYTE_INT_MAX = 2147483647 -FOUR_BYTE_INT_MIN = -2147483648 - -FLOAT_FORMAT = "!d" -FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) -COMPLEX_FORMAT = "!dd" -COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) - - -class _Stop(Exception): - pass - - -class opcode: - """Container for name -> num mappings.""" - - BUILDTUPLE = b"@" - BYTES = b"A" - CHANNEL = b"B" - FALSE = b"C" - FLOAT = b"D" - FROZENSET = b"E" - INT = b"F" - LONG = b"G" - LONGINT = b"H" - LONGLONG = b"I" - NEWDICT = b"J" - NEWLIST = b"K" - NONE = b"L" - STRING = b"N" - SET = b"O" - SETITEM = b"P" - STOP = b"Q" - TRUE = b"R" - COMPLEX = b"T" - - -class Unserializer: - num2func: dict[bytes, Callable[[Unserializer], None]] = {} - - def __init__( - self, - stream: ReadIO, - channel_or_gateway: Channel | BaseGateway | None = None, - ) -> None: - if isinstance(channel_or_gateway, Channel): - gw: BaseGateway | None = channel_or_gateway.gateway - else: - gw = channel_or_gateway - self.stream = stream - if gw is None: - self.channelfactory = None - else: - self.channelfactory = gw._channelfactory - - def load(self, versioned: bool = False) -> Any: - if versioned: - ver = self.stream.read(1) - if ver != DUMPFORMAT_VERSION: - raise LoadError("wrong dumpformat version %r" % ver) - self.stack: list[object] = [] - try: - while True: - opcode = self.stream.read(1) - if not opcode: - raise EOFError - try: - loader = self.num2func[opcode] - except KeyError: - raise LoadError( - f"unknown opcode {opcode!r} - wire protocol corruption?" - ) from None - loader(self) - except _Stop: - if len(self.stack) != 1: - raise LoadError("internal unserialization error") from None - return self.stack.pop(0) - else: - raise LoadError("didn't get STOP") - - def load_none(self) -> None: - self.stack.append(None) - - num2func[opcode.NONE] = load_none - - def load_true(self) -> None: - self.stack.append(True) - - num2func[opcode.TRUE] = load_true - - def load_false(self) -> None: - self.stack.append(False) - - num2func[opcode.FALSE] = load_false - - def load_int(self) -> None: - i = self._read_int4() - self.stack.append(i) - - num2func[opcode.INT] = load_int - - def load_longint(self) -> None: - s = self._read_byte_string() - self.stack.append(int(s)) - - num2func[opcode.LONGINT] = load_longint - - load_long = load_int - num2func[opcode.LONG] = load_long - load_longlong = load_longint - num2func[opcode.LONGLONG] = load_longlong - - def load_float(self) -> None: - binary = self.stream.read(FLOAT_FORMAT_SIZE) - self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) - - num2func[opcode.FLOAT] = load_float - - def load_complex(self) -> None: - binary = self.stream.read(COMPLEX_FORMAT_SIZE) - self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) - - num2func[opcode.COMPLEX] = load_complex - - def _read_int4(self) -> int: - value: int = struct.unpack("!i", self.stream.read(4))[0] - return value - - def _read_byte_string(self) -> bytes: - length = self._read_int4() - as_bytes = self.stream.read(length) - return as_bytes - - def load_string(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.STRING] = load_string - - def load_bytes(self) -> None: - s = self._read_byte_string() - self.stack.append(s) - - num2func[opcode.BYTES] = load_bytes - - def load_newlist(self) -> None: - length = self._read_int4() - self.stack.append([None] * length) - - num2func[opcode.NEWLIST] = load_newlist - - def load_setitem(self) -> None: - if len(self.stack) < 3: - raise LoadError("not enough items for setitem") - value = self.stack.pop() - key = self.stack.pop() - self.stack[-1][key] = value # type: ignore[index] - - num2func[opcode.SETITEM] = load_setitem - - def load_newdict(self) -> None: - self.stack.append({}) - - num2func[opcode.NEWDICT] = load_newdict - - def _load_collection(self, type_: type) -> None: - length = self._read_int4() - if length: - res = type_(self.stack[-length:]) - del self.stack[-length:] - self.stack.append(res) - else: - self.stack.append(type_()) - - def load_buildtuple(self) -> None: - self._load_collection(tuple) - - num2func[opcode.BUILDTUPLE] = load_buildtuple - - def load_set(self) -> None: - self._load_collection(set) - - num2func[opcode.SET] = load_set - - def load_frozenset(self) -> None: - self._load_collection(frozenset) - - num2func[opcode.FROZENSET] = load_frozenset - - def load_stop(self) -> None: - raise _Stop - - num2func[opcode.STOP] = load_stop - - def load_channel(self) -> None: - id = self._read_int4() - assert self.channelfactory is not None - newchannel = self.channelfactory.new(id) - self.stack.append(newchannel) - - num2func[opcode.CHANNEL] = load_channel - - -def dumps(obj: object) -> bytes: - """Serialize the given obj to a bytestring. - - The obj and all contained objects must be of a builtin - Python type (so nested dicts, sets, etc. are all OK but - not user-level instances). - """ - return _Serializer().save(obj, versioned=True) # type: ignore[return-value] - - -def dump(byteio, obj: object) -> None: - """write a serialized bytestring of the given obj to the given stream.""" - _Serializer(write=byteio.write).save(obj, versioned=True) - - -def loads(bytestring: bytes) -> Any: - """Deserialize the given bytestring to an object. - - If the bytestring was dumped with an incompatible protocol - version or if the bytestring is corrupted, the - ``execnet.DataFormatError`` will be raised. - """ - return load(BytesIO(bytestring)) - - -def load(io: ReadIO) -> Any: - """Derserialize an object form the specified stream. - - Behaviour is otherwise the same as with ``loads`` - """ - return Unserializer(io).load(versioned=True) - - -def loads_internal(bytestring: bytes, channelfactory=None) -> Any: - io = BytesIO(bytestring) - return Unserializer(io, channelfactory).load() - - -def dumps_internal(obj: object) -> bytes: - return _Serializer().save(obj) # type: ignore[return-value] - - -class _Serializer: - _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} - - def __init__(self, write: Callable[[bytes], None] | None = None) -> None: - if write is None: - self._streamlist: list[bytes] = [] - write = self._streamlist.append - self._write = write - - def save(self, obj: object, versioned: bool = False) -> bytes | None: - # calling here is not re-entrant but multiple instances - # may write to the same stream because of the common platform - # atomic-write guarantee (concurrent writes each happen atomically) - if versioned: - self._write(DUMPFORMAT_VERSION) - self._save(obj) - self._write(opcode.STOP) - try: - streamlist = self._streamlist - except AttributeError: - return None - return b"".join(streamlist) - - def _save(self, obj: object) -> None: - tp = type(obj) - try: - dispatch = self._dispatch[tp] - except KeyError: - methodname = "save_" + tp.__name__ - meth: Callable[[_Serializer, object], None] | None = getattr( - self.__class__, methodname, None - ) - if meth is None: - raise DumpError(f"can't serialize {tp}") from None - dispatch = self._dispatch[tp] = meth - dispatch(self, obj) - - def save_NoneType(self, non: None) -> None: - self._write(opcode.NONE) - - def save_bool(self, boolean: bool) -> None: - if boolean: - self._write(opcode.TRUE) - else: - self._write(opcode.FALSE) - - def save_bytes(self, bytes_: bytes) -> None: - self._write(opcode.BYTES) - self._write_byte_sequence(bytes_) - - def save_str(self, s: str) -> None: - self._write(opcode.STRING) - self._write_unicode_string(s) - - def _write_unicode_string(self, s: str) -> None: - try: - as_bytes = s.encode("utf-8") - except UnicodeEncodeError as e: - raise DumpError("strings must be utf-8 encodable") from e - self._write_byte_sequence(as_bytes) - - def _write_byte_sequence(self, bytes_: bytes) -> None: - self._write_int4(len(bytes_), "string is too long") - self._write(bytes_) - - def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - # The short op packs a signed 4-byte int; anything outside that range - # (in either direction) goes through the arbitrary-precision long op. - if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: - self._write(short_op) - self._write_int4(i) - else: - self._write(long_op) - self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) - - def save_int(self, i: int) -> None: - self._save_integral(i, opcode.INT, opcode.LONGINT) - - def save_long(self, l: int) -> None: - self._save_integral(l, opcode.LONG, opcode.LONGLONG) - - def save_float(self, flt: float) -> None: - self._write(opcode.FLOAT) - self._write(struct.pack(FLOAT_FORMAT, flt)) - - def save_complex(self, cpx: complex) -> None: - self._write(opcode.COMPLEX) - self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) - - def _write_int4( - self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) - ) -> None: - if i > FOUR_BYTE_INT_MAX: - raise DumpError(error) - self._write(struct.pack("!i", i)) - - def save_list(self, L: list[object]) -> None: - self._write(opcode.NEWLIST) - self._write_int4(len(L), "list is too long") - for i, item in enumerate(L): - self._write_setitem(i, item) - - def _write_setitem(self, key: object, value: object) -> None: - self._save(key) - self._save(value) - self._write(opcode.SETITEM) - - def save_dict(self, d: dict[object, object]) -> None: - self._write(opcode.NEWDICT) - for key, value in d.items(): - self._write_setitem(key, value) - - def save_tuple(self, tup: tuple[object, ...]) -> None: - for item in tup: - self._save(item) - self._write(opcode.BUILDTUPLE) - self._write_int4(len(tup), "tuple is too long") - - def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: - for item in s: - self._save(item) - self._write(op) - self._write_int4(len(s), "set is too long") - - def save_set(self, s: set[object]) -> None: - self._write_set(s, opcode.SET) - - def save_frozenset(self, s: frozenset[object]) -> None: - self._write_set(s, opcode.FROZENSET) - - def save_Channel(self, channel: Channel) -> None: - self._write(opcode.CHANNEL) - self._write_int4(channel.id) - - def save_AsyncChannel(self, channel: Any) -> None: - # trio-native channel (execnet._trio_gateway); same wire opcode, - # duck-typed here to avoid importing the async core. - self._write(opcode.CHANNEL) - self._write_int4(channel.id) +from ._shim import forwarder + +_MOVED = { + # tracing + "DEBUG": "._trace", + "pid": "._trace", + "trace": "._trace", + "notrace": "._trace", + # errors and error texts + "sysex": "._errors", + "INTERRUPT_TEXT": "._errors", + "MAIN_THREAD_ONLY_DEADLOCK_TEXT": "._errors", + "GatewayReceivedTerminate": "._errors", + "HostNotFound": "._errors", + "geterrortext": "._errors", + "RemoteError": "._errors", + "TimeoutError": "._errors", + "DataFormatError": "._errors", + "DumpError": "._errors", + "LoadError": "._errors", + # execution model presets + "ExecModel": "._execmodel", + "EXECMODEL_PROFILES": "._execmodel", + "get_execmodel": "._execmodel", + # wire protocol + "WriteIO": "._message", + "ReadIO": "._message", + "IO": "._message", + "Message": "._message", + "gateway_info": "._message", + "FrameDecoder": "._message", + # serializer + "bchr": "._serialize", + "DUMPFORMAT_VERSION": "._serialize", + "FOUR_BYTE_INT_MAX": "._serialize", + "FOUR_BYTE_INT_MIN": "._serialize", + "FLOAT_FORMAT": "._serialize", + "FLOAT_FORMAT_SIZE": "._serialize", + "COMPLEX_FORMAT": "._serialize", + "COMPLEX_FORMAT_SIZE": "._serialize", + "opcode": "._serialize", + "Unserializer": "._serialize", + "_Serializer": "._serialize", + "_Stop": "._serialize", + "dumps": "._serialize", + "dump": "._serialize", + "loads": "._serialize", + "load": "._serialize", + "dumps_internal": "._serialize", + "loads_internal": "._serialize", + # channels + "Channel": "._channel", + "ChannelFactory": "._channel", + "ChannelFile": "._channel", + "ChannelFileWrite": "._channel", + "ChannelFileRead": "._channel", + "ENDMARKER": "._channel", + "NO_ENDMARKER_WANTED": "._channel", + # gateways + "BaseGateway": "._gateway_base", + "WorkerGateway": "._gateway_base", +} + +__getattr__ = forwarder("gateway_base", _MOVED) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 7c0810a7..06683457 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -1,419 +1,22 @@ -""" -Managing Gateway Groups and interactions with multiple channels. +"""Deprecated alias for :mod:`execnet._multi`. -(c) 2008-2014, Holger Krekel and others +``Group``, ``MultiChannel``, ``default_group``, ``makegateway`` and +``set_execmodel`` are exported from :mod:`execnet` and :mod:`execnet.sync`; +use those. """ from __future__ import annotations -import atexit -import queue -import threading -import time -import types -from collections.abc import Callable -from collections.abc import Iterable -from collections.abc import Iterator -from collections.abc import Sequence -from contextlib import suppress -from threading import Lock -from typing import TYPE_CHECKING -from typing import Any -from typing import Literal -from typing import TypeAlias -from typing import overload - -from ._boundary import wakener_names -from .gateway_base import EXECMODEL_PROFILES -from .gateway_base import Channel -from .gateway_base import ExecModel -from .gateway_base import get_execmodel -from .gateway_base import trace -from .xspec import XSpec - -if TYPE_CHECKING: - from .gateway import Gateway - - -NO_ENDMARKER_WANTED = object() - - -class Group: - """Gateway Group.""" - - defaultspec = "popen" - - def __init__( - self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" - ) -> None: - """Initialize a group and make gateways as specified. - - execmodel can be one of the supported execution models. - """ - self._gateways: list[Gateway] = [] - self._autoidcounter = 0 - self._autoidlock = Lock() - self._gateways_to_join: list[Gateway] = [] - self._trio_host: Any = None - self._async_group: Any = None - # we use the same execmodel for all of the Gateway objects - # we spawn on our side. Probably we should not allow different - # execmodels between different groups but not clear. - # Note that "other side" execmodels may differ and is typically - # specified by the spec passed to makegateway. - self.set_execmodel(execmodel) - for xspec in xspecs: - self.makegateway(xspec) - atexit.register(self._cleanup_atexit) - - def _ensure_trio_host(self) -> Any: - if self._trio_host is None: - from . import _trio_host - - self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") - self._trio_host.start() - return self._trio_host - - def _ensure_async_group(self) -> Any: - """The FacadeAsyncGroup owning the async side, running on the host.""" - if self._async_group is None: - from . import _trio_host - - host = self._ensure_trio_host() - - async def _start() -> Any: - async_group = _trio_host.FacadeAsyncGroup(self, host) - return await host._nursery.start(async_group.run) - - self._async_group = host.call(_start) - return self._async_group - - @property - def execmodel(self) -> ExecModel: - return self._execmodel - - @property - def remote_execmodel(self) -> ExecModel: - return self._remote_execmodel - - def set_execmodel( - self, execmodel: str, remote_execmodel: str | None = None - ) -> None: - """Set the execution model for local and remote site. - - execmodel can be one of the supported execution models. - It determines the execution model for any newly created gateway. - If remote_execmodel is not specified it takes on the value of execmodel. - - NOTE: Execution models can only be set before any gateway is created. - """ - if self._gateways: - raise ValueError( - "can not set execution models if gateways have been created already" - ) - if remote_execmodel is None: - remote_execmodel = execmodel - self._execmodel = get_execmodel(execmodel) - self._remote_execmodel = get_execmodel(remote_execmodel) - - def __repr__(self) -> str: - idgateways = [gw.id for gw in self] - return "" % idgateways - - def __getitem__(self, key: int | str | Gateway) -> Gateway: - if isinstance(key, int): - return self._gateways[key] - for gw in self._gateways: - if gw == key or gw.id == key: - return gw - raise KeyError(key) - - def __contains__(self, key: str) -> bool: - try: - self[key] - return True - except KeyError: - return False - - def __len__(self) -> int: - return len(self._gateways) - - def __iter__(self) -> Iterator[Gateway]: - return iter(list(self._gateways)) - - def makegateway(self, spec: XSpec | str | None = None) -> Gateway: - """Create and configure a gateway to a Python interpreter. - - The ``spec`` string encodes the target gateway type - and configuration information. The general format is:: - - key1=value1//key2=value2//... - - If you leave out the ``=value`` part a True value is assumed. - Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. - Valid configuration:: - - id= specifies the gateway id - python= specifies which python interpreter to execute - execmodel=name worker profile: where exec'd code runs relative - to the worker's protocol loop. 'thread' (pool - threads) or 'main_thread_only' (serialized on - the worker main thread, GUI/signal-safe). - wait=backend wakener for blocking waits ('thread' default) - chdir= specifies to which directory to change - nice= specifies process priority of new process - env:NAME=value specifies a remote environment variable setting. - - If no spec is given, self.defaultspec is used. - """ - if not spec: - spec = self.defaultspec - if not isinstance(spec, XSpec): - spec = XSpec(spec) - self.allocate_id(spec) - if spec.execmodel is None: - spec.execmodel = self.remote_execmodel.backend - elif spec.execmodel not in EXECMODEL_PROFILES: - raise ValueError( - f"unknown execmodel {spec.execmodel!r}" - f" (known profiles: {list(EXECMODEL_PROFILES)})" - ) - if spec.wait is not None and spec.wait not in wakener_names(): - raise ValueError( - f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" - ) - from . import _trio_host - - if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): - raise ValueError(f"no gateway type found for {spec._spec!r}") - gw = _trio_host.makegateway_trio(self, spec) - gw.spec = spec - self._register(gw) - # chdir/nice/env travel in the worker config and are applied at - # worker startup -- no remote_exec, so no exec slot is claimed. - return gw - - def allocate_id(self, spec: XSpec) -> None: - """(re-entrant) allocate id for the given xspec object.""" - if spec.id is None: - with self._autoidlock: - id = "gw" + str(self._autoidcounter) - self._autoidcounter += 1 - if id in self: - raise ValueError(f"already have gateway with id {id!r}") - spec.id = id - - def _register(self, gateway: Gateway) -> None: - assert not hasattr(gateway, "_group") - assert gateway.id - assert gateway.id not in self - self._gateways.append(gateway) - gateway._group = self - - def _unregister(self, gateway: Gateway) -> None: - self._gateways.remove(gateway) - self._gateways_to_join.append(gateway) - - def _cleanup_atexit(self) -> None: - trace(f"=== atexit cleanup {self!r} ===") - self.terminate(timeout=1.0) - if self._async_group is not None: - with suppress(Exception): - self._trio_host.call_sync(self._async_group.shutdown.set) - self._async_group = None - if self._trio_host is not None: - self._trio_host.stop(timeout=1.0) - self._trio_host = None - - def terminate(self, timeout: float | None = None) -> None: - """Trigger exit of member gateways and wait for termination - of member gateways and associated subprocesses. - - After waiting timeout seconds try to to kill local sub processes of - popen- and ssh-gateways. - - Timeout defaults to None meaning open-ended waiting and no kill - attempts. - """ - while self or self._gateways_to_join: - vias: set[str] = set() - for gw in self: - if gw.spec.via: - vias.add(gw.spec.via) - for gw in self: - if gw.id not in vias: - gw.exit() - if self._async_group is not None: - # Tunneled (via) gateways terminate before their masters, - # each with a GATEWAY_TERMINATE + timeout grace, then kill; - # bounded at roughly twice the timeout (issues #43 / #221). - try: - self._host_terminate(timeout) - except Exception as exc: - trace("group terminate error:", exc) - for gw in self._gateways_to_join: - gw.join() - self._gateways_to_join[:] = [] - - def _host_terminate(self, timeout: float | None) -> None: - """Terminate the async group, parking correctly for wait backends. - - A member gateway created with a non-thread ``wait=`` implies the - caller may be a greenlet: wait on a OneShot instead of blocking - the OS thread (which would stall the hub for the whole grace). - """ - backends = {gw._wait_backend for gw in self._gateways_to_join} | { - gw._wait_backend for gw in self - } - backends.discard("thread") - if backends: - from ._boundary import make_wakener - - self._trio_host.call_pending( - self._async_group.terminate, - timeout, - wakener=make_wakener(backends.pop()), - ).wait() - else: - self._trio_host.call(self._async_group.terminate, timeout) - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs, - ) -> MultiChannel: - """remote_exec source on all member gateways and return - a MultiChannel connecting to all sub processes.""" - channels = [] - for gw in self: - channels.append(gw.remote_exec(source, **kwargs)) - return MultiChannel(channels) - - -class MultiChannel: - def __init__(self, channels: Sequence[Channel]) -> None: - self._channels = channels - - def __len__(self) -> int: - return len(self._channels) - - def __iter__(self) -> Iterator[Channel]: - return iter(self._channels) - - def __getitem__(self, key: int) -> Channel: - return self._channels[key] - - def __contains__(self, chan: Channel) -> bool: - return chan in self._channels - - def send_each(self, item: object) -> None: - for ch in self._channels: - ch.send(item) - - @overload - def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: - pass - - @overload - def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: - pass - - def receive_each( - self, withchannel: bool = False - ) -> list[tuple[Channel, Any]] | list[Any]: - assert not hasattr(self, "_queue") - l: list[object] = [] - for ch in self._channels: - obj = ch.receive() - if withchannel: - l.append((ch, obj)) - else: - l.append(obj) - return l - - def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): - try: - return self._queue # type: ignore[has-type] - except AttributeError: - self._queue: queue.Queue[tuple[Channel, Any]] | None = None - for ch in self._channels: - if self._queue is None: - self._queue = queue.Queue() - - def putreceived(obj, channel: Channel = ch) -> None: - self._queue.put((channel, obj)) # type: ignore[union-attr] - - if endmarker is NO_ENDMARKER_WANTED: - ch.setcallback(putreceived) - else: - ch.setcallback(putreceived, endmarker=endmarker) - return self._queue - - def waitclose(self) -> None: - first = None - for ch in self._channels: - try: - ch.waitclose() - except ch.RemoteError as exc: - if first is None: - first = exc - if first: - raise first - - -TermKillFunc: TypeAlias = Callable[[], object] -TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] - - -def safe_terminate( - execmodel: ExecModel, - timeout: float | None, - list_of_paired_functions: Sequence[TermKillPair], -) -> None: - """Run terminate/kill pairs in parallel with a hard wait bound. - - Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - The final wait is also bounded so a stuck kill cannot hang the caller - forever (see issues #43 / #221). ``execmodel`` is accepted for - backward compatibility and unused (daemon threads do the waiting). - """ - errors: list[BaseException] = [] - - def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - term_done = threading.Event() - term_errors: list[BaseException] = [] - - def run_term() -> None: - try: - termfunc() - except BaseException as exc: - term_errors.append(exc) - finally: - term_done.set() - - threading.Thread(target=run_term, daemon=True).start() - if not term_done.wait(timeout): - killfunc() - return - if term_errors: - errors.append(term_errors[0]) - - threads = [ - threading.Thread(target=termkill, args=pair, daemon=True) - for pair in list_of_paired_functions - ] - for thread in threads: - thread.start() - # Allow term timeout plus a kill attempt; never block indefinitely. - wait_timeout = None if timeout is None else timeout * 2 - deadline = None if wait_timeout is None else time.monotonic() + wait_timeout - for thread in threads: - remaining = None if deadline is None else max(0, deadline - time.monotonic()) - thread.join(remaining) - if errors: - raise errors[0] +from ._shim import forwarder +_MOVED = { + "Group": "._multi", + "MultiChannel": "._multi", + "default_group": "._multi", + "makegateway": "._multi", + "set_execmodel": "._multi", + "safe_terminate": "._multi", + "NO_ENDMARKER_WANTED": "._multi", +} -default_group = Group() -makegateway = default_group.makegateway -set_execmodel = default_group.set_execmodel +__getattr__ = forwarder("multi", _MOVED) diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index 876e38fc..786ed42b 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -1,248 +1,12 @@ -""" -1:N rsync implementation on top of execnet. +"""Deprecated alias for :mod:`execnet._rsync`. -(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +``RSync`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import os -import stat -from collections.abc import Callable -from hashlib import md5 -from queue import Queue -from typing import Literal - -import execnet.rsync_remote -from execnet.gateway import Gateway -from execnet.gateway_base import BaseGateway -from execnet.gateway_base import Channel - - -class RSync: - """This class allows to send a directory structure (recursively) - to one or multiple remote filesystems. - - There is limited support for symlinks, which means that symlinks - pointing to the sourcetree will be send "as is" while external - symlinks will be just copied (regardless of existence of such - a path on remote side). - """ - - def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: - self._sourcedir = str(sourcedir) - self._verbose = verbose - assert callback is None or callable(callback) - self._callback = callback - self._channels: dict[Channel, Callable[[], None] | None] = {} - self._receivequeue: Queue[ - tuple[ - Channel, - ( - None - | tuple[Literal["send"], tuple[list[str], bytes]] - | tuple[Literal["list_done"], None] - | tuple[Literal["ack"], str] - | tuple[Literal["links"], None] - | tuple[Literal["done"], None] - ), - ] - ] = Queue() - self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] - - def filter(self, path: str) -> bool: - return True - - def _end_of_channel(self, channel: Channel) -> None: - if channel in self._channels: - # too early! we must have got an error - channel.waitclose() - # or else we raise one - raise OSError(f"connection unexpectedly closed: {channel.gateway} ") - - def _process_link(self, channel: Channel) -> None: - for link in self._links: - channel.send(link) - # completion marker, this host is done - channel.send(42) - - def _done(self, channel: Channel) -> None: - """Call all callbacks.""" - finishedcallback = self._channels.pop(channel) - if finishedcallback: - finishedcallback() - channel.waitclose() - - def _list_done(self, channel: Channel) -> None: - # sum up all to send - if self._callback: - s = sum([self._paths[i] for i in self._to_send[channel]]) - self._callback("list", s, channel) - - def _send_item( - self, - channel: Channel, - modified_rel_path_components: list[str], - checksum: bytes, - ) -> None: - """Send one item.""" - modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) - try: - with open(modified_path, "rb") as fp: - data = fp.read() - except OSError: - data = None - - # provide info to progress callback function - modified_rel_path = "/".join(modified_rel_path_components) - if data is not None: - self._paths[modified_rel_path] = len(data) - else: - self._paths[modified_rel_path] = 0 - if channel not in self._to_send: - self._to_send[channel] = [] - self._to_send[channel].append(modified_rel_path) - # print "sending", modified_rel_path, data and len(data) or 0, checksum - - if data is not None: - if checksum is not None and checksum == md5(data).digest(): - data = None # not really modified - else: - self._report_send_file(channel.gateway, modified_rel_path) - channel.send(data) - - def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: - if self._verbose: - print(f"{gateway} <= {modified_rel_path}") - - def send(self, raises: bool = True) -> None: - """Sends a sourcedir to all added targets. - - raises indicates whether to raise an error or return in case of lack of - targets. - """ - if not self._channels: - if raises: - raise OSError( - "no targets available, maybe you are trying call send() twice?" - ) - return - # normalize a trailing '/' away - self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) - # send directory structure and file timestamps/sizes - self._send_directory_structure(self._sourcedir) - - # paths and to_send are only used for doing - # progress-related callbacks - self._paths: dict[str, int] = {} - self._to_send: dict[Channel, list[str]] = {} - - # send modified file to clients - while self._channels: - channel, req = self._receivequeue.get() - if req is None: - self._end_of_channel(channel) - else: - if req[0] == "links": - self._process_link(channel) - elif req[0] == "done": - self._done(channel) - elif req[0] == "ack": - if self._callback: - self._callback("ack", self._paths[req[1]], channel) - elif req[0] == "list_done": - self._list_done(channel) - elif req[0] == "send": - self._send_item(channel, req[1][0], req[1][1]) - else: - assert "Unknown command %s" % req[0] # type: ignore[unreachable] - - def add_target( - self, - gateway: Gateway, - destdir: str | os.PathLike[str], - finishedcallback: Callable[[], None] | None = None, - **options, - ) -> None: - """Add a remote target specified via a gateway and a remote destination - directory.""" - for name in options: - assert name in ("delete",) - - def itemcallback(req) -> None: - self._receivequeue.put((channel, req)) - - channel = gateway.remote_exec(execnet.rsync_remote) - channel.setcallback(itemcallback, endmarker=None) - channel.send((str(destdir), options)) - self._channels[channel] = finishedcallback - - def _broadcast(self, msg: object) -> None: - for channel in self._channels: - channel.send(msg) - - def _send_link( - self, - linktype: Literal["linkbase", "link"], - basename: str, - linkpoint: str, - ) -> None: - self._links.append((linktype, basename, linkpoint)) - - def _send_directory(self, path: str) -> None: - # dir: send a list of entries - names = [] - subpaths = [] - for name in os.listdir(path): - p = os.path.join(path, name) - if self.filter(p): - names.append(name) - subpaths.append(p) - mode = os.lstat(path).st_mode - self._broadcast([mode, *names]) - for p in subpaths: - self._send_directory_structure(p) +from ._shim import forwarder - def _send_link_structure(self, path: str) -> None: - sourcedir = self._sourcedir - basename = path[len(self._sourcedir) + 1 :] - linkpoint = os.readlink(path) - # On Windows, readlink returns an extended path (//?/) for - # absolute links, but relpath doesn't like mixing extended - # and non-extended paths. So fix it up ourselves. - if ( - os.path.__name__ == "ntpath" - and linkpoint.startswith("\\\\?\\") - and not self._sourcedir.startswith("\\\\?\\") - ): - sourcedir = "\\\\?\\" + self._sourcedir - try: - relpath = os.path.relpath(linkpoint, sourcedir) - except ValueError: - relpath = None - if ( - relpath is not None - and relpath not in (os.curdir, os.pardir) - and not relpath.startswith(os.pardir + os.sep) - ): - self._send_link("linkbase", basename, relpath) - else: - # relative or absolute link, just send it - self._send_link("link", basename, linkpoint) - self._broadcast(None) +_MOVED = {"RSync": "._rsync"} - def _send_directory_structure(self, path: str) -> None: - try: - st = os.lstat(path) - except OSError: - self._broadcast((None, 0, 0)) - return - if stat.S_ISREG(st.st_mode): - # regular file: send a mode/timestamp/size pair - self._broadcast((st.st_mode, st.st_mtime, st.st_size)) - elif stat.S_ISDIR(st.st_mode): - self._send_directory(path) - elif stat.S_ISLNK(st.st_mode): - self._send_link_structure(path) - else: - raise ValueError(f"cannot sync {path!r}") +__getattr__ = forwarder("rsync", _MOVED) diff --git a/src/execnet/rsync_remote.py b/src/execnet/rsync_remote.py index a8467b76..4770918a 100644 --- a/src/execnet/rsync_remote.py +++ b/src/execnet/rsync_remote.py @@ -1,126 +1,13 @@ -""" -(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +"""Deprecated alias for :mod:`execnet._rsync_remote`. + +The worker half of the rsync protocol; :class:`execnet.RSync` ships it to the +remote side itself, so there is no reason to reference this module. """ from __future__ import annotations -from contextlib import suppress -from typing import TYPE_CHECKING -from typing import Literal -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - - -def serve_rsync(channel: Channel) -> None: - import os - import shutil - import stat - from hashlib import md5 - - destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) - modifiedfiles = [] - - def remove(path: str) -> None: - assert path.startswith(destdir) - try: - os.unlink(path) - except OSError: - # assume it's a dir - shutil.rmtree(path, True) - - def receive_directory_structure(path: str, relcomponents: list[str]) -> None: - try: - st = os.lstat(path) - except OSError: - st = None - msg = channel.receive() - if isinstance(msg, list): - if st and not stat.S_ISDIR(st.st_mode): - os.unlink(path) - st = None - if not st: - os.makedirs(path) - mode = msg.pop(0) - if mode: - # Ensure directories are writable, otherwise a - # permission denied error (EACCES) would be raised - # when attempting to receive read-only directory - # structures. - os.chmod(path, mode | 0o700) - entrynames = {} - for entryname in msg: - destpath = os.path.join(path, entryname) - receive_directory_structure(destpath, [*relcomponents, entryname]) - entrynames[entryname] = True - if options.get("delete"): - for othername in os.listdir(path): - if othername not in entrynames: - otherpath = os.path.join(path, othername) - remove(otherpath) - elif msg is not None: - assert isinstance(msg, tuple) - checksum = None - if st: - if stat.S_ISREG(st.st_mode): - msg_mode, msg_mtime, msg_size = msg - if msg_size != st.st_size: - pass - elif msg_mtime != st.st_mtime: - with open(path, "rb") as fp: - checksum = md5(fp.read()).digest() - elif msg_mode and msg_mode != st.st_mode: - os.chmod(path, msg_mode | 0o700) - return - else: - return # already fine - else: - remove(path) - channel.send(("send", (relcomponents, checksum))) - modifiedfiles.append((path, msg)) - - receive_directory_structure(destdir, []) - - STRICT_CHECK = False # seems most useful this way for py.test - channel.send(("list_done", None)) - - for path, (mode, time, size) in modifiedfiles: - data = cast(bytes, channel.receive()) - channel.send(("ack", path[len(destdir) + 1 :])) - if data is not None: - if STRICT_CHECK and len(data) != size: - raise OSError(f"file modified during rsync: {path!r}") - with open(path, "wb") as fp: - fp.write(data) - try: - if mode: - os.chmod(path, mode) - os.utime(path, (time, time)) - except OSError: - pass - del data - channel.send(("links", None)) - - msg = channel.receive() - while msg != 42: - # we get symlink - _type, relpath, linkpoint = cast( - "tuple[Literal['linkbase', 'link'], str, str]", msg - ) - path = os.path.join(destdir, relpath) - with suppress(OSError): - remove(path) - - if _type == "linkbase": - src = os.path.join(destdir, linkpoint) - else: - assert _type == "link", _type - src = linkpoint - os.symlink(src, path) - msg = channel.receive() - channel.send(("done", None)) +from ._shim import forwarder +_MOVED = {"serve_rsync": "._rsync_remote"} -if __name__ == "__channelexec__": - serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 +__getattr__ = forwarder("rsync_remote", _MOVED) diff --git a/src/execnet/sync.py b/src/execnet/sync.py index e5e026c7..37259bb7 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -6,21 +6,21 @@ into this module. """ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from ._channel import Channel +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._gateway import Gateway +from ._multi import Group +from ._multi import MultiChannel +from ._multi import default_group +from ._multi import makegateway +from ._multi import set_execmodel +from ._rsync import RSync +from ._xspec import XSpec __all__ = [ "Channel", diff --git a/src/execnet/trio.py b/src/execnet/trio.py index 4b712c0a..9704b6ef 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -16,40 +16,32 @@ async def main(): The error types are shared with the blocking API in :mod:`execnet.sync`. Items you send must already be simple builtin data (plus channels); the -standalone serializer is intentionally not part of the public API -- see -``DumpError``. +standalone serializer is intentionally not part of the public API -- +``execnet.can_send`` checks a value before you send it; see ``DumpError``. """ +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup -from ._trio_gateway import ByteStream -from ._trio_gateway import RawChannel -from ._trio_gateway import RawChannelStream from ._trio_gateway import open_popen_gateway -from ._trio_gateway import serve_gateway -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .xspec import XSpec +from ._xspec import XSpec __all__ = [ "AsyncChannel", "AsyncGateway", "AsyncGroup", - "ByteStream", "DataFormatError", "DumpError", "HostNotFound", "LoadError", - "RawChannel", - "RawChannelStream", "RemoteError", "TimeoutError", "XSpec", "open_popen_gateway", - "serve_gateway", ] diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index afc58f4c..9f5a2634 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -1,74 +1,13 @@ -""" -(c) 2008-2013, holger krekel +"""Deprecated alias for :mod:`execnet._xspec`. + +``XSpec`` is exported from :mod:`execnet`, :mod:`execnet.sync`, +:mod:`execnet.trio` and :mod:`execnet.aio`; use those. """ from __future__ import annotations +from ._shim import forwarder -class XSpec: - """Execution Specification: key1=value1//key2=value2 ... - - * Keys need to be unique within the specification scope - * Neither key nor value are allowed to contain "//" - * Keys are not allowed to contain "=" - * Keys are not allowed to start with underscore - * If no "=value" is given, assume a boolean True value - """ - - # XXX allow customization, for only allow specific key names - chdir: str | None = None - dont_write_bytecode: bool | None = None - execmodel: str | None = None - id: str | None = None - installvia: str | None = None - nice: str | None = None - popen: bool | None = None - python: str | None = None - socket: str | None = None - ssh: str | None = None - ssh_config: str | None = None - vagrant_ssh: str | None = None - via: str | None = None - wait: str | None = None - - def __init__(self, string: str) -> None: - self._spec = string - self.env = {} - for keyvalue in string.split("//"): - i = keyvalue.find("=") - value: str | bool - if i == -1: - key, value = keyvalue, True - else: - key, value = keyvalue[:i], keyvalue[i + 1 :] - if key[0] == "_": - raise AttributeError("%r not a valid XSpec key" % key) - if key in self.__dict__: - raise ValueError(f"duplicate key: {key!r} in {string!r}") - if key.startswith("env:"): - self.env[key[4:]] = value - else: - setattr(self, key, value) - - def __getattr__(self, name: str) -> None | bool | str: - if name[0] == "_": - raise AttributeError(name) - return None - - def __repr__(self) -> str: - return f"" - - def __str__(self) -> str: - return self._spec - - def __hash__(self) -> int: - return hash(self._spec) - - def __eq__(self, other: object) -> bool: - return self._spec == getattr(other, "_spec", None) - - def __ne__(self, other: object) -> bool: - return self._spec != getattr(other, "_spec", None) +_MOVED = {"XSpec": "._xspec"} - def _samefilesystem(self) -> bool: - return self.popen is not None and self.chdir is None +__getattr__ = forwarder("xspec", _MOVED) diff --git a/testing/conftest.py b/testing/conftest.py index 04dbe631..de1e2866 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -11,9 +11,9 @@ import pytest import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_base import get_execmodel +from execnet import Gateway +from execnet._execmodel import ExecModel +from execnet._execmodel import get_execmodel collect_ignore = ["build", "doc/_build"] diff --git a/testing/test_basics.py b/testing/test_basics.py index d5606610..a81ed1f6 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -16,11 +16,14 @@ import execnet from execnet import _boundary -from execnet import gateway -from execnet import gateway_base -from execnet.gateway_base import ChannelFactory -from execnet.gateway_base import ExecModel -from execnet.gateway_base import Message +from execnet import _errors +from execnet import _exec_source +from execnet import _gateway_base +from execnet import _message +from execnet import _serialize +from execnet._channel import ChannelFactory +from execnet._execmodel import ExecModel +from execnet._message import Message skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -28,45 +31,49 @@ ) -# The standalone serializer is an internal detail (execnet.gateway_base), +# The standalone serializer is an internal detail (execnet._serialize), # not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = gateway_base.dumps(val) - val2 = gateway_base.loads(dumped) + dumped = _serialize.dumps(val) + val2 = _serialize.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(gateway_base.dumps(val)) + p.write_bytes(_serialize.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = gateway_base.load(m) + val2 = _serialize.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - gateway_base.dump(f, val) + _serialize.dump(f, val) read = BytesIO(f.getvalue()) - val2 = gateway_base.load(read) + val2 = _serialize.load(read) assert val == val2 def test_serializer_not_public() -> None: - # dumps/loads/dump/load are internal to gateway_base only. - for name in ("dumps", "loads", "dump", "load"): + # dumps/loads/dump/load stay internal to execnet._serialize. ``dumps`` + # is still *reachable* as a temporary pytest-xdist shim, but it is not + # part of the surface -- see test_namespaces.py. + for name in ("loads", "dump", "load"): assert not hasattr(execnet, name), name + for name in ("dumps", "loads", "dump", "load"): + assert name not in execnet.__all__, name def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: - bchr = gateway_base.bchr - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = gateway_base.dumps(42) - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: gateway_base.loads(dumped)) + bchr = _serialize.bchr + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(1)) + dumped = _serialize.dumps(42) + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(2)) + pytest.raises(execnet.DataFormatError, lambda: _serialize.loads(dumped)) def test_errors_on_execnet() -> None: @@ -75,26 +82,22 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def standalone_gateway_base_source() -> str: - """gateway_base's source as a self-contained script. +def standalone_protocol_source() -> str: + """The wire-protocol core as a self-contained script. - The module imports the trio-free boundary kit relatively; for the - run-on-any-python checks the kit source is inlined in place of those - imports (minus its own future import, which must stay file-leading). + ``_errors``, ``_message`` and ``_serialize`` are the layers a peer needs + to frame and serialize; between them the only runtime execnet imports are + of each other, so concatenating their sources -- dropping those imports + and keeping a single file-leading future import -- yields a script that + runs on any interpreter, which is what these checks verify. The + ``if TYPE_CHECKING:`` imports are left in place; they never execute. """ - boundary_source = inspect.getsource(_boundary).replace( - "from __future__ import annotations\n", "" - ) - lines = [] - inlined = False - for line in inspect.getsource(gateway_base).splitlines(keepends=True): - if line.startswith("from ._boundary import"): - if not inlined: - inlined = True - lines.append(boundary_source) - else: + lines = ["from __future__ import annotations\n"] + for module in (_errors, _message, _serialize): + for line in inspect.getsource(module).splitlines(keepends=True): + if line.startswith(("from __future__ import annotations", "from ._")): + continue lines.append(line) - assert inlined return "".join(lines) @@ -156,14 +159,14 @@ def checker(anypython: str, tmp_path: Path) -> Checker: def test_io_message(checker: Checker) -> None: - out = checker.run_check(standalone_gateway_base_source() + IO_MESSAGE_EXTRA_SOURCE) + out = checker.run_check(standalone_protocol_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - standalone_gateway_base_source() + standalone_protocol_source() + """ class Arg(Exception): pass @@ -230,7 +233,7 @@ def close(self, errortext: str | None = None) -> None: def test_exectask(execmodel: ExecModel) -> None: io = BytesIO() io.execmodel = execmodel # type: ignore[attr-defined] - gw = gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] + gw = _gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] ch = PseudoChannel() gw.executetask((ch, ("raise ValueError()", None, {}))) # type: ignore[arg-type] assert "ValueError" in str(ch._closed[0]) @@ -261,7 +264,7 @@ def _messages(self) -> list[Message]: ] def test_single_feed_yields_all(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = b"".join(m.pack() for m in self._messages()) got = list(decoder.feed(blob)) assert [(m.msgcode, m.channelid, m.data) for m in got] == [ @@ -271,7 +274,7 @@ def test_single_feed_yields_all(self) -> None: @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) def test_adversarial_chunk_splits(self, chunksize: int) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = b"".join(m.pack() for m in self._messages()) got: list[Message] = [] for start in range(0, len(blob), chunksize): @@ -282,14 +285,14 @@ def test_adversarial_chunk_splits(self, chunksize: int) -> None: decoder.close() def test_close_mid_frame_raises(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() assert list(decoder.feed(blob[:-2])) == [] with pytest.raises(EOFError, match="mid-frame"): decoder.close() def test_feed_buffers_even_when_not_iterated(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() decoder.feed(blob[:4]) # result deliberately not iterated (msg,) = decoder.feed(blob[4:]) @@ -317,7 +320,7 @@ async def sender() -> None: await theirs.send_eof() async def receiver() -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() while True: data = await ours.receive_some(4096) if not data: @@ -382,13 +385,13 @@ def test_channel_makefile_incompatmode(self, fac) -> None: class TestSourceOfFunction: def test_lambda_unsupported(self) -> None: - pytest.raises(ValueError, gateway._source_of_function, lambda: 1) + pytest.raises(ValueError, _exec_source._source_of_function, lambda: 1) def test_wrong_prototype_fails(self) -> None: def prototype(wrong) -> None: pass - pytest.raises(ValueError, gateway._source_of_function, prototype) + pytest.raises(ValueError, _exec_source._source_of_function, prototype) def test_function_without_known_source_fails(self) -> None: # this one won't be able to find the source @@ -396,7 +399,7 @@ def test_function_without_known_source_fails(self) -> None: exec("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) with pytest.raises(ValueError): - gateway._source_of_function(mess["fail"]) + _exec_source._source_of_function(mess["fail"]) def test_function_with_closure_fails(self) -> None: mess: dict[str, Any] = {} @@ -405,13 +408,13 @@ def closure(channel: object) -> None: print(mess) with pytest.raises(ValueError): - gateway._source_of_function(closure) + _exec_source._source_of_function(closure) def test_source_of_nested_function(self) -> None: def working(channel: object) -> None: pass - send_source = gateway._source_of_function(working).lstrip("\r\n") + send_source = _exec_source._source_of_function(working).lstrip("\r\n") expected = "def working(channel: object) -> None:\n pass\n" assert send_source == expected @@ -420,7 +423,7 @@ class TestGlobalFinder: def check(self, func) -> list[str]: src = textwrap.dedent(inspect.getsource(func)) code = func.__code__ - return gateway._find_non_builtin_globals(src, code) + return _exec_source._find_non_builtin_globals(src, code) def test_local(self) -> None: def f(a, b, c): @@ -447,7 +450,7 @@ def test_function_with_global_fails(self) -> None: def func(channel) -> None: sys - pytest.raises(ValueError, gateway._source_of_function, func) + pytest.raises(ValueError, _exec_source._source_of_function, func) def test_method_call(self) -> None: # method names are reason @@ -460,7 +463,7 @@ def f(channel): @skip_win_pypy def test_remote_exec_function_with_kwargs( - anypython: str, makegateway: Callable[[str], gateway.Gateway] + anypython: str, makegateway: Callable[[str], execnet.Gateway] ) -> None: def func(channel, data) -> None: channel.send(data) @@ -473,17 +476,17 @@ def func(channel, data) -> None: assert result == 1 -def test_remote_exc__no_kwargs(makegateway: Callable[[], gateway.Gateway]) -> None: +def test_remote_exc__no_kwargs(makegateway: Callable[[], execnet.Gateway]) -> None: gw = makegateway() with pytest.raises(TypeError): - gw.remote_exec(gateway_base, kwarg=1) + gw.remote_exec(_message, kwarg=1) with pytest.raises(TypeError): gw.remote_exec("pass", kwarg=1) @skip_win_pypy def test_remote_exec_inspect_stack( - makegateway: Callable[[], gateway.Gateway], + makegateway: Callable[[], execnet.Gateway], ) -> None: gw = makegateway() ch = gw.remote_exec( diff --git a/testing/test_channel.py b/testing/test_channel.py index f1b0b454..8bb292a0 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -9,8 +9,8 @@ import pytest -from execnet.gateway import Gateway -from execnet.gateway_base import Channel +from execnet import Gateway +from execnet._channel import Channel needs_early_gc = pytest.mark.skipif("not hasattr(sys, 'getrefcount')") needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py index 12f69d47..f5637efb 100644 --- a/testing/test_channel_stress.py +++ b/testing/test_channel_stress.py @@ -24,7 +24,7 @@ from hypothesis import given # noqa: E402 from hypothesis import strategies as st # noqa: E402 -from execnet.gateway import Gateway # noqa: E402 +from execnet import Gateway # noqa: E402 # High --stress levels replay one test many times; lift the per-test timeout # well above the default so that only a real hang (bounded by TESTTIMEOUT on diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 837c0c6d..ed06be4e 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -1,8 +1,8 @@ -from execnet import gateway_base +from execnet import _serialize def test_opcodes() -> None: - data = vars(gateway_base.opcode) + data = vars(_serialize.opcode) computed = {k: v for k, v in data.items() if "__" not in k} assert computed == { "BUILDTUPLE": b"@", diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 84585ba9..4e568d7e 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -14,7 +14,7 @@ import execnet import execnet.trio -from execnet.gateway import Gateway +from execnet import Gateway TESTTIMEOUT = 10.0 diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 6b6d5cf5..ef8e8aa9 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -16,8 +16,9 @@ import pytest import execnet -from execnet import gateway_base -from execnet.gateway import Gateway +from execnet import Gateway +from execnet import _execmodel +from execnet import _trace TESTTIMEOUT = 10.0 # seconds needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") @@ -169,7 +170,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -178,7 +179,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest.run_me) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -346,9 +347,9 @@ def test_chdir_separation( assert x.lower() == str(tmp_path).lower() def test_remoteerror_readable_traceback(self, gw: Gateway) -> None: - with pytest.raises(gateway_base.RemoteError) as e: + with pytest.raises(execnet.RemoteError) as e: gw.remote_exec("x y").waitclose() - assert "gateway_base" in e.value.formatted + assert "_gateway_base" in e.value.formatted def test_many_popen(self, makegateway: Callable[[str], Gateway]) -> None: num = 4 @@ -523,9 +524,7 @@ def test_popen_filetracing( monkeypatch.setenv("EXECNET_DEBUG", "1") gw = makegateway("popen") # hack out the debuffilename - fn = gw.remote_exec( - "import execnet;channel.send(execnet.gateway_base.fn)" - ).receive() + fn = gw.remote_exec("import execnet;channel.send(execnet._trace.fn)").receive() assert isinstance(fn, str) workerfile = pathlib.Path(fn) assert workerfile.exists() @@ -555,7 +554,7 @@ def test_popen_stderr_tracing( gw.exit() def test_no_tracing_by_default(self): - assert gateway_base.trace == gateway_base.notrace, ( + assert _trace.trace == _trace.notrace, ( "trace does not to default to empty tracing" ) @@ -584,7 +583,7 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: def test_assert_main_thread_only( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] + execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") @@ -623,7 +622,7 @@ def test_assert_main_thread_only( def test_main_thread_only_concurrent_remote_exec_deadlock( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] + execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") @@ -651,7 +650,7 @@ def test_main_thread_only_concurrent_remote_exec_deadlock( expected_results = ( True, - execnet.gateway_base.MAIN_THREAD_ONLY_DEADLOCK_TEXT, + execnet._errors.MAIN_THREAD_ONLY_DEADLOCK_TEXT, ) for expected, ch in zip(expected_results, channels, strict=True): try: diff --git a/testing/test_gevent.py b/testing/test_gevent.py index 91f74ef4..cbec77a9 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -155,9 +155,9 @@ def test_status_reports_gevent(self, worker_gw) -> None: def test_provisioning_adds_gevent_requirement() -> None: + from execnet import XSpec from execnet._provision import _extra_with_tokens from execnet._provision import worker_cli_arg - from execnet.xspec import XSpec spec = XSpec("popen//id=g1//execmodel=gevent") config = worker_cli_arg(spec) diff --git a/testing/test_multi.py b/testing/test_multi.py index 2f1b634e..156879b3 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -12,12 +12,12 @@ import pytest import execnet +from execnet import Gateway +from execnet import Group from execnet import XSpec -from execnet.gateway import Gateway -from execnet.gateway_base import Channel -from execnet.gateway_base import ExecModel -from execnet.multi import Group -from execnet.multi import safe_terminate +from execnet._channel import Channel +from execnet._execmodel import ExecModel +from execnet._multi import safe_terminate class TestMultiChannelAndGateway: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 0317bc2b..822f8d50 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -1,20 +1,33 @@ -"""The three public namespaces: execnet.sync / execnet.trio / execnet.portal. +"""The public namespaces, and the deprecation shims for the private modules. -Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the -trio namespace exposes the async-native core; the portal namespace the -cross-thread primitives. +Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the trio +and aio namespaces expose the async-native core and its asyncio bridge; the +portal namespace the cross-thread primitives. Everything else in the package +is private -- the pre-Trio module names survive only as warning shims. """ from __future__ import annotations +import importlib +import pkgutil import subprocess import sys +import warnings + +import pytest import execnet +import execnet.aio import execnet.portal import execnet.sync import execnet.trio +#: the only modules that may be reachable without a leading underscore +PUBLIC_NAMESPACES = ("aio", "portal", "sync", "trio") + +#: pre-Trio module names kept as deprecated forwarding shims +SHIMS = ("gateway", "gateway_base", "multi", "rsync", "rsync_remote", "xspec") + def test_top_level_names_alias_execnet_sync() -> None: for name in execnet.sync.__all__: @@ -22,7 +35,30 @@ def test_top_level_names_alias_execnet_sync() -> None: def test_top_level_all_matches_sync_surface() -> None: - assert set(execnet.__all__) == set(execnet.sync.__all__) | {"__version__"} + # the top level is the sync surface plus the two package-level names: + # the version, and can_send (a wire-format fact, not a gateway API) + assert set(execnet.__all__) == set(execnet.sync.__all__) | { + "__version__", + "can_send", + } + + +@pytest.mark.parametrize( + "namespace", + [execnet, *[importlib.import_module(f"execnet.{n}") for n in PUBLIC_NAMESPACES]], +) +def test_namespace_all_resolves(namespace: object) -> None: + missing = [n for n in namespace.__all__ if not hasattr(namespace, n)] # type: ignore[attr-defined] + assert not missing + + +def test_no_unexpected_public_modules() -> None: + found = { + info.name + for info in pkgutil.iter_modules(execnet.__path__) + if not info.name.startswith("_") + } + assert found == set(PUBLIC_NAMESPACES) | set(SHIMS) def test_trio_namespace_exposes_async_core() -> None: @@ -38,6 +74,35 @@ def test_trio_namespace_exposes_async_core() -> None: assert not hasattr(execnet.trio, "dumps") +def test_trio_namespace_hides_raw_plumbing() -> None: + # the raw-channel/stream layer is internal routing detail: reachable from + # execnet._trio_gateway, not advertised on the public namespace + for name in ("ByteStream", "RawChannel", "RawChannelStream", "serve_gateway"): + assert name not in execnet.trio.__all__, name + + +def test_can_send_lives_only_on_the_top_level() -> None: + assert execnet.can_send({"a": [1, 2.0, b"x", None, (True, frozenset({3}))]}) + assert not execnet.can_send(object()) + # the wire contract does not vary by surface, so it is not mirrored + for namespace in (execnet.sync, execnet.trio, execnet.aio, execnet.portal): + assert "can_send" not in namespace.__all__, namespace.__name__ + + +def test_dumps_is_a_temporary_xdist_shim() -> None: + # FOLLOW-UP: delete this test together with execnet._XDIST_COMPAT once + # pytest-xdist stops probing with ``execnet.dumps`` / ``except DumpError`` + # and uses ``execnet.can_send`` instead. + from execnet import _serialize + + assert execnet._XDIST_COMPAT == ("dumps",) + with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): + assert execnet.dumps is _serialize.dumps + # reachable, but never advertised + assert "dumps" not in execnet.__all__ + assert "dumps" not in dir(execnet) + + def test_portal_namespace() -> None: assert execnet.portal.__all__ == [ "LoopPortal", @@ -81,3 +146,48 @@ def test_import_execnet_does_not_import_trio() -> None: check=True, ) assert out.stdout.strip() == "False" + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_reachable_as_package_attribute(shim: str) -> None: + # pytest-xdist reaches these as ``execnet.gateway_base.X`` after a plain + # ``import execnet``; that used to work because the import chain pulled + # them in, and must keep working for as long as the shims exist. + out = subprocess.run( + [sys.executable, "-c", f"import execnet; print(execnet.{shim}.__name__)"], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == f"execnet.{shim}" + + +def shim_attributes() -> list[tuple[str, str, str]]: + """``(shim, attribute, private module)`` for every forwarded name.""" + cases = [] + for shim in SHIMS: + module = importlib.import_module(f"execnet.{shim}") + for name, target in module._MOVED.items(): + cases.append((shim, name, target)) + return cases + + +@pytest.mark.parametrize(("shim", "name", "target"), shim_attributes(), ids=str) +def test_shim_warns_and_forwards(shim: str, name: str, target: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + private = importlib.import_module(f"execnet{target}") + if not hasattr(private, name): + # ``trace``/``notrace``/``fn`` exist only under a given EXECNET_DEBUG + pytest.skip(f"execnet{target}.{name} not defined in this configuration") + with pytest.warns(DeprecationWarning, match=f"execnet.{shim} is private"): + value = getattr(module, name) + assert value is getattr(private, name) + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_rejects_unknown_attribute(shim: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with pytest.raises(AttributeError, match="no attribute 'nonexistent'"): + getattr(module, "nonexistent") # noqa: B009 diff --git a/testing/test_rsync.py b/testing/test_rsync.py index 2112439f..8190efc7 100644 --- a/testing/test_rsync.py +++ b/testing/test_rsync.py @@ -7,8 +7,8 @@ import pytest import execnet +from execnet import Gateway from execnet import RSync -from execnet.gateway import Gateway @pytest.fixture(scope="module") diff --git a/testing/test_serializer.py b/testing/test_serializer.py index ec8e86ae..ae88f80d 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,8 @@ import execnet -# The package parent: the serializer is imported as execnet.gateway_base -# (it needs its trio-free sibling execnet._boundary; only stdlib beyond that). +# The package parent: the serializer is imported as execnet._serialize +# (it needs its sibling execnet._errors; only stdlib beyond that). pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) @@ -25,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import execnet.gateway_base as serializer +import execnet._serialize as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -41,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import execnet.gateway_base as serializer +import execnet._serialize as serializer from io import BytesIO data = {data!r} io = BytesIO(data) @@ -190,4 +190,4 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: def test_py2_string_opcode_is_retired() -> None: """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" with pytest.raises(execnet.DataFormatError, match="unknown opcode"): - execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") + execnet._serialize.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_termination.py b/testing/test_termination.py index 01c57427..03d6bbb8 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -12,8 +12,8 @@ from test_gateway import TESTTIMEOUT import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel +from execnet import Gateway +from execnet._execmodel import ExecModel execnetdir = pathlib.Path(execnet.__file__).parent.parent diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index ef52e232..35a62c50 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -15,14 +15,14 @@ import trio import trio.testing -from execnet import gateway_base +from execnet import _errors +from execnet._errors import RemoteError +from execnet._message import Message +from execnet._serialize import dumps_internal +from execnet._serialize import loads_internal from execnet._trio_gateway import AsyncGateway from execnet._trio_gateway import AsyncGroup from execnet._trio_gateway import open_popen_gateway -from execnet.gateway_base import Message -from execnet.gateway_base import RemoteError -from execnet.gateway_base import dumps_internal -from execnet.gateway_base import loads_internal @asynccontextmanager @@ -224,7 +224,7 @@ def test_channel_receive_timeout() -> None: async def main() -> None: async with gateway_pair() as (left, _right): channel = left.open_channel() - with pytest.raises(gateway_base.TimeoutError): + with pytest.raises(_errors.TimeoutError): await channel.receive(timeout=0.05) trio.run(main) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 850acbe1..3bcee98a 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -10,9 +10,9 @@ import pytest import execnet +from execnet import Gateway from execnet import XSpec from execnet import _provision -from execnet.gateway import Gateway skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), From 7aa17fed8ef4f703260050cd332ef9dd314106b5 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 22:22:09 +0200 Subject: [PATCH 61/66] refactor!: retire the wakener extension point, add execnet.gevent There is no plan to let third parties plug in event loops, so the Wakener registry was machinery in search of a user: execnet.portal published the protocol and the carriers but not register_wakener, and the promised asyncio wakener was never built (P4 chose per-call bridging). What is left is two wait backends -- OS threads and gevent greenlets -- because every other concurrency library gets a facade. So make the boundary kit private (execnet.portal -> execnet._portal), collapse the registry into a two-branch make_wakener over a WaitBackend literal, and give gevent the facade it was missing. wait= disappears as a spec key: which primitive a *caller* parks on is what picking a namespace already says, and the worker's backend was derived from its profile anyway. Two warts fixed while the carriers were private again: they now raise execnet.TimeoutError instead of the builtin (dropping the translation in Channel.receive), and OneShot's double-resolve check is a RuntimeError rather than an assert that vanishes under -O. Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 18 ++--- src/execnet/_boundary.py | 67 ++++++++---------- src/execnet/_channel.py | 6 +- src/execnet/_gateway.py | 2 - src/execnet/_gateway_base.py | 8 ++- src/execnet/_gevent_support.py | 13 ++-- src/execnet/_multi.py | 42 ++++++----- src/execnet/{portal.py => _portal.py} | 16 ++--- src/execnet/_provision.py | 6 +- src/execnet/_shim.py | 4 +- src/execnet/_trio_host.py | 14 ++-- src/execnet/_trio_worker.py | 2 +- src/execnet/_xspec.py | 1 - src/execnet/gateway_base.py | 2 +- src/execnet/gevent.py | 73 ++++++++++++++++++++ testing/{test_portal.py => test_boundary.py} | 18 +++-- testing/test_gevent.py | 11 +-- testing/test_namespaces.py | 60 ++++++++++------ testing/test_xspec.py | 10 +-- 19 files changed, 228 insertions(+), 145 deletions(-) rename src/execnet/{portal.py => _portal.py} (83%) create mode 100644 src/execnet/gevent.py rename testing/{test_portal.py => test_boundary.py} (76%) diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 5912ae9a..87626d99 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,16 +4,18 @@ pure python lib for connecting to local and remote Python Interpreters. -Four public namespaces: +One namespace per concurrency library you drive execnet from: -* :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` - names below are aliases into it. +* :mod:`execnet.sync` — the blocking API for plain threads; the top-level + ``execnet.*`` names below are aliases into it. * :mod:`execnet.trio` — the trio-native API, awaited inside your own ``trio.run``. -* :mod:`execnet.aio` — the asyncio-native API, bridged over a Trio - host thread. -* :mod:`execnet.portal` — cross-thread / cross-loop communication - primitives shared by all of them. +* :mod:`execnet.aio` — the asyncio-native API. +* :mod:`execnet.gevent` — the blocking API with greenlet-parking waits. + +:mod:`execnet.trio` is the only one that runs gateways *directly* as tasks +in your own nursery. The other three drive a shared Trio host thread, so +their blocking calls must not be made from inside a running event loop. ``can_send`` sits here rather than on any one of them: the wire-format contract is the same whichever surface you drive a gateway from. @@ -68,7 +70,7 @@ #: that still do ``execnet.gateway_base.X`` must reach the warning shim. _LAZY_MODULES = ( "aio", - "portal", + "gevent", "trio", "gateway", "gateway_base", diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index 9c7c69f4..fcd4bae6 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -1,7 +1,12 @@ """Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. Importable without loading any event loop (``import execnet`` must not -import trio); ``execnet.portal`` re-exports these next to LoopPortal. +import trio); :mod:`execnet._portal` re-exports these next to LoopPortal. + +All of this is internal. There are exactly two wait backends -- OS +threads and gevent greenlets -- and no plan to let third parties add +more: every other concurrency library gets a facade of its own +(:mod:`execnet.trio`, :mod:`execnet.aio`) rather than a wakener. """ from __future__ import annotations @@ -9,25 +14,30 @@ import queue import threading import time -from collections.abc import Callable from typing import Any from typing import Generic +from typing import Literal from typing import Protocol from typing import TypeVar from typing import cast +from ._errors import TimeoutError + __all__ = [ "Flag", "Mailbox", "OneShot", "ThreadWakener", + "WaitBackend", "Wakener", "make_wakener", - "register_wakener", ] T = TypeVar("T") +#: which primitive a facade's blocking waits park on +WaitBackend = Literal["thread", "gevent"] + class Wakener(Protocol): """Thread-safe consumer wakeup fired by the loop. @@ -36,8 +46,7 @@ class Wakener(Protocol): the only thing the loop side ever calls. The blocking mailbox/oneshot waits additionally need :meth:`wait`/:meth:`clear` executed in the consumer's own context (a thread here, a greenlet for a gevent - wakener); loop-native consumers such as asyncio wait on their own - side of ``notify`` instead. + wakener). """ def notify(self) -> None: ... @@ -141,14 +150,16 @@ def is_set(self) -> bool: def set(self, value: T) -> None: """Thread-safe; usable from a loop thread (never blocks).""" - assert not self._done, "OneShot already resolved" + if self._done: + raise RuntimeError("OneShot already resolved") self._value = value self._done = True self._wakener.notify() def set_error(self, error: BaseException) -> None: """Resolve with an error that :meth:`wait` will re-raise.""" - assert not self._done, "OneShot already resolved" + if self._done: + raise RuntimeError("OneShot already resolved") self._error = error self._done = True self._wakener.notify() @@ -203,35 +214,17 @@ def wait(self, timeout: float | None = None) -> bool: return self._flag -# wait= axis: named wakener factories; each call returns a fresh instance -# (carriers own their wakener exclusively, see Flag). Backends register -# here next to the built-in "thread"; entries in the lazy table import -# their module (which registers itself) on first use. -_WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { - "thread": ThreadWakener, -} -_LAZY_WAKENER_MODULES: dict[str, str] = { - "gevent": "execnet._gevent_support", -} - - -def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: - """Register a wakener factory for the ``wait=`` spec axis.""" - _WAKENER_FACTORIES[name] = factory +def make_wakener(backend: WaitBackend) -> Wakener: + """Create a fresh wakener for a wait backend. + Each carrier owns its wakener exclusively (see :class:`Flag`), so this + always returns a new instance. gevent is imported lazily -- it is an + optional dependency of the :mod:`execnet.gevent` facade. + """ + if backend == "thread": + return ThreadWakener() + if backend == "gevent": + from ._gevent_support import GeventWakener -def wakener_names() -> list[str]: - return sorted(set(_WAKENER_FACTORIES) | set(_LAZY_WAKENER_MODULES)) - - -def make_wakener(name: str) -> Wakener: - """Create a fresh wakener for the named wait backend.""" - if name not in _WAKENER_FACTORIES and name in _LAZY_WAKENER_MODULES: - import importlib - - importlib.import_module(_LAZY_WAKENER_MODULES[name]) - try: - factory = _WAKENER_FACTORIES[name] - except KeyError: - raise ValueError(f"unknown wait backend {name!r}") from None - return factory() + return GeventWakener() + raise ValueError(f"unknown wait backend {backend!r}") diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index e1f5449b..a13f846b 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -8,7 +8,6 @@ from __future__ import annotations -import builtins import threading import weakref from collections.abc import Callable @@ -326,10 +325,7 @@ def receive(self, timeout: float | None = None) -> Any: mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") - try: - x = mailbox.get(timeout) - except builtins.TimeoutError: - raise self.TimeoutError("no item after %r seconds" % timeout) from None + x = mailbox.get(timeout) if x is ENDMARKER: mailbox.put(x) # for other receivers raise self._getremoteerror() or EOFError() diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index facbdeca..b8cfe863 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -39,8 +39,6 @@ def __init__(self, io: IO, spec: XSpec) -> None: """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - if spec.wait: - self._wait_backend = spec.wait @property def remoteaddress(self) -> str: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 16b21602..55e8bb09 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -23,6 +23,7 @@ from contextlib import suppress from typing import Any +from ._boundary import WaitBackend from ._boundary import Wakener from ._boundary import make_wakener from ._channel import Channel @@ -41,9 +42,10 @@ class BaseGateway: _trio_session: Any = None # Set by the receiver on EOF without a prior termination message. _error: BaseException | None = None - #: wait= axis: which wakener backend this gateway's blocking waits - #: park on (channels, write-acks, join) - _wait_backend: str = "thread" + #: which primitive this gateway's blocking waits park on (channels, + #: write-acks, join). Inherited from the facade coordinator-side, and + #: derived from the worker profile worker-side. + _wait_backend: WaitBackend = "thread" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py index a999ab96..17cb7993 100644 --- a/src/execnet/_gevent_support.py +++ b/src/execnet/_gevent_support.py @@ -1,9 +1,9 @@ -"""The gevent wait backend (``wait=gevent``). +"""The gevent wait backend behind :mod:`execnet.gevent`. -Importing this module registers the ``gevent`` wakener factory; the -boundary kit imports it lazily when a spec asks for ``wait=gevent``. +The boundary kit imports this lazily (``make_wakener("gevent")``) so +gevent stays an optional dependency. -A blocking wait then parks only the calling greenlet: the carrier's +A blocking wait parks only the calling greenlet: the carrier's event lives in the waiting greenlet's hub, and the loop side's ``notify()`` crosses threads through a ``loop.async_`` watcher -- the one libev/libuv primitive gevent documents as safe to use from other @@ -18,8 +18,6 @@ import gevent.event from gevent.hub import get_hub -from ._boundary import register_wakener - class GeventWakener: """Wakener parking greenlets instead of OS threads. @@ -71,6 +69,3 @@ def clear(self) -> None: event = self._event if event is not None: event.clear() - - -register_wakener("gevent", GeventWakener) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 429103f7..113ff1c2 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -23,7 +23,7 @@ from typing import TypeAlias from typing import overload -from ._boundary import wakener_names +from ._boundary import WaitBackend from ._channel import Channel from ._execmodel import EXECMODEL_PROFILES from ._execmodel import ExecModel @@ -43,6 +43,12 @@ class Group: defaultspec = "popen" + #: which primitive this group's blocking waits park on. Set by the + #: facade, not by a spec: it describes the *caller's* concurrency + #: library, which is exactly what picking a namespace already says. + #: ``execnet.gevent.Group`` overrides it. + _wait_backend: WaitBackend = "thread" + def __init__( self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" ) -> None: @@ -178,10 +184,6 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: f"unknown execmodel {spec.execmodel!r}" f" (known profiles: {list(EXECMODEL_PROFILES)})" ) - if spec.wait is not None and spec.wait not in wakener_names(): - raise ValueError( - f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" - ) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -256,26 +258,22 @@ def terminate(self, timeout: float | None = None) -> None: self._gateways_to_join[:] = [] def _host_terminate(self, timeout: float | None) -> None: - """Terminate the async group, parking correctly for wait backends. + """Terminate the async group, parking correctly for the wait backend. - A member gateway created with a non-thread ``wait=`` implies the - caller may be a greenlet: wait on a OneShot instead of blocking - the OS thread (which would stall the hub for the whole grace). + A non-thread backend implies the caller may be a greenlet: wait on + a OneShot instead of blocking the OS thread (which would stall the + hub for the whole grace). """ - backends = {gw._wait_backend for gw in self._gateways_to_join} | { - gw._wait_backend for gw in self - } - backends.discard("thread") - if backends: - from ._boundary import make_wakener - - self._trio_host.call_pending( - self._async_group.terminate, - timeout, - wakener=make_wakener(backends.pop()), - ).wait() - else: + if self._wait_backend == "thread": self._trio_host.call(self._async_group.terminate, timeout) + return + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(self._wait_backend), + ).wait() def remote_exec( self, diff --git a/src/execnet/portal.py b/src/execnet/_portal.py similarity index 83% rename from src/execnet/portal.py rename to src/execnet/_portal.py index 5cc3e30e..6c1253c4 100644 --- a/src/execnet/portal.py +++ b/src/execnet/_portal.py @@ -1,17 +1,17 @@ """Cross-thread / cross-loop communication primitives — the boundary kit. -A :class:`LoopPortal` is a handle to a running trio loop that foreign -threads use to run functions on the loop or push work into it (the +Internal. A :class:`LoopPortal` is a handle to a running trio loop that +foreign threads use to run functions on the loop or push work into it (the consumer -> loop direction). Because each direction only needs the -*receiving* loop's token, two trio loops in two threads can communicate -by holding each other's portal. +*receiving* loop's token, two trio loops in two threads can communicate by +holding each other's portal. The loop -> consumer direction never blocks the loop and never knows who is listening: the loop fires a :class:`Wakener`, a single thread-safe -``notify()`` supplied by the consumer. Implementing that one method is -the entire integration surface for an event loop backend (threads here; -asyncio/gevent wakeners come with their facades). On top of it sit the -two carriers: +``notify()`` supplied by the consumer. There are exactly two wakeners -- +OS threads and gevent greenlets -- because every other concurrency library +gets a facade of its own (:mod:`execnet.trio`, :mod:`execnet.aio`) instead. +On top of the wakener sit the two carriers: * :class:`Mailbox` -- an item stream (channel payloads, exec requests), * :class:`OneShot` -- a single result (write acknowledgements, call diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 4da4f689..dcfaf1a4 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,12 +164,12 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet - # the gevent profile parks its greenlets on gevent wakeners - default_wait = "gevent" if spec.execmodel == "gevent" else "thread" config: dict[str, Any] = { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, - "wait": spec.wait or default_wait, + # derived, not configurable: the gevent profile parks its greenlets + # on gevent wakeners, every other profile is thread-shaped. + "wait": "gevent" if spec.execmodel == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py index 35e7682c..0839c2e3 100644 --- a/src/execnet/_shim.py +++ b/src/execnet/_shim.py @@ -8,7 +8,7 @@ whose ``__getattr__`` warns and forwards. The supported surfaces are :mod:`execnet` / :mod:`execnet.sync`, -:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.portal`. +:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.gevent`. """ from __future__ import annotations @@ -38,7 +38,7 @@ def __getattr__(name: str) -> Any: warnings.warn( f"execnet.{shim} is private and will be removed in {REMOVED_IN}; " f"{name} now lives in execnet{module}. The supported surfaces are " - f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.portal.", + f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.gevent.", DeprecationWarning, stacklevel=2, ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index a1ea4acb..8e1d695b 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -45,8 +45,8 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from .portal import LoopPortal -from .portal import OneShot +from ._portal import LoopPortal +from ._portal import OneShot #: default cap on concurrent threadpool threads running receiver callbacks DEFAULT_CALLBACK_THREADS = 40 @@ -719,6 +719,8 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + # the caller's concurrency library, inherited from the facade + sync_gw._wait_backend = self.group._wait_backend return SyncBridgeGateway( stream, id=spec.id, sync_gateway=sync_gw, host=self.host ) @@ -762,16 +764,16 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - if spec.wait and spec.wait != "thread": + if group._wait_backend == "thread": + bridge = host.call(async_group.makegateway, spec) + else: # e.g. a gevent app: wait on a OneShot so only the calling # greenlet parks while the gateway comes up, not the whole hub. from ._boundary import make_wakener bridge = host.call_pending( - async_group.makegateway, spec, wakener=make_wakener(spec.wait) + async_group.makegateway, spec, wakener=make_wakener(group._wait_backend) ).wait() - else: - bridge = host.call(async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 633b7d94..741c5eff 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -18,7 +18,7 @@ from ._gateway_base import WorkerGateway from ._serialize import loads_internal from ._trace import trace -from .portal import Mailbox +from ._boundary import Mailbox if TYPE_CHECKING: from . import _trio_host diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py index afc58f4c..0559ed8c 100644 --- a/src/execnet/_xspec.py +++ b/src/execnet/_xspec.py @@ -29,7 +29,6 @@ class XSpec: ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None - wait: str | None = None def __init__(self, string: str) -> None: self._spec = string diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index b54c68bf..ba1eb35f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -6,7 +6,7 @@ to them with a :class:`DeprecationWarning`. Use :mod:`execnet` / :mod:`execnet.sync`, :mod:`execnet.trio`, -:mod:`execnet.aio` or :mod:`execnet.portal` instead. The standalone +:mod:`execnet.aio` or :mod:`execnet.gevent` instead. The standalone serializer stays internal; :func:`execnet.can_send` answers "can this value cross a channel?" without it. """ diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py new file mode 100644 index 00000000..f3f12609 --- /dev/null +++ b/src/execnet/gevent.py @@ -0,0 +1,73 @@ +"""The blocking execnet API for gevent applications. + +Identical to :mod:`execnet.sync` except that every blocking wait parks the +calling *greenlet* rather than its OS thread:: + + import execnet.gevent + + group = execnet.gevent.Group() + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + print(channel.receive()) # parks this greenlet, not the hub + +Protocol IO runs on the shared Trio host thread as it does for every +blocking surface; the difference is only which primitive a waiter parks +on, so a slow ``receive`` no longer stalls the whole hub. Requires +gevent (``execnet[gevent]``). + +This is about the *caller*: the worker's own shape is the ``profile=`` +spec key, and ``profile=gevent`` is an independent choice. Importing this +module does not monkey-patch anything -- do that yourself, as early as +usual. +""" + +from __future__ import annotations + +import gevent # noqa: F401 -- fail at import time when gevent is missing + +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._multi import Group as _SyncGroup +from ._multi import MultiChannel +from ._xspec import XSpec +from .sync import Channel +from .sync import Gateway +from .sync import RSync + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "makegateway", +] + + +class Group(_SyncGroup): + """A gateway group whose blocking waits park greenlets. + + Every gateway it creates inherits the gevent wait backend, so + ``channel.receive()``, ``waitclose()``, sends waiting on their write + acknowledgement, ``join()`` and ``Group.terminate()`` all yield to the + hub instead of blocking the thread running it. + """ + + _wait_backend = "gevent" + + +#: convenience group for scripts; real applications should own a Group +default_group = Group() +makegateway = default_group.makegateway diff --git a/testing/test_portal.py b/testing/test_boundary.py similarity index 76% rename from testing/test_portal.py rename to testing/test_boundary.py index 1a3759af..ce2a5302 100644 --- a/testing/test_portal.py +++ b/testing/test_boundary.py @@ -7,8 +7,9 @@ import pytest -from execnet.portal import Mailbox -from execnet.portal import OneShot +from execnet import TimeoutError as ExecnetTimeoutError +from execnet._boundary import Mailbox +from execnet._boundary import OneShot class TestMailbox: @@ -24,8 +25,10 @@ def test_get_nowait_empty(self) -> None: box.get_nowait() def test_get_timeout(self) -> None: + # the carriers speak execnet's TimeoutError, so Channel.receive + # does not have to translate the builtin one box: Mailbox[int] = Mailbox() - with pytest.raises(TimeoutError): + with pytest.raises(ExecnetTimeoutError): box.get(timeout=0.01) def test_get_blocks_until_put_from_other_thread(self) -> None: @@ -54,7 +57,7 @@ def test_set_then_wait(self) -> None: def test_wait_timeout(self) -> None: shot: OneShot[int] = OneShot() - with pytest.raises(TimeoutError): + with pytest.raises(ExecnetTimeoutError): shot.wait(timeout=0.01) assert not shot.is_set() @@ -69,8 +72,11 @@ def test_wait_blocks_until_set_from_other_thread(self) -> None: threading.Timer(0.05, shot.set, args=["done"]).start() assert shot.wait(timeout=5.0) == "done" - def test_single_resolution_asserted(self) -> None: + def test_single_resolution_enforced(self) -> None: + # a real error, not an assert: it must survive python -O shot: OneShot[int] = OneShot() shot.set(1) - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError, match="already resolved"): shot.set(2) + with pytest.raises(RuntimeError, match="already resolved"): + shot.set_error(RuntimeError("boom")) diff --git a/testing/test_gevent.py b/testing/test_gevent.py index cbec77a9..5d8a84c3 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -1,4 +1,4 @@ -"""The gevent wait backend (wait=gevent): greenlet-parking blocking waits. +"""The execnet.gevent facade: greenlet-parking blocking waits. Opt-in: requires the ``gevent`` dependency group (``uv sync --group gevent``); skipped when gevent is not installed. No monkey-patching is @@ -15,6 +15,7 @@ gevent = pytest.importorskip("gevent") import execnet # noqa: E402 +import execnet.gevent # noqa: E402 from execnet._boundary import Flag # noqa: E402 from execnet._boundary import Mailbox # noqa: E402 from execnet._boundary import make_wakener # noqa: E402 @@ -24,9 +25,9 @@ @pytest.fixture def gevent_gw(): - group = execnet.Group() + group = execnet.gevent.Group() try: - yield group.makegateway("popen//wait=gevent") + yield group.makegateway("popen") finally: group.terminate(timeout=5.0) @@ -88,7 +89,7 @@ def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: def test_makegateway_parks_greenlet_not_hub(self) -> None: # management ops (makegateway/terminate) from a greenlet must not # stall the hub: they wait on a OneShot with a gevent wakener. - group = execnet.Group() + group = execnet.gevent.Group() progressed: list[int] = [] def other() -> None: @@ -98,7 +99,7 @@ def other() -> None: try: ticker = gevent.spawn(other) - maker = gevent.spawn(group.makegateway, "popen//wait=gevent") + maker = gevent.spawn(group.makegateway, "popen") gw = maker.get(timeout=TESTTIMEOUT) channel = gw.remote_exec("channel.send(42)") assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 822f8d50..49f47bb4 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -1,9 +1,11 @@ """The public namespaces, and the deprecation shims for the private modules. -Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the trio -and aio namespaces expose the async-native core and its asyncio bridge; the -portal namespace the cross-thread primitives. Everything else in the package -is private -- the pre-Trio module names survive only as warning shims. +There is one namespace per concurrency library the caller drives execnet +from: top-level ``execnet.*`` is an alias surface over ``execnet.sync`` +(threads), ``execnet.trio`` and ``execnet.aio`` expose the async-native core +and its asyncio bridge, and ``execnet.gevent`` the greenlet-parking blocking +surface. Everything else in the package is private -- the pre-Trio module +names survive only as warning shims. """ from __future__ import annotations @@ -18,12 +20,14 @@ import execnet import execnet.aio -import execnet.portal import execnet.sync import execnet.trio #: the only modules that may be reachable without a leading underscore -PUBLIC_NAMESPACES = ("aio", "portal", "sync", "trio") +PUBLIC_NAMESPACES = ("aio", "gevent", "sync", "trio") + +#: those importable without an optional dependency (execnet.gevent needs gevent) +ALWAYS_IMPORTABLE = tuple(n for n in PUBLIC_NAMESPACES if n != "gevent") #: pre-Trio module names kept as deprecated forwarding shims SHIMS = ("gateway", "gateway_base", "multi", "rsync", "rsync_remote", "xspec") @@ -45,13 +49,22 @@ def test_top_level_all_matches_sync_surface() -> None: @pytest.mark.parametrize( "namespace", - [execnet, *[importlib.import_module(f"execnet.{n}") for n in PUBLIC_NAMESPACES]], + [execnet, *[importlib.import_module(f"execnet.{n}") for n in ALWAYS_IMPORTABLE]], ) def test_namespace_all_resolves(namespace: object) -> None: missing = [n for n in namespace.__all__ if not hasattr(namespace, n)] # type: ignore[attr-defined] assert not missing +def test_gevent_namespace_all_resolves() -> None: + pytest.importorskip("gevent") + namespace = importlib.import_module("execnet.gevent") + assert not [n for n in namespace.__all__ if not hasattr(namespace, n)] + # the facade is the sync surface with greenlet parking wired in + assert namespace.Group._wait_backend == "gevent" + assert issubclass(namespace.Group, execnet.Group) + + def test_no_unexpected_public_modules() -> None: found = { info.name @@ -85,7 +98,7 @@ def test_can_send_lives_only_on_the_top_level() -> None: assert execnet.can_send({"a": [1, 2.0, b"x", None, (True, frozenset({3}))]}) assert not execnet.can_send(object()) # the wire contract does not vary by surface, so it is not mirrored - for namespace in (execnet.sync, execnet.trio, execnet.aio, execnet.portal): + for namespace in (execnet.sync, execnet.trio, execnet.aio): assert "can_send" not in namespace.__all__, namespace.__name__ @@ -103,32 +116,35 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: assert "dumps" not in dir(execnet) -def test_portal_namespace() -> None: - assert execnet.portal.__all__ == [ - "LoopPortal", - "Mailbox", - "OneShot", - "ThreadWakener", - "Wakener", - ] - assert execnet.portal.LoopPortal is not None +def test_boundary_kit_is_private() -> None: + # There is no third-party event-loop extension point: the two wait + # backends are threads and gevent, and every other concurrency library + # gets a facade instead of a wakener. + assert not hasattr(execnet, "portal") + for name in ("Wakener", "Mailbox", "OneShot", "LoopPortal"): + assert not hasattr(execnet, name), name + from execnet import _boundary + + assert not hasattr(_boundary, "register_wakener") + assert _boundary.make_wakener("thread") is not None + with pytest.raises(ValueError, match="unknown wait backend"): + _boundary.make_wakener("nope") # type: ignore[arg-type] def test_lazy_submodule_attribute_access() -> None: - # After ``import execnet`` alone, execnet.trio / execnet.portal are - # reachable as attributes (PEP 562) without having been imported. + # After ``import execnet`` alone, execnet.trio is reachable as an + # attribute (PEP 562) without having been imported. out = subprocess.run( [ sys.executable, "-c", - "import execnet; print(execnet.trio.AsyncGroup.__name__);" - " print(execnet.portal.LoopPortal.__name__)", + "import execnet; print(execnet.trio.AsyncGroup.__name__)", ], capture_output=True, text=True, check=True, ) - assert out.stdout.split() == ["AsyncGroup", "LoopPortal"] + assert out.stdout.strip() == "AsyncGroup" def test_import_execnet_does_not_import_trio() -> None: diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 3bcee98a..891b6a1a 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -119,10 +119,12 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) - def test_wait_axis(self, makegateway: Callable[[str], Gateway]) -> None: - with pytest.raises(ValueError, match="unknown wait backend"): - makegateway("popen//wait=nope") - gw = makegateway("popen//wait=thread") + def test_wait_backend_comes_from_the_facade( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # not a spec key: the blocking surface decides how *it* parks, and + # a thread-shaped worker profile parks on threads either way + gw = makegateway("popen") assert gw._wait_backend == "thread" channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") assert channel.receive() == "thread" From c0faa503ab5ef2ab86c61dd23cae655126d57e91 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 08:59:33 +0200 Subject: [PATCH 62/66] feat!: rename execmodel= to profile=, deprecate main_thread_only "execmodel" named a *local* execution model that the Trio port deleted: Group(execmodel=), set_execmodel() and group.execmodel have had no behavioural effect for a while, while the spec key of the same name selects something else entirely -- the worker's profile. So the spec key becomes profile=, execmodel= stays as a permanently accepted alias (pytest-xdist passes both the kwarg and the spec key), and the local half is deprecated down to the remote default it actually sets. main_thread_only goes with it, as an accepted alias for thread. The restored hybrid thread profile already hands the first remote_exec the worker's real main thread, which is the GUI/signal property the profile existed for; what it additionally did was refuse a *second* concurrent remote_exec, and that was a deadlock guard built on a 1s timeout that misread a merely slow predecessor under load. Behaviour change worth noting: such a request now runs on a pool thread instead of failing. Fixes a latent livelock this surfaced: execnet.dumps warned on every access, and xdist calls it from serialize_warning_message -- i.e. from inside pytest's warning-recording hook -- so a single DeprecationWarning in a worker recorded a warning that recorded a warning, unbounded, and wedged the run. The shim now warns once per process. Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 32 ++++++--- src/execnet/_errors.py | 3 - src/execnet/_execmodel.py | 51 +++++++++++--- src/execnet/_gateway.py | 10 ++- src/execnet/_gateway_base.py | 9 +-- src/execnet/_multi.py | 119 +++++++++++++++++++++++---------- src/execnet/_provision.py | 9 ++- src/execnet/_trio_gateway.py | 14 +++- src/execnet/_trio_host.py | 7 +- src/execnet/_trio_worker.py | 99 ++++++++++----------------- src/execnet/_xspec.py | 20 +++++- src/execnet/gateway_base.py | 1 - src/execnet/sync.py | 2 + testing/conftest.py | 29 ++++---- testing/test_execmodel_trio.py | 28 ++++++-- testing/test_gateway.py | 60 ++++++----------- testing/test_namespaces.py | 13 ++++ 17 files changed, 305 insertions(+), 201 deletions(-) diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 87626d99..315dd0bf 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -42,6 +42,7 @@ from .sync import default_group from .sync import makegateway from .sync import set_execmodel +from .sync import set_profile __all__ = [ "Channel", @@ -61,6 +62,7 @@ "default_group", "makegateway", "set_execmodel", + "set_profile", ] @@ -92,6 +94,14 @@ #: then delete this and its test. Nothing else may be added here. _XDIST_COMPAT = ("dumps",) +#: Warn about the shim at most once per process. xdist reaches it from +#: ``serialize_warning_message``, i.e. from inside pytest's +#: warning-recording hook: warning every time makes recording one warning +#: record another, which records another -- an unbounded loop that wedges +#: the worker. Once is also simply the right amount of noise for a name +#: whose caller is a library, not the user. +_xdist_compat_warned: set[str] = set() + def __getattr__(name: str) -> Any: if name in _LAZY_MODULES: @@ -100,14 +110,18 @@ def __getattr__(name: str) -> Any: return importlib.import_module(f".{name}", __name__) if name in _XDIST_COMPAT: import importlib - import warnings - - warnings.warn( - f"execnet.{name} is a temporary pytest-xdist compatibility shim and" - " will be removed; the standalone serializer is not public. Use" - " execnet.can_send(obj) to test whether a value can cross a channel.", - DeprecationWarning, - stacklevel=2, - ) + + if name not in _xdist_compat_warned: + import warnings + + _xdist_compat_warned.add(name) + warnings.warn( + f"execnet.{name} is a temporary pytest-xdist compatibility shim" + " and will be removed; the standalone serializer is not public." + " Use execnet.can_send(obj) to test whether a value can cross a" + " channel.", + DeprecationWarning, + stacklevel=2, + ) return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py index e90ee2d0..276fbb5e 100644 --- a/src/execnet/_errors.py +++ b/src/execnet/_errors.py @@ -17,9 +17,6 @@ sysex = (KeyboardInterrupt, SystemExit) INTERRUPT_TEXT = "keyboard-interrupted" -MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( - "concurrent remote_exec would cause deadlock for main_thread_only execmodel" -) class GatewayReceivedTerminate(Exception): diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index 9cb1d75c..adb43950 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -1,15 +1,19 @@ -"""Deprecated execution-model presets. +"""Worker profiles, and the deprecated ExecModel shim. -The machinery behind execution models was retired during the Trio port; what -survives is the preset *name*, which maps onto the worker config axes -(``loop=`` / ``exec=`` / ``wait=``). Kept because pytest-xdist's remote worker -still builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. +The machinery behind "execution models" was retired during the Trio port. +What the name actually selected -- where exec'd code runs relative to the +worker's protocol loop -- survives as the ``profile=`` spec key and +:data:`WORKER_PROFILES`. + +:class:`ExecModel` itself is kept only because pytest-xdist's remote worker +builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. """ from __future__ import annotations import os import threading +import warnings class ExecModel: @@ -76,17 +80,46 @@ def Event(self) -> threading.Event: #: worker profiles: where exec'd code runs relative to the protocol loop -EXECMODEL_PROFILES = ( +WORKER_PROFILES = ( "thread", # hybrid: primary on the main thread, overflow on pool threads - "main_thread_only", # exec serialized on the main thread (GUI/pytest) "trio", # pure async: loop owns the main thread, async sources as tasks "gevent", # greenlets on a main-thread hub, one per remote_exec ) +#: profiles kept as accepted spellings, mapped to what they now select. +#: ``main_thread_only`` predates the restored hybrid ``thread`` profile, +#: which already hands the first remote_exec the real main thread -- the +#: GUI/signal property it existed for. What it additionally did was refuse +#: a *second* concurrent remote_exec instead of overflowing to a pool +#: thread; that guard is gone. +DEPRECATED_PROFILES = {"main_thread_only": "thread"} + + +def resolve_profile(name: str) -> str: + """Validate a ``profile=`` value, mapping deprecated spellings.""" + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + warnings.warn( + f"the {name!r} worker profile is deprecated and now behaves like" + f" {replacement!r}, which already runs the first remote_exec on" + " the worker's main thread. A second concurrent remote_exec no" + " longer fails -- it runs on a pool thread.", + DeprecationWarning, + stacklevel=3, + ) + return replacement + if name not in WORKER_PROFILES: + raise ValueError( + f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})" + ) + return name + + def get_execmodel(backend: str | ExecModel) -> ExecModel: + """Deprecated: build the xdist-facing shim for a profile name.""" if isinstance(backend, ExecModel): return backend - if backend in EXECMODEL_PROFILES: + if backend in WORKER_PROFILES or backend in DEPRECATED_PROFILES: return ExecModel(backend) - raise ValueError(f"unknown execmodel {backend!r}") + raise ValueError(f"unknown profile {backend!r}") diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index b8cfe863..bbbb936c 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -6,6 +6,7 @@ from __future__ import annotations import types +import warnings from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any @@ -53,7 +54,7 @@ def __repr__(self) -> str: except AttributeError: r = "uninitialized" i = "no" - return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" + return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.spec.profile} profile, {i} active channels>" def exit(self) -> None: """Trigger gateway exit. @@ -137,7 +138,12 @@ def remote_exec( def remote_init_threads(self, num: int | None = None) -> None: """DEPRECATED. Is currently a NO-OPERATION already.""" - print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") + warnings.warn( + "remote_init_threads() has been a no-operation since execnet 1.2" + " and will be removed; drop the call.", + DeprecationWarning, + stacklevel=2, + ) class RInfo: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 55e8bb09..4d0f762e 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -17,7 +17,6 @@ import os import sys -import threading from _thread import interrupt_main from collections.abc import Callable from contextlib import suppress @@ -69,7 +68,7 @@ def _attach_trio_session(self, session: Any) -> None: session.bind_sync_channel(channel) def _new_wakener(self) -> Wakener: - """A fresh wakener for one blocking-wait carrier (wait= axis).""" + """A fresh wakener for one blocking-wait carrier.""" return make_wakener(self._wait_backend) def _bind_channel(self, channel: Channel) -> None: @@ -171,7 +170,6 @@ class WorkerGateway(BaseGateway): _trio_exec: Any = None # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). _execpool: Any = None - _executetask_complete: threading.Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: trio_exec = self._trio_exec @@ -232,8 +230,3 @@ def executetask( channel.close(errortext) return channel.close() - if self._executetask_complete is not None: - # Indicate that this task has finished executing, meaning - # that there is no possibility of it triggering a deadlock - # for the next spawn call. - self._executetask_complete.set() diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 113ff1c2..ee751597 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -11,6 +11,7 @@ import threading import time import types +import warnings from collections.abc import Callable from collections.abc import Iterable from collections.abc import Iterator @@ -25,9 +26,9 @@ from ._boundary import WaitBackend from ._channel import Channel -from ._execmodel import EXECMODEL_PROFILES from ._execmodel import ExecModel from ._execmodel import get_execmodel +from ._execmodel import resolve_profile from ._trace import trace from ._xspec import XSpec @@ -50,24 +51,36 @@ class Group: _wait_backend: WaitBackend = "thread" def __init__( - self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" + self, + xspecs: Iterable[XSpec | str | None] = (), + profile: str | None = None, + *, + execmodel: str | None = None, ) -> None: """Initialize a group and make gateways as specified. - execmodel can be one of the supported execution models. + ``profile`` is the default worker profile for gateways created + without an explicit ``profile=`` in their spec. ``execmodel`` is + the deprecated spelling (pytest-xdist still passes it). """ + if execmodel is not None: + if profile is not None: + raise TypeError("pass either profile= or execmodel=, not both") + warnings.warn( + "Group(execmodel=...) is deprecated; use Group(profile=...)." + " execnet has no local execution model any more -- the value" + " only selects the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + profile = execmodel self._gateways: list[Gateway] = [] self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] self._trio_host: Any = None self._async_group: Any = None - # we use the same execmodel for all of the Gateway objects - # we spawn on our side. Probably we should not allow different - # execmodels between different groups but not clear. - # Note that "other side" execmodels may differ and is typically - # specified by the spec passed to makegateway. - self.set_execmodel(execmodel) + self.set_profile("thread" if profile is None else profile) for xspec in xspecs: self.makegateway(xspec) atexit.register(self._cleanup_atexit) @@ -94,33 +107,63 @@ async def _start() -> Any: self._async_group = host.call(_start) return self._async_group + @property + def profile(self) -> str: + """Default worker profile for gateways created by this group.""" + return self._profile + + def set_profile(self, profile: str) -> None: + """Set the default worker profile for newly created gateways. + + NOTE: only settable before any gateway is created. + """ + if self._gateways: + raise ValueError( + "can not set the profile if gateways have been created already" + ) + self._profile = resolve_profile(profile) + @property def execmodel(self) -> ExecModel: - return self._execmodel + """Deprecated: there is no local execution model any more.""" + warnings.warn( + "Group.execmodel is deprecated: execnet has no local execution" + " model. Use Group.profile for the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) @property def remote_execmodel(self) -> ExecModel: - return self._remote_execmodel + """Deprecated alias for :attr:`profile`, as an ExecModel shim.""" + warnings.warn( + "Group.remote_execmodel is deprecated; use Group.profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) def set_execmodel( self, execmodel: str, remote_execmodel: str | None = None ) -> None: - """Set the execution model for local and remote site. + """Deprecated alias for :meth:`set_profile`. - execmodel can be one of the supported execution models. - It determines the execution model for any newly created gateway. - If remote_execmodel is not specified it takes on the value of execmodel. - - NOTE: Execution models can only be set before any gateway is created. + The *local* execution model it used to set no longer exists -- all + protocol IO runs on the Trio host -- so only the worker profile is + taken from these arguments (``remote_execmodel`` when given, else + ``execmodel``). """ - if self._gateways: - raise ValueError( - "can not set execution models if gateways have been created already" - ) - if remote_execmodel is None: - remote_execmodel = execmodel - self._execmodel = get_execmodel(execmodel) - self._remote_execmodel = get_execmodel(remote_execmodel) + warnings.warn( + "Group.set_execmodel is deprecated; use Group.set_profile(profile)." + " execnet has no local execution model, so only the remote value" + " has an effect.", + DeprecationWarning, + stacklevel=2, + ) + self.set_profile( + execmodel if remote_execmodel is None else remote_execmodel + ) def __repr__(self) -> str: idgateways = [gw.id for gw in self] @@ -161,11 +204,14 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=name worker profile: where exec'd code runs relative - to the worker's protocol loop. 'thread' (pool - threads) or 'main_thread_only' (serialized on - the worker main thread, GUI/signal-safe). - wait=backend wakener for blocking waits ('thread' default) + profile=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' + (default; the first remote_exec claims the + worker main thread, further ones overflow to + pool threads), 'trio' (async sources as tasks, + single-threaded) or 'gevent' (a greenlet per + remote_exec). Spelled 'execmodel=' before + execnet 3.0; that spelling still works. chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. @@ -177,13 +223,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: if not isinstance(spec, XSpec): spec = XSpec(spec) self.allocate_id(spec) - if spec.execmodel is None: - spec.execmodel = self.remote_execmodel.backend - elif spec.execmodel not in EXECMODEL_PROFILES: - raise ValueError( - f"unknown execmodel {spec.execmodel!r}" - f" (known profiles: {list(EXECMODEL_PROFILES)})" - ) + if spec.profile is None: + spec.profile = self._profile + else: + spec.profile = resolve_profile(spec.profile) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -414,4 +457,6 @@ def run_term() -> None: default_group = Group() makegateway = default_group.makegateway +set_profile = default_group.set_profile +#: deprecated alias, see Group.set_execmodel set_execmodel = default_group.set_execmodel diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index dcfaf1a4..67b02d17 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -166,10 +166,13 @@ def worker_cli_arg(spec: Any) -> str: config: dict[str, Any] = { "id": f"{spec.id}-worker", - "execmodel": spec.execmodel, + "profile": spec.profile, + # pre-3.0 spelling, same value: a worker from an older execnet + # reads this one. Drop with the XSpec alias. + "execmodel": spec.profile, # derived, not configurable: the gevent profile parks its greenlets # on gevent wakeners, every other profile is thread-shaped. - "wait": "gevent" if spec.execmodel == "gevent" else "thread", + "wait": "gevent" if spec.profile == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through @@ -213,7 +216,7 @@ def _extra_with_tokens(config: str) -> list[str]: backend needs gevent importable in the worker. """ parsed = json.loads(config) - if parsed.get("execmodel") == "gevent" or parsed.get("wait") == "gevent": + if parsed.get("profile") == "gevent" or parsed.get("wait") == "gevent": return ["--with", "gevent"] return [] diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 90246fb7..db60920e 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -42,6 +42,7 @@ from ._errors import HostNotFound from ._errors import RemoteError from ._errors import TimeoutError +from ._execmodel import resolve_profile from ._exec_source import normalize_exec_source from ._message import FrameDecoder from ._message import Message @@ -667,6 +668,8 @@ def _dispatch(self, message: Message) -> None: status = { "numchannels": len(self._channels), "numexecuting": task_exec.active_count() if task_exec else 0, + "profile": "trio", + # legacy key, same value -- pytest-xdist reads it "execmodel": "trio", } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) @@ -905,9 +908,14 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: raise RuntimeError(f"{self!r} is not entered") if not isinstance(spec, XSpec): spec = XSpec(spec) - if spec.execmodel is None: - # the sync Group normally stamps this before spawning - spec.execmodel = "thread" + if spec.profile is None: + # An async coordinator does not imply an async worker: the + # worker's shape is its own choice, and the default stays the + # thread profile. Pass ``profile=trio`` for a worker that runs + # exec'd async sources as tasks. + spec.profile = "thread" + else: + spec.profile = resolve_profile(spec.profile) if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 8e1d695b..82ad9c37 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -31,6 +31,7 @@ from ._errors import GatewayReceivedTerminate from ._errors import RemoteError from ._execmodel import ExecModel +from ._execmodel import get_execmodel from ._message import FrameDecoder from ._message import Message from ._message import gateway_info @@ -224,6 +225,8 @@ def _answer_status(self, message: Message) -> None: d = { "numchannels": len(gateway._channelfactory._channels), "numexecuting": execpool.active_count() if execpool is not None else 0, + "profile": gateway.execmodel.backend, + # legacy key, same value -- pytest-xdist reads it "execmodel": gateway.execmodel.backend, } gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) @@ -718,7 +721,7 @@ def __init__(self, group: Group, host: TrioHost) -> None: def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet - sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + sync_gw = execnet.Gateway(_TempIO(get_execmodel(spec.profile)), spec) # the caller's concurrency library, inherited from the facade sync_gw._wait_backend = self.group._wait_backend return SyncBridgeGateway( @@ -777,7 +780,7 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( - group.execmodel, + get_execmodel(spec.profile), bridge, process=async_group._processes.get(bridge), remoteaddress=bridge.remoteaddress, diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 741c5eff..0146c00b 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,7 +2,6 @@ from __future__ import annotations -import functools import os import sys import threading @@ -12,7 +11,6 @@ import trio -from ._errors import MAIN_THREAD_ONLY_DEADLOCK_TEXT from ._errors import geterrortext from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway @@ -59,25 +57,15 @@ def trigger_shutdown(self) -> None: pass -#: How long admission waits for the previous main-thread exec to finish -#: before declaring a deadlock. Kept small: a genuine second concurrent -#: remote_exec never returns, so this only affects how fast that is -#: reported. (Under heavy CPU contention a merely slow predecessor can be -#: misread as a deadlock, but main_thread_only is an xdist-only transitional -#: profile slated for removal, so the whole guard goes away with it.) -MAIN_THREAD_ONLY_ADMIT_TIMEOUT = 1.0 +class PrimaryThreadPump: + """Runs exec requests handed to it on the process main thread. - -class MainExec: - """Exec strategy: serialize each request onto the process main thread. - - The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist - runs this way). Admission waits for the previous request to finish and - closes the channel with the deadlock text when it cannot within - ``MAIN_THREAD_ONLY_ADMIT_TIMEOUT`` (a second concurrent remote_exec would - deadlock the requester). + The building block for every strategy that needs a real main thread: + :meth:`integrate_as_primary_thread` parks there draining a mailbox, and + :meth:`run` hands one request over and awaits its completion. """ + #: whether _run_worker must hand this strategy the process main thread needs_primary_thread = True def __init__(self, gateway: WorkerGateway) -> None: @@ -87,15 +75,7 @@ def __init__(self, gateway: WorkerGateway) -> None: ) async def admit(self, channel: Channel, item: ExecItem) -> bool: - complete = self.gateway._executetask_complete - assert complete is not None - wait_slot = functools.partial( - complete.wait, timeout=MAIN_THREAD_ONLY_ADMIT_TIMEOUT - ) - if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return False - complete.clear() + """FIFO admission gate; always open.""" return True async def run(self, channel: Channel, item: ExecItem) -> None: @@ -119,14 +99,19 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -class HybridExec(MainExec): +class HybridExec(PrimaryThreadPump): """Exec strategy: primary on the main thread, overflow on pool threads. - The classic ``thread`` execmodel shape: a request arriving while the + The classic ``thread`` profile shape: a request arriving while the main thread is idle claims it (pytest and friends get a true main thread); requests arriving while it is busy run on worker threads instead of queueing. The claim is decided during FIFO admission so the *first* request always gets the main thread. + + This is also what the retired ``main_thread_only`` profile now maps to: + it existed for the main-thread guarantee, which the claim provides, + and its extra behaviour -- refusing a second concurrent remote_exec + rather than overflowing -- was a deadlock guard, not a feature. """ def __init__(self, gateway: WorkerGateway) -> None: @@ -160,8 +145,8 @@ async def run(self, channel: Channel, item: ExecItem) -> None: class GreenletExec: """Exec strategy: greenlets on a gevent hub owning the main thread. - ``execmodel=gevent``: each request runs as a greenlet spawned by the - integrate loop; the worker's ``wait=gevent`` wakeners make channel + ``profile=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's gevent wakeners make channel operations park the greenlet, so concurrent remote_execs cooperate on the one main thread. Requires gevent in the worker environment (provisioning adds the ``gevent`` requirement automatically). @@ -207,18 +192,17 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -# execmodel profile -> exec strategy for the sync-facade worker; the -# "trio" profile serves a plain AsyncGateway instead (TaskExec below). +# worker profile -> exec strategy for the sync-facade worker; the "trio" +# profile serves a plain AsyncGateway instead (TaskExec below). # Future placement strategies slot in here (e.g. subinterpreters). WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": HybridExec, - "main_thread_only": MainExec, "gevent": GreenletExec, } class TaskExec: - """Exec strategy for the pure-async profile (``execmodel=trio``). + """Exec strategy for the pure-async profile (``profile=trio``). Sources run as tasks on the worker's own loop, in the one and only thread of the process, and receive an ``AsyncChannel``. Sources must @@ -261,7 +245,7 @@ async def _run_exec(self, channel: Any, item: ExecItem) -> None: toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) if not toplevel_await and not call_name: await channel.aclose( - "sync source under execmodel=trio: the source must use" + "sync source under profile=trio: the source must use" " top-level await (or pass an async function)" ) return @@ -274,7 +258,7 @@ async def _run_exec(self, channel: Any, item: ExecItem) -> None: result = function(channel, **kwargs) if not hasattr(result, "__await__"): await channel.aclose( - f"sync function {call_name!r} under execmodel=trio:" + f"sync function {call_name!r} under profile=trio:" " remote_exec functions must be async" ) return @@ -299,7 +283,7 @@ class TrioWorkerExec: Exec requests flow through a single pump task so admission happens strictly in message-arrival order (trio task scheduling order is deliberately unordered, so per-request tasks would race for e.g. the - main_thread_only slot). Where an admitted request runs is the + main-thread claim). Where an admitted request runs is the strategy's business (:data:`WORKER_EXEC_STRATEGIES`). """ @@ -454,7 +438,7 @@ def _build_worker_gateway( strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] except KeyError: raise ValueError( - f"execmodel {model.backend!r} has no worker exec strategy " + f"profile {model.backend!r} has no worker exec strategy " f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" ) from None strategy = strategy_factory(gateway) @@ -462,10 +446,6 @@ def _build_worker_gateway( # Duck-type as the exec pool for STATUS / _terminate_execution. gateway._execpool = trio_exec gateway._trio_exec = trio_exec - gateway._executetask_complete = None - if strategy.needs_primary_thread: - gateway._executetask_complete = threading.Event() - gateway._executetask_complete.set() return gateway, trio_exec @@ -504,7 +484,7 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: async def _serve_async_worker(stream: Any, id: str) -> None: - """Serve a plain AsyncGateway with task-based exec (execmodel=trio). + """Serve a plain AsyncGateway with task-based exec (profile=trio). The whole worker is this one trio run on the process main thread: the dispatch loop and every exec'd source share it. Termination cancels @@ -523,7 +503,7 @@ async def _serve_async_worker(stream: Any, id: str) -> None: def serve_popen_async(id: str) -> None: - """Serve the pure-async worker over the stdio pipes (execmodel=trio).""" + """Serve the pure-async worker over the stdio pipes (profile=trio).""" from ._trio_gateway import staple_fd_stream read_fd, write_fd = _prepare_protocol_fds() @@ -548,11 +528,11 @@ async def main() -> None: os._exit(0) -def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: +def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host - model = get_execmodel(execmodel) + model = get_execmodel(profile) read_fd, write_fd = _prepare_protocol_fds() # Bootstrap handshake: the coordinator waits for this byte on our stdout # before starting the Message protocol. We are launched as a plain module @@ -566,7 +546,7 @@ def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") - def serve_socket_trio( - id: str, execmodel: str, socket_fd: int, wait: str = "thread" + id: str, profile: str, socket_fd: int, wait: str = "thread" ) -> None: """Serve a WorkerGateway over an inherited socket fd. @@ -576,7 +556,7 @@ def serve_socket_trio( """ from . import _trio_host - model = get_execmodel(execmodel) + model = get_execmodel(profile) host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_trio_host.adopt_socket, socket_fd) @@ -640,7 +620,7 @@ def _main() -> None: """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "execmodel", "coordinator_version"}``. With ``--socket-fd`` the + ``{"id", "profile", "wait", "coordinator_version"}``. With ``--socket-fd`` the worker serves over that inherited socket (socketserver); otherwise over the stdio pipes (popen / ssh). The worker imports execnet + trio from the environment; no source is sent over the wire to bootstrap it. @@ -658,25 +638,20 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) _apply_worker_setup(config) - if config["execmodel"] == "trio": + # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed + # coordinator still connects. + profile = config.get("profile") or config["execmodel"] + wait = config.get("wait", "thread") + if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: serve_socket_async(config["id"], ns.socket_fd) else: serve_popen_async(config["id"]) elif ns.socket_fd is not None: - serve_socket_trio( - config["id"], - config["execmodel"], - ns.socket_fd, - wait=config.get("wait", "thread"), - ) + serve_socket_trio(config["id"], profile, ns.socket_fd, wait=wait) else: - serve_popen_trio( - id=config["id"], - execmodel=config["execmodel"], - wait=config.get("wait", "thread"), - ) + serve_popen_trio(id=config["id"], profile=profile, wait=wait) if __name__ == "__main__": diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py index 0559ed8c..f13d0521 100644 --- a/src/execnet/_xspec.py +++ b/src/execnet/_xspec.py @@ -15,10 +15,15 @@ class XSpec: * If no "=value" is given, assume a boolean True value """ + #: keys accepted under an older spelling, mapped to the canonical one. + #: ``execmodel`` described a *local* execution model that no longer + #: exists; what the key actually selects is the worker's profile. + _ALIASES = {"execmodel": "profile"} + # XXX allow customization, for only allow specific key names chdir: str | None = None dont_write_bytecode: bool | None = None - execmodel: str | None = None + profile: str | None = None id: str | None = None installvia: str | None = None nice: str | None = None @@ -42,7 +47,9 @@ def __init__(self, string: str) -> None: key, value = keyvalue[:i], keyvalue[i + 1 :] if key[0] == "_": raise AttributeError("%r not a valid XSpec key" % key) - if key in self.__dict__: + # duplicates are checked on the canonical name, so + # ``execmodel=x//profile=y`` is rejected like any other clash + if self._ALIASES.get(key, key) in self.__dict__: raise ValueError(f"duplicate key: {key!r} in {string!r}") if key.startswith("env:"): self.env[key[4:]] = value @@ -54,6 +61,15 @@ def __getattr__(self, name: str) -> None | bool | str: raise AttributeError(name) return None + @property + def execmodel(self) -> str | None: + """Deprecated alias for :attr:`profile` (accepted indefinitely).""" + return self.profile + + @execmodel.setter + def execmodel(self, value: str | None) -> None: + self.profile = value + def __repr__(self) -> str: return f"" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index ba1eb35f..a82a1c0f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -24,7 +24,6 @@ # errors and error texts "sysex": "._errors", "INTERRUPT_TEXT": "._errors", - "MAIN_THREAD_ONLY_DEADLOCK_TEXT": "._errors", "GatewayReceivedTerminate": "._errors", "HostNotFound": "._errors", "geterrortext": "._errors", diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 37259bb7..1160b89c 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -19,6 +19,7 @@ from ._multi import default_group from ._multi import makegateway from ._multi import set_execmodel +from ._multi import set_profile from ._rsync import RSync from ._xspec import XSpec @@ -38,4 +39,5 @@ "default_group", "makegateway", "set_execmodel", + "set_profile", ] diff --git a/testing/conftest.py b/testing/conftest.py index de1e2866..a9797ada 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -179,46 +179,49 @@ def group() -> Iterator[execnet.Group]: @pytest.fixture def gw( request: pytest.FixtureRequest, - execmodel: ExecModel, + profile: str, group: execnet.Group, ) -> Gateway: try: return group[request.param] except KeyError: if request.param == "popen": - gw = group.makegateway("popen//id=popen//execmodel=%s" % execmodel.backend) + gw = group.makegateway("popen//id=popen//profile=%s" % profile) elif request.param == "socket": - # if execmodel.backend != "thread": - # pytest.xfail( - # "cannot set remote non-thread execmodel for sockets") pname = "sproxy1" if pname not in group: proxygw = group.makegateway("popen//id=%s" % pname) # assert group['proxygw'].remote_status().receiving gw = group.makegateway( - f"socket//id=socket//installvia={pname}//execmodel={execmodel.backend}" + f"socket//id=socket//installvia={pname}//profile={profile}" ) # TODO(typing): Clarify this assignment. gw.proxygw = proxygw # type: ignore[attr-defined] assert pname in group elif request.param == "ssh": sshhost = request.getfixturevalue("specssh").ssh - # we don't use execmodel.backend here - # but you can set it when specifying the ssh spec + # the profile is not forced here; set it in the ssh spec instead gw = group.makegateway(f"ssh={sshhost}//id=ssh") elif request.param == "proxy": group.makegateway("popen//id=proxy-transport") gw = group.makegateway( - "popen//via=proxy-transport//id=proxy//execmodel=%s" % execmodel.backend + "popen//via=proxy-transport//id=proxy//profile=%s" % profile ) else: - assert 0, f"unknown execmodel: {request.param}" + assert 0, f"unknown gateway type: {request.param}" return gw -@pytest.fixture(params=["thread", "main_thread_only"], scope="session") -def execmodel(request: pytest.FixtureRequest) -> ExecModel: - return get_execmodel(request.param) +@pytest.fixture(params=["thread"], scope="session") +def profile(request: pytest.FixtureRequest) -> str: + """The worker profile gateways in this test run are created with.""" + return request.param + + +@pytest.fixture(scope="session") +def execmodel(profile: str) -> ExecModel: + """The deprecated ExecModel shim for ``profile`` (pytest-xdist compat).""" + return get_execmodel(profile) @pytest.fixture diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 4e568d7e..2ef456ef 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -1,4 +1,4 @@ -"""The pure-async worker profile (execmodel=trio). +"""The pure-async worker profile (profile=trio). One single thread in the worker: the trio loop owns the main thread and exec'd sources run as tasks on it, talking through AsyncChannels. Sync @@ -21,7 +21,7 @@ @pytest.fixture def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: - return makegateway("popen//execmodel=trio") + return makegateway("popen//profile=trio") class TestSyncCoordinator: @@ -95,21 +95,37 @@ async def source(channel) -> None: channel.receive(TESTTIMEOUT) def test_status_and_rinfo(self, trio_gw: Gateway) -> None: - assert trio_gw.remote_status().execmodel == "trio" + status = trio_gw.remote_status() + assert status.profile == "trio" + # legacy STATUS key, kept for pytest-xdist + assert status.execmodel == "trio" rinfo = trio_gw._rinfo() assert rinfo.pid assert rinfo.version_info -def test_unknown_execmodel_rejected(makegateway: Callable[[str], Gateway]) -> None: - with pytest.raises(ValueError, match="unknown execmodel"): +def test_unknown_profile_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown profile"): + makegateway("popen//profile=nope") + # the pre-3.0 spelling routes to the same validation + with pytest.raises(ValueError, match="unknown profile"): makegateway("popen//execmodel=nope") +def test_execmodel_is_an_accepted_alias_for_profile( + makegateway: Callable[[str], Gateway], +) -> None: + gw = makegateway("popen//execmodel=trio") + assert gw.spec.profile == "trio" + assert gw.spec.execmodel == "trio" + with pytest.raises(ValueError, match="duplicate key"): + execnet.XSpec("popen//execmodel=trio//profile=thread") + + def test_trio_native_coordinator() -> None: async def main() -> None: async with execnet.trio.AsyncGroup() as group: - gateway = await group.makegateway("popen//execmodel=trio") + gateway = await group.makegateway("popen//profile=trio") channel = await gateway.remote_exec( "await channel.send(await channel.receive() * 2)" ) diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ef8e8aa9..e28a98eb 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -582,22 +582,15 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] -def test_assert_main_thread_only( - execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] +def test_sequential_remote_exec_claims_the_main_thread( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") - + # The `thread` profile hands each request the worker main thread while + # it is idle -- the GUI/signal-safety property `main_thread_only` + # existed for. Sequential remote_execs therefore all land there. + gw = makegateway("profile=thread//popen") try: - # Submit multiple remote_exec requests in quick succession and - # assert that all tasks execute in the main thread. It is - # necessary to call receive on each channel before the next - # remote_exec call, since the channel will raise an error if - # concurrent remote_exec requests are submitted as in - # test_main_thread_only_concurrent_remote_exec_deadlock. - for i in range(10): + for _ in range(10): ch = gw.remote_exec( """ import time, threading @@ -605,7 +598,6 @@ def test_assert_main_thread_only( channel.send(threading.current_thread() is threading.main_thread()) """ ) - try: res = ch.receive() finally: @@ -621,43 +613,29 @@ def test_assert_main_thread_only( gw.join() -def test_main_thread_only_concurrent_remote_exec_deadlock( - execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] +def test_main_thread_only_is_deprecated_and_overflows( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") + # `main_thread_only` now maps to `thread`. Where it used to refuse a + # second concurrent remote_exec with a deadlock error, the request now + # overflows to a pool thread -- the first one still gets the main + # thread, which is what the profile was for. + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway("profile=main_thread_only//popen") channels = [] try: - # Submit multiple remote_exec requests in quick succession and - # assert that MAIN_THREAD_ONLY_DEADLOCK_TEXT is raised if - # concurrent remote_exec requests are submitted for the - # main_thread_only execmodel (as compensation for the lack of - # back pressure in remote_exec calls which do not attempt to - # block until the remote main thread is idle). - for i in range(2): + for _ in range(2): channels.append( gw.remote_exec( """ import threading channel.send(threading.current_thread() is threading.main_thread()) - # Wait forever, ensuring that the deadlock case triggers. - threading.Event().wait() + channel.receive() """ ) ) - - expected_results = ( - True, - execnet._errors.MAIN_THREAD_ONLY_DEADLOCK_TEXT, - ) - for expected, ch in zip(expected_results, channels, strict=True): - try: - res = ch.receive() - except execnet.RemoteError as e: - res = e.formatted - assert res == expected + # both run: first on the main thread, second on an overflow thread + assert [ch.receive(TESTTIMEOUT) for ch in channels] == [True, False] finally: for ch in channels: ch.close() diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 49f47bb4..dc6f2d9d 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -109,6 +109,7 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: from execnet import _serialize assert execnet._XDIST_COMPAT == ("dumps",) + execnet._xdist_compat_warned.discard("dumps") with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): assert execnet.dumps is _serialize.dumps # reachable, but never advertised @@ -116,6 +117,18 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: assert "dumps" not in dir(execnet) +def test_dumps_shim_warns_only_once() -> None: + # xdist reaches this from serialize_warning_message, i.e. from inside + # pytest's warning-recording hook: warning every time makes recording + # one warning record another, ad infinitum, and the worker wedges. + execnet._xdist_compat_warned.discard("dumps") + with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): + execnet.dumps # noqa: B018 + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + execnet.dumps # noqa: B018 + + def test_boundary_kit_is_private() -> None: # There is no third-party event-loop extension point: the two wait # backends are threads and gevent, and every other concurrency library From e75cd0ad92b005e6591f2cc51c6a0731042c3447 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 12:38:53 +0200 Subject: [PATCH 63/66] feat!: share one Trio host, guard loop misuse, fix aio cancellation Three things that only make sense together, because they are all about the boundary between execnet and the caller's concurrency library. execnet.trio is the only surface that runs gateways directly, as tasks in the caller's own nursery. sync/aio/gevent have no loop to put them on, so they drive a Trio host thread -- and every Group used to spawn its own. A host is a thread and an event loop, not something groups need isolated from each other, so there is now one shared host per process, started lazily and stopped at exit, with an explicit execnet.Host for callers who want isolation or deterministic teardown. Driving a blocking surface from inside a running loop used to hang: the wait parks the loop's own thread. Blocking coordinator operations now detect a running asyncio or trio loop and raise, naming the namespace to use instead. Worker-side channels are deliberately exempt -- exec'd code may run its own loop and talk to its channel from inside it. execnet.aio lost data on cancellation: the bridge abandoned the asyncio future while the host task ran on, so `asyncio.timeout` around a receive consumed an item and dropped it -- on the most idiomatic asyncio code path. Each bridged call now owns a trio CancelScope that a cancelled await cancels, while send/send_eof/aclose/terminate are shielded so they cannot tear halfway. Its classes gain the Async prefix to match execnet.trio, AsyncGroup can be driven with start()/aclose() from lifespan hooks, and open_popen_gateway becomes open_gateway on both async surfaces (it always accepted any spec). Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 2 + src/execnet/_channel.py | 3 + src/execnet/_gateway.py | 1 + src/execnet/_gateway_base.py | 10 ++ src/execnet/_host.py | 184 ++++++++++++++++++++++++ src/execnet/_multi.py | 39 +++-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_host.py | 12 +- src/execnet/aio.py | 272 ++++++++++++++++++++++++----------- src/execnet/gevent.py | 2 + src/execnet/sync.py | 2 + src/execnet/trio.py | 4 +- testing/test_aio.py | 90 ++++++++++-- testing/test_basics.py | 3 + testing/test_host.py | 201 ++++++++++++++++++++++++++ testing/test_namespaces.py | 2 +- testing/test_trio_gateway.py | 12 +- 17 files changed, 714 insertions(+), 127 deletions(-) create mode 100644 src/execnet/_host.py create mode 100644 testing/test_host.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 315dd0bf..2ed1293f 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -32,6 +32,7 @@ from .sync import DumpError from .sync import Gateway from .sync import Group +from .sync import Host from .sync import HostNotFound from .sync import LoadError from .sync import MultiChannel @@ -50,6 +51,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index a13f846b..98c5d862 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -286,6 +286,7 @@ def waitclose(self, timeout: float | None = None) -> None: # For a callback channel wait on the consumer task finishing (so every # callback, including the endmarker, has run); otherwise wait for the # non-"opened" state directly. + self.gateway._check_event_loop("channel.waitclose()") signal = ( self._consumer_done if self._consumer_done is not None @@ -309,6 +310,7 @@ def send(self, item: object) -> None: """ if self.isclosed(): raise OSError(f"cannot send to {self!r}") + self.gateway._check_event_loop("channel.send()") self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) def receive(self, timeout: float | None = None) -> Any: @@ -325,6 +327,7 @@ def receive(self, timeout: float | None = None) -> Any: mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") + self.gateway._check_event_loop("channel.receive()") x = mailbox.get(timeout) if x is ENDMARKER: mailbox.put(x) # for other receivers diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index bbbb936c..8f992e09 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -31,6 +31,7 @@ class Gateway(BaseGateway): """Gateway to a local or remote Python Interpreter.""" _group: Group + _guard_event_loop = True def __init__(self, io: IO, spec: XSpec) -> None: """:private: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 4d0f762e..11920dae 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -45,6 +45,16 @@ class BaseGateway: #: write-acks, join). Inherited from the facade coordinator-side, and #: derived from the worker profile worker-side. _wait_backend: WaitBackend = "thread" + #: whether blocking operations refuse to run inside a foreign event + #: loop. Only coordinator-side: exec'd code in a worker may legitimately + #: run its own loop and talk to its channel from inside it. + _guard_event_loop = False + + def _check_event_loop(self, what: str) -> None: + if self._guard_event_loop: + from ._host import check_not_in_event_loop + + check_not_in_event_loop(what) def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel diff --git a/src/execnet/_host.py b/src/execnet/_host.py new file mode 100644 index 00000000..56af3b4f --- /dev/null +++ b/src/execnet/_host.py @@ -0,0 +1,184 @@ +"""The Trio host thread that the blocking surfaces drive. + +:mod:`execnet.trio` runs gateways *directly*, as tasks in the caller's own +nursery. Every other surface -- :mod:`execnet.sync`, :mod:`execnet.aio`, +:mod:`execnet.gevent` -- has no loop of its own to put them on, so protocol +IO runs on a :class:`Host`: one OS thread running ``trio.run``. + +There is one shared host per process by default, because a host is a +thread and a trio loop, not a resource groups need isolated from each +other. Pass an explicit :class:`Host` when you do want isolation or +deterministic teardown:: + + with execnet.Host() as host: + group = execnet.Group(host=host) + ... + # the thread is joined here, rather than at interpreter exit + +This module stays free of ``import trio`` so ``import execnet`` does not +load the event loop machinery: the real :class:`~execnet._trio_host.TrioHost` +is built on first use. +""" + +from __future__ import annotations + +import atexit +import os +import sys +import threading +from types import TracebackType +from typing import Any + +__all__ = ["Host", "check_not_in_event_loop", "default_host"] + +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 + + +def _running_event_loop() -> str | None: + """``"asyncio"`` / ``"trio"`` when called from inside one, else None. + + Both checks go through ``sys.modules`` first, so a program that never + imported asyncio or trio pays two dict lookups. asyncio is probed with + the private ``_get_running_loop`` because it *returns* None rather than + raising -- this sits in front of every blocking channel operation, and + building an exception per send is not free. + """ + asyncio = sys.modules.get("asyncio") + if asyncio is not None: + get_running = getattr(asyncio.events, "_get_running_loop", None) + if get_running is not None: + if get_running() is not None: + return "asyncio" + else: # pragma: no cover - every supported CPython has the private one + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + return "asyncio" + trio = sys.modules.get("trio") + if trio is not None: + try: + trio.lowlevel.current_trio_token() + except RuntimeError: + pass + else: + return "trio" + return None + + +#: which namespace to point a caller at, per detected loop +_SURFACE_FOR_LOOP = { + "asyncio": "execnet.aio (AsyncGroup)", + "trio": "execnet.trio (AsyncGroup)", +} + + +def check_not_in_event_loop(what: str) -> None: + """Raise if ``what`` is about to block a running event loop's thread. + + The blocking surfaces park the calling thread on a wakener, which + inside a running loop stalls every task on it -- usually as a hang, so + it is worth turning into an error that names the right namespace. + """ + loop = _running_event_loop() + if loop is None: + return + raise RuntimeError( + f"{what} blocks the calling thread and you are inside a running" + f" {loop} event loop, which would stall every task on it." + f" Use {_SURFACE_FOR_LOOP[loop]} instead, or run this in a" + " worker thread." + ) + + +class Host: + """A Trio event loop on a dedicated thread, shared by gateway groups. + + Starting is lazy: constructing a Host costs nothing, and the thread + comes up when a group first needs it. + """ + + def __init__( + self, + name: str = "execnet-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: + self.name = name + self.callback_threads = callback_threads + self._lock = threading.Lock() + self._trio_host: Any = None + + def __repr__(self) -> str: + state = "running" if self._trio_host is not None else "idle" + return f"" + + @property + def running(self) -> bool: + return self._trio_host is not None + + def _ensure_started(self) -> Any: + """The started :class:`~execnet._trio_host.TrioHost` (internal).""" + with self._lock: + if self._trio_host is None: + from . import _trio_host + + trio_host = _trio_host.TrioHost( + name=self.name, callback_threads=self.callback_threads + ) + trio_host.start() + self._trio_host = trio_host + return self._trio_host + + def close(self, timeout: float | None = 5.0) -> None: + """Stop the loop and join the thread (no-op when not running). + + Gateways served by this host must already be terminated; closing + does not terminate them for you. + """ + with self._lock: + trio_host, self._trio_host = self._trio_host, None + if trio_host is not None: + trio_host.stop(timeout=timeout) + + def __enter__(self) -> Host: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + +_default_lock = threading.Lock() +_default: Host | None = None +#: the pid _default was created in; a forked child inherits a dead thread +_default_pid: int | None = None + + +def default_host() -> Host: + """The process-wide host, started lazily and stopped at interpreter exit. + + After ``os.fork()`` the child inherits a Host whose thread does not + exist there, so the first use in a child builds a fresh one. + """ + global _default, _default_pid + with _default_lock: + pid = os.getpid() + if _default is None or _default_pid != pid: + _default = Host() + _default_pid = pid + return _default + + +def _close_default_atexit() -> None: + host = _default + if host is not None and _default_pid == os.getpid(): + host.close(timeout=1.0) + + +atexit.register(_close_default_atexit) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index ee751597..c739eedc 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -29,6 +29,9 @@ from ._execmodel import ExecModel from ._execmodel import get_execmodel from ._execmodel import resolve_profile +from ._host import Host +from ._host import check_not_in_event_loop +from ._host import default_host from ._trace import trace from ._xspec import XSpec @@ -55,13 +58,16 @@ def __init__( xspecs: Iterable[XSpec | str | None] = (), profile: str | None = None, *, + host: Host | None = None, execmodel: str | None = None, ) -> None: """Initialize a group and make gateways as specified. ``profile`` is the default worker profile for gateways created - without an explicit ``profile=`` in their spec. ``execmodel`` is - the deprecated spelling (pytest-xdist still passes it). + without an explicit ``profile=`` in their spec. ``host`` is the + Trio host thread to serve this group's protocol IO on; it defaults + to the process-wide one. ``execmodel`` is the deprecated spelling + of ``profile`` (pytest-xdist still passes it). """ if execmodel is not None: if profile is not None: @@ -78,20 +84,20 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] - self._trio_host: Any = None + self._host = default_host() if host is None else host self._async_group: Any = None self.set_profile("thread" if profile is None else profile) for xspec in xspecs: self.makegateway(xspec) atexit.register(self._cleanup_atexit) - def _ensure_trio_host(self) -> Any: - if self._trio_host is None: - from . import _trio_host + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host - self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") - self._trio_host.start() - return self._trio_host + def _ensure_trio_host(self) -> Any: + return self._host._ensure_started() def _ensure_async_group(self) -> Any: """The FacadeAsyncGroup owning the async side, running on the host.""" @@ -218,6 +224,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: If no spec is given, self.defaultspec is used. """ + check_not_in_event_loop("Group.makegateway()") if not spec: spec = self.defaultspec if not isinstance(spec, XSpec): @@ -260,15 +267,16 @@ def _unregister(self, gateway: Gateway) -> None: self._gateways_to_join.append(gateway) def _cleanup_atexit(self) -> None: + # The host is shared and stops itself at exit; a group only owns + # its gateways and the async group task running on that host. trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) if self._async_group is not None: with suppress(Exception): - self._trio_host.call_sync(self._async_group.shutdown.set) + self._host._ensure_started().call_sync( + self._async_group.shutdown.set + ) self._async_group = None - if self._trio_host is not None: - self._trio_host.stop(timeout=1.0) - self._trio_host = None def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination @@ -307,12 +315,13 @@ def _host_terminate(self, timeout: float | None) -> None: a OneShot instead of blocking the OS thread (which would stall the hub for the whole grace). """ + trio_host = self._host._ensure_started() if self._wait_backend == "thread": - self._trio_host.call(self._async_group.terminate, timeout) + trio_host.call(self._async_group.terminate, timeout) return from ._boundary import make_wakener - self._trio_host.call_pending( + trio_host.call_pending( self._async_group.terminate, timeout, wakener=make_wakener(self._wait_backend), diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index db60920e..b1fccbf5 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1033,7 +1033,7 @@ async def _terminate_one( @asynccontextmanager -async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: +async def open_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: """Spawn one popen worker and serve an AsyncGateway over its stdio. Runs inside the caller's own trio run -- no host thread involved. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 82ad9c37..ed5ee8f0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -46,11 +46,10 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args +from ._host import DEFAULT_CALLBACK_THREADS from ._portal import LoopPortal from ._portal import OneShot -#: default cap on concurrent threadpool threads running receiver callbacks -DEFAULT_CALLBACK_THREADS = 40 #: bound on how long the endmarker callback may run during host shutdown CONSUMER_ENDMARKER_GRACE = 10.0 @@ -550,8 +549,13 @@ def is_alive(self) -> bool: class TrioHost: """Dedicated OS thread running ``trio.run`` for protocol IO.""" - def __init__(self, name: str = "execnet-trio-host") -> None: + def __init__( + self, + name: str = "execnet-trio-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: self._name = name + self._callback_threads = callback_threads self._thread: threading.Thread | None = None self._portal: LoopPortal | None = None self._nursery: trio.Nursery | None = None @@ -591,7 +595,7 @@ def _run(self) -> None: async def _main(self) -> None: self._portal = LoopPortal() self._shutdown = trio.Event() - self._callback_limiter = trio.CapacityLimiter(DEFAULT_CALLBACK_THREADS) + self._callback_limiter = trio.CapacityLimiter(self._callback_threads) try: async with trio.open_nursery() as nursery: self._nursery = nursery diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 36bd8751..3e3deee6 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -1,4 +1,4 @@ -"""The asyncio-native execnet API, bridged over a Trio host thread. +"""The asyncio-native execnet API. Everything here is awaited inside your own asyncio event loop:: @@ -6,22 +6,25 @@ import execnet.aio async def main(): - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(6 * 7)") print(await channel.receive()) asyncio.run(main()) -Protocol IO keeps running on a dedicated Trio host thread (the same -engine as the blocking and trio-native APIs, all transports included); -each awaited operation runs as a task on that host and resolves an -asyncio future via ``loop.call_soon_threadsafe``. No anyio port and no -executor threads per call. +Protocol IO keeps running on a Trio host thread (the same engine as the +blocking and trio-native APIs, all transports included); each awaited +operation runs as a task on that host and resolves an asyncio future via +``loop.call_soon_threadsafe``. No anyio port and no executor threads per +call. -Note: cancelling a bridged await abandons the operation on the asyncio -side only -- the host-side task runs to completion (a cancelled -``receive`` may still consume the next item). +Cancellation crosses the bridge. Cancelling an awaited ``receive`` (say +by ``asyncio.timeout``) cancels the host-side operation too, so no item +is consumed and dropped. Operations that must not tear halfway -- +``send``, ``send_eof``, ``aclose``, ``terminate`` -- are shielded +instead: the ``CancelledError`` reaches you, but the operation still +completes on the host. The error types are shared with :mod:`execnet.sync` and :mod:`execnet.trio`. Items you send must already be simple builtin data @@ -52,41 +55,59 @@ async def main(): from ._errors import LoadError from ._errors import RemoteError from ._errors import TimeoutError -from ._trio_gateway import AsyncChannel -from ._trio_gateway import AsyncGateway -from ._trio_gateway import AsyncGroup -from ._trio_host import TrioHost +from ._host import Host +from ._host import default_host +from ._trio_gateway import AsyncChannel as _TrioChannel +from ._trio_gateway import AsyncGateway as _TrioGateway +from ._trio_gateway import AsyncGroup as _TrioGroup from ._xspec import XSpec if TYPE_CHECKING: from typing_extensions import Self __all__ = [ - "Channel", + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", "DataFormatError", "DumpError", - "Gateway", - "Group", + "Host", "HostNotFound", "LoadError", "RemoteError", "TimeoutError", "XSpec", - "open_popen_gateway", + "default_host", + "open_gateway", ] T = TypeVar("T") class _HostBridge: - """Await trio-native coroutines on a TrioHost from asyncio.""" + """Await trio-native coroutines on a Trio host from asyncio.""" - def __init__(self, host: TrioHost) -> None: - self._host = host + def __init__(self, trio_host: Any) -> None: + self._host = trio_host - async def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + async def call( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + shield: bool = False, + ) -> T: + """Run ``async_fn`` on the host and await its result. + + Unless ``shield``, cancelling the await cancels the host-side + operation too, so a cancelled ``receive`` does not consume an item + that nobody will ever see. + """ loop = asyncio.get_running_loop() future: asyncio.Future[T] = loop.create_future() + # Built here rather than inside the task: the cancel post can + # otherwise overtake the task's first step, and cancelling a scope + # nobody has entered yet still cancels it once entered. + scope = trio.CancelScope() def resolve(result: Any, error: BaseException | None) -> None: if future.cancelled(): @@ -104,15 +125,19 @@ def post_result(result: Any, error: BaseException | None) -> None: async def runner() -> None: try: - result = await async_fn(*args) + with scope: + result = await async_fn(*args) except trio.Cancelled: # host shutdown: the nursery cancel must propagate post_result(None, RuntimeError("execnet aio host was shut down")) raise except BaseException as exc: post_result(None, exc) - else: - post_result(result, None) + return + if scope.cancelled_caught: + # cancelled by us -- the awaiter is already gone + return + post_result(result, None) def spawn() -> None: self._host.start_soon(runner) @@ -121,10 +146,20 @@ def spawn() -> None: self._host.portal.post(spawn) except trio.RunFinishedError: raise RuntimeError("execnet aio group is not running") from None - return await future + + if shield: + # The host-side work runs to completion either way; shielding + # keeps a cancelled caller from abandoning a half-done send. + return await asyncio.shield(future) + try: + return await future + except asyncio.CancelledError: + with suppress(trio.RunFinishedError): + self._host.portal.post(scope.cancel) + raise -class _HostedGroup(AsyncGroup): +class _HostedGroup(_TrioGroup): """Trio AsyncGroup living as a task on the host nursery.""" def __init__(self, termination_timeout: float) -> None: @@ -141,13 +176,13 @@ async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: self.finished.set() -class Channel: - """asyncio facade over a trio-native :class:`AsyncChannel`.""" +class AsyncChannel: + """asyncio facade over a trio-native channel.""" RemoteError = RemoteError TimeoutError = TimeoutError - def __init__(self, bridge: _HostBridge, channel: AsyncChannel) -> None: + def __init__(self, bridge: _HostBridge, channel: _TrioChannel) -> None: self._bridge = bridge self._channel = channel @@ -156,41 +191,49 @@ def id(self) -> int: return self._channel.id def __repr__(self) -> str: - return f"" + return f"" def isclosed(self) -> bool: """Return True if the channel is closed for sending.""" return self._channel.isclosed() async def send(self, item: object) -> None: - """Serialize ``item`` and send it to the other side.""" - await self._bridge.call(self._channel.send, item) + """Serialize ``item`` and send it to the other side. + + Shielded: cancelling raises in the caller but the item is still + sent, rather than leaving a half-written frame on the wire. + """ + await self._bridge.call(self._channel.send, item, shield=True) async def receive(self, timeout: float | None = None) -> Any: """Receive the next item sent from the other side. EOFError once the peer closed or sent EOF, RemoteError for a peer close-with-error, TimeoutError after ``timeout`` seconds. A - received channel reference arrives as an :class:`~execnet.aio.Channel`. + received channel reference arrives as an + :class:`~execnet.aio.AsyncChannel`. + + Cancellable: no item is consumed if the await is cancelled, so + ``asyncio.timeout`` is equivalent to passing ``timeout``. """ result = await self._bridge.call(self._channel.receive, timeout) - if isinstance(result, AsyncChannel): - return Channel(self._bridge, result) + if isinstance(result, _TrioChannel): + return AsyncChannel(self._bridge, result) return result async def send_eof(self) -> None: """Signal that no more items follow (peer keeps its send side).""" - await self._bridge.call(self._channel.send_eof) + await self._bridge.call(self._channel.send_eof, shield=True) async def aclose(self, error: str | None = None) -> None: """Close the channel; ``error`` reaches the peer as a RemoteError.""" - await self._bridge.call(self._channel.aclose, error) + await self._bridge.call(self._channel.aclose, error, shield=True) async def wait_closed(self) -> None: """Wait until the peer closed or sent EOF; reraise remote errors.""" await self._bridge.call(self._channel.wait_closed) - def __aiter__(self) -> Channel: + def __aiter__(self) -> AsyncChannel: return self async def __anext__(self) -> Any: @@ -200,10 +243,10 @@ async def __anext__(self) -> Any: raise StopAsyncIteration from None -class Gateway: - """asyncio facade over a trio-native :class:`AsyncGateway`.""" +class AsyncGateway: + """asyncio facade over a trio-native gateway.""" - def __init__(self, bridge: _HostBridge, gateway: AsyncGateway) -> None: + def __init__(self, bridge: _HostBridge, gateway: _TrioGateway) -> None: self._bridge = bridge self._gateway = gateway @@ -216,13 +259,13 @@ def remoteaddress(self) -> str | None: return self._gateway.remoteaddress def __repr__(self) -> str: - return f"" + return f"" async def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, **kwargs: object, - ) -> Channel: + ) -> AsyncChannel: """Connect a new channel to remote execution of ``source``. Accepts the same source kinds as ``Gateway.remote_exec``: a source @@ -232,86 +275,141 @@ async def remote_exec( channel = await self._bridge.call( functools.partial(self._gateway.remote_exec, source, **kwargs) ) - return Channel(self._bridge, channel) + return AsyncChannel(self._bridge, channel) async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" - await self._bridge.call(self._gateway.terminate) + await self._bridge.call(self._gateway.terminate, shield=True) -class Group: - """asyncio-native gateway group over a dedicated Trio host thread. +class AsyncGroup: + """asyncio-native gateway group served on a Trio host thread. - An async context manager mirroring :class:`execnet.trio.AsyncGroup`; - leaving the ``async with`` block terminates every gateway with the - same bounded contract, then stops the host thread. + Usable as an async context manager, or driven explicitly with + :meth:`start` / :meth:`aclose` from application lifespan hooks. + Either way, shutting down terminates every gateway with the same + bounded contract as :class:`execnet.trio.AsyncGroup`. """ - def __init__(self, termination_timeout: float = 10.0) -> None: + def __init__( + self, + termination_timeout: float = 10.0, + *, + host: Host | None = None, + ) -> None: self._termination_timeout = termination_timeout - self._host: TrioHost | None = None + self._host = default_host() if host is None else host self._bridge: _HostBridge | None = None self._group: _HostedGroup | None = None def __repr__(self) -> str: state = "running" if self._group is not None else "idle" - return f"" + return f"" - async def __aenter__(self) -> Self: - assert self._host is None, "group already entered" - loop = asyncio.get_running_loop() - host = TrioHost(name="execnet-aio-group") - # start() blocks until the host loop is ready; keep it off the - # asyncio loop thread. - await loop.run_in_executor(None, host.start) - self._host = host - self._bridge = _HostBridge(host) + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host + + async def start(self) -> None: + """Bring the host up and start the group task on it.""" + if self._group is not None: + raise RuntimeError(f"{self!r} is already started") + trio_host = await _start_host(self._host) + bridge = _HostBridge(trio_host) async def start_group() -> _HostedGroup: # runs on the host loop group = _HostedGroup(self._termination_timeout) - assert host._nursery is not None - started: _HostedGroup = await host._nursery.start(group.run) + assert trio_host._nursery is not None + started: _HostedGroup = await trio_host._nursery.start(group.run) return started - self._group = await self._bridge.call(start_group) - return self + self._bridge = bridge + self._group = await bridge.call(start_group, shield=True) - async def __aexit__(self, *exc_info: object) -> None: - group, bridge, host = self._group, self._bridge, self._host + async def aclose(self) -> None: + """Terminate every gateway and stop the group task (idempotent). + + The host thread is shared, so it keeps running for other groups. + """ + group, bridge = self._group, self._bridge self._group = self._bridge = None - if group is not None and bridge is not None: + if group is None or bridge is None: + return - async def stop_group() -> None: - group.shutdown.set() - await group.finished.wait() + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() - with suppress(RuntimeError): - await bridge.call(stop_group) - if host is not None: - self._host = None - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, functools.partial(host.stop, 5.0)) + with suppress(RuntimeError): + await bridge.call(stop_group, shield=True) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *exc_info: object) -> None: + await self.aclose() - async def makegateway(self, spec: str | XSpec = "popen") -> Gateway: + async def makegateway(self, spec: str | XSpec = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's host. All transports are supported: popen (including uv-provisioned ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with - ``installvia=``), and ``via=`` sub-gateways. + ``installvia=``), and ``via=`` sub-gateways. The worker profile + defaults to ``thread``; pass ``profile=trio`` for a worker that + runs exec'd async sources as tasks. """ group, bridge = self._group, self._bridge if group is None or bridge is None: - raise RuntimeError(f"{self!r} is not entered") + raise RuntimeError(f"{self!r} is not started") gateway = await bridge.call(group.makegateway, spec) - return Gateway(bridge, gateway) + return AsyncGateway(bridge, gateway) + + +async def _start_host(host: Host) -> Any: + """Start ``host`` without blocking the asyncio loop or its executor. + + ``Host._ensure_started`` blocks until the trio loop is ready, so it + runs on a throwaway thread whose completion is posted back to the + loop -- never on the default executor, which belongs to the caller's + application. + """ + if host.running: + return host._ensure_started() + import threading + + loop = asyncio.get_running_loop() + future: asyncio.Future[Any] = loop.create_future() + + def start() -> None: + try: + trio_host = host._ensure_started() + except BaseException as exc: # noqa: BLE001 + loop.call_soon_threadsafe(_set_future_error, future, exc) + else: + loop.call_soon_threadsafe(_set_future_result, future, trio_host) + + threading.Thread(target=start, name="execnet-host-start", daemon=True).start() + return await future + + +def _set_future_result(future: asyncio.Future[Any], value: Any) -> None: + if not future.cancelled(): + future.set_result(value) + + +def _set_future_error(future: asyncio.Future[Any], error: BaseException) -> None: + if not future.cancelled(): + future.set_exception(error) @asynccontextmanager -async def open_popen_gateway(spec: str | XSpec = "popen") -> AsyncIterator[Gateway]: - """Spawn one popen worker and yield an asyncio Gateway to it. +async def open_gateway(spec: str | XSpec = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one worker for ``spec`` and yield an asyncio gateway to it. - Convenience for a single-gateway :class:`Group`. + Convenience for a single-gateway :class:`AsyncGroup`. """ - async with Group() as group: + async with AsyncGroup() as group: yield await group.makegateway(spec) diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py index f3f12609..df5aadfb 100644 --- a/src/execnet/gevent.py +++ b/src/execnet/gevent.py @@ -36,6 +36,7 @@ from ._xspec import XSpec from .sync import Channel from .sync import Gateway +from .sync import Host from .sync import RSync __all__ = [ @@ -44,6 +45,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 1160b89c..e32951d8 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -14,6 +14,7 @@ from ._errors import RemoteError from ._errors import TimeoutError from ._gateway import Gateway +from ._host import Host from ._multi import Group from ._multi import MultiChannel from ._multi import default_group @@ -29,6 +30,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/trio.py b/src/execnet/trio.py index 9704b6ef..0fc3b5df 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -29,7 +29,7 @@ async def main(): from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup -from ._trio_gateway import open_popen_gateway +from ._trio_gateway import open_gateway from ._xspec import XSpec __all__ = [ @@ -43,5 +43,5 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "open_popen_gateway", + "open_gateway", ] diff --git a/testing/test_aio.py b/testing/test_aio.py index 6bdb7724..8c51ecf2 100644 --- a/testing/test_aio.py +++ b/testing/test_aio.py @@ -22,7 +22,7 @@ def run(main: Awaitable[T]) -> T: def test_popen_roundtrip() -> None: async def main() -> None: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") await channel.send(41) @@ -32,9 +32,9 @@ async def main() -> None: run(main()) -def test_open_popen_gateway_iteration() -> None: +def test_open_gateway_iteration() -> None: async def main() -> list[int]: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec( "for i in range(4): channel.send(i * 2)" ) @@ -45,7 +45,7 @@ async def main() -> list[int]: def test_receive_timeout() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec("channel.receive()") with pytest.raises(channel.TimeoutError): await channel.receive(timeout=0.05) @@ -56,7 +56,7 @@ async def main() -> None: def test_remote_error() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec("raise ValueError(17)") with pytest.raises(execnet.aio.RemoteError, match="ValueError"): await channel.receive() @@ -66,7 +66,7 @@ async def main() -> None: def test_channel_passing_wraps_aio() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec( """ c = channel.gateway.newchannel() @@ -75,7 +75,7 @@ async def main() -> None: """ ) passed = await channel.receive() - assert isinstance(passed, execnet.aio.Channel) + assert isinstance(passed, execnet.aio.AsyncChannel) assert await passed.receive() == 42 run(main()) @@ -83,7 +83,7 @@ async def main() -> None: def test_multiple_gateways_and_send_each() -> None: async def main() -> list[Any]: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateways = [await group.makegateway("popen") for _ in range(2)] channels = [ await gw.remote_exec("channel.send(channel.receive() * 2)") @@ -98,8 +98,8 @@ async def main() -> list[Any]: def test_group_not_entered() -> None: async def main() -> None: - group = execnet.aio.Group() - with pytest.raises(RuntimeError, match="not entered"): + group = execnet.aio.AsyncGroup() + with pytest.raises(RuntimeError, match="not started"): await group.makegateway("popen") run(main()) @@ -107,10 +107,78 @@ async def main() -> None: def test_terminate_gateway_explicitly() -> None: async def main() -> None: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(1)") assert await channel.receive() == 1 await gateway.terminate() run(main()) + + +def test_cancelled_receive_does_not_consume_an_item() -> None: + # The bridge cancels the host-side receive, so the item stays queued + # instead of being consumed and dropped -- asyncio.timeout around a + # receive must behave like passing timeout=. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + """ + channel.receive() + for i in range(3): + channel.send(i) + """ + ) + with pytest.raises(asyncio.TimeoutError): + async with asyncio.timeout(0.05): + await channel.receive() + # release the worker; nothing was consumed by the cancelled wait + await channel.send("go") + assert [await channel.receive() for _ in range(3)] == [0, 1, 2] + + run(main()) + + +def test_cancelled_send_still_arrives() -> None: + # send is shielded: the caller sees CancelledError but the item is on + # the wire, rather than a frame half-written to the peer. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 2)" + ) + task = asyncio.ensure_future(channel.send(21)) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await channel.receive() == 42 + + run(main()) + + +def test_group_start_and_aclose_explicitly() -> None: + # asyncio apps drive this from lifespan hooks rather than "async with" + async def main() -> None: + group = execnet.aio.AsyncGroup() + await group.start() + try: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(7)") + assert await channel.receive() == 7 + finally: + await group.aclose() + await group.aclose() # idempotent + with pytest.raises(RuntimeError, match="not started"): + await group.makegateway("popen") + + run(main()) + + +def test_groups_share_the_default_host() -> None: + async def main() -> None: + async with execnet.aio.AsyncGroup() as a, execnet.aio.AsyncGroup() as b: + assert a.host is b.host + assert a.host is execnet.aio.default_host() + + run(main()) diff --git a/testing/test_basics.py b/testing/test_basics.py index a81ed1f6..0afcdd0d 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -346,6 +346,9 @@ class FakeGateway: _trio_session = None _new_wakener = staticmethod(_boundary.ThreadWakener) + def _check_event_loop(self, what: str) -> None: + pass + def _trace(self, *args) -> None: pass diff --git a/testing/test_host.py b/testing/test_host.py new file mode 100644 index 00000000..686c022f --- /dev/null +++ b/testing/test_host.py @@ -0,0 +1,201 @@ +"""The Trio host thread: sharing, explicit override, and loop-misuse guards. + +``execnet.trio`` runs gateways directly in the caller's own nursery; every +other surface drives a :class:`execnet.Host`. One is shared per process, +and blocking on it from inside a running event loop is an error rather +than a hang. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import threading + +import pytest +import trio + +import execnet +from execnet._host import Host +from execnet._host import default_host + +TESTTIMEOUT = 30.0 + + +def host_thread_names() -> list[str]: + return [t.name for t in threading.enumerate() if t.name.startswith("execnet-host")] + + +class TestSharedHost: + def test_groups_share_the_default_host(self) -> None: + a = execnet.Group() + b = execnet.Group() + assert a.host is b.host is default_host() + + def test_many_groups_run_one_thread(self) -> None: + groups = [execnet.Group() for _ in range(3)] + try: + for group in groups: + group.makegateway("popen") + assert len(host_thread_names()) == 1 + for group in groups: + channel = group[0].remote_exec("channel.send(1)") + assert channel.receive(TESTTIMEOUT) == 1 + finally: + for group in groups: + group.terminate(timeout=5.0) + + def test_explicit_host_is_isolated_and_closes(self) -> None: + host = Host(name="execnet-host-isolated") + assert not host.running + group = execnet.Group(host=host) + assert group.host is host + assert group.host is not default_host() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + assert host.running + assert "execnet-host-isolated" in host_thread_names() + finally: + group.terminate(timeout=5.0) + host.close() + assert not host.running + assert "execnet-host-isolated" not in host_thread_names() + + def test_host_context_manager_closes(self) -> None: + with Host(name="execnet-host-ctx") as host: + group = execnet.Group(host=host) + group.makegateway("popen") + group.terminate(timeout=5.0) + assert not host.running + + def test_starting_is_lazy(self) -> None: + host = Host(name="execnet-host-lazy") + execnet.Group(host=host) + # constructing a group must not cost a thread + assert not host.running + assert "execnet-host-lazy" not in host_thread_names() + + @pytest.mark.skipif( + not hasattr(os, "fork"), reason="requires os.fork" + ) + def test_forked_child_gets_a_fresh_default_host(self) -> None: + # the child inherits a Host object whose thread does not exist + # there, so the first use must build a new one + default_host()._ensure_started() + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # pragma: no cover - runs in the child + code = 1 + try: + os.close(read_fd) + child_host = default_host() + group = execnet.Group() + gateway = group.makegateway("popen") + got = gateway.remote_exec("channel.send(3)").receive(TESTTIMEOUT) + group.terminate(timeout=5.0) + code = 0 if (got == 3 and child_host.running) else 1 + finally: + os.write(write_fd, bytes([code])) + os._exit(0) + os.close(write_fd) + try: + result = os.read(read_fd, 1) + finally: + os.close(read_fd) + os.waitpid(pid, 0) + assert result == b"\x00" + + +class TestEventLoopGuard: + """Blocking on the host from inside a running loop must not hang.""" + + def test_makegateway_inside_asyncio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.aio"): + execnet.Group().makegateway("popen") + + asyncio.run(main()) + + def test_makegateway_inside_trio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.trio"): + execnet.Group().makegateway("popen") + + trio.run(main) + + def test_channel_receive_inside_asyncio_raises(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(1)") + + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.receive(TESTTIMEOUT) + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.send(1) + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.waitclose(TESTTIMEOUT) + + asyncio.run(main()) + # still usable from a plain thread afterwards + assert channel.receive(TESTTIMEOUT) == 1 + finally: + group.terminate(timeout=5.0) + + def test_worker_channels_are_not_guarded(self) -> None: + # exec'd code may run its own event loop and talk to its channel + # from inside it -- that is the caller's own loop to block + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec( + """ + import asyncio + + async def main(): + channel.send(channel.receive() + 1) + + asyncio.run(main()) + """ + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + + def test_a_worker_thread_is_still_fine(self) -> None: + group = execnet.Group() + result: list[object] = [] + + def work() -> None: + gateway = group.makegateway("popen") + result.append(gateway.remote_exec("channel.send(5)").receive(TESTTIMEOUT)) + + async def main() -> None: + # inside a loop, but the blocking call happens off it + await asyncio.to_thread(work) + + try: + asyncio.run(main()) + assert result == [5] + finally: + group.terminate(timeout=5.0) + + def test_no_asyncio_import_no_cost(self) -> None: + # the probe must not import asyncio/trio into a program that has + # neither; it goes through sys.modules first + code = ( + "import sys, execnet;" + " execnet._host.check_not_in_event_loop('x');" + " print('asyncio' in sys.modules, 'trio' in sys.modules)" + ) + import subprocess + + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.split() == ["False", "False"] diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index dc6f2d9d..856268ea 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -80,7 +80,7 @@ def test_trio_namespace_exposes_async_core() -> None: assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel - assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway + assert execnet.trio.open_gateway is _trio_gateway.open_gateway # error types are shared with the sync surface; the standalone serializer # is intentionally not exposed on any public namespace assert execnet.trio.RemoteError is execnet.RemoteError diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 35a62c50..9473df89 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -22,7 +22,7 @@ from execnet._serialize import loads_internal from execnet._trio_gateway import AsyncGateway from execnet._trio_gateway import AsyncGroup -from execnet._trio_gateway import open_popen_gateway +from execnet._trio_gateway import open_gateway @asynccontextmanager @@ -284,7 +284,7 @@ class TestPopenAsyncGateway: def test_remote_exec_roundtrip(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec( "channel.send(channel.receive() + 1)" ) @@ -296,7 +296,7 @@ async def main() -> None: def test_remote_exec_function_with_kwargs(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec(_remote_add, a=40, b=2) assert await channel.receive() == 42 @@ -304,7 +304,7 @@ async def main() -> None: def test_remote_error_propagates(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec("raise ValueError('kaboom')") with pytest.raises(RemoteError, match="kaboom"): await channel.receive() @@ -313,7 +313,7 @@ async def main() -> None: def test_exec_finish_closes_channel_ending_iteration(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec( "for i in range(3): channel.send(i)" ) @@ -323,7 +323,7 @@ async def main() -> None: def test_concurrent_remote_execs(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: results = [] async def run_one(value: int) -> None: From ddce312abf368cbb1141e26dab6e84d7e123731e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 16:49:46 +0200 Subject: [PATCH 64/66] docs: record the surface review in the changelog and handoffs Adds the changelog entries for the four namespaces, the shared Host, the loop guards, profile=/main_thread_only and the aio cancellation fix, and corrects the two earlier entries that still advertised execnet.portal. The phase C handoff gains a "Surface review" section that wins over everything below it, and the boundary-protocol handoff is marked superseded where it promised a Wakener extension point. Also drops the EXECMODEL_PROFILES entry from the gateway_base shim: that name was added on this branch, so it never existed pre-Trio and nothing can be forwarding to it. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 61 ++++++++++++++++++++-- handoff-boundary-protocol-rethink.md | 20 +++++-- handoff-phase-c-worker-axes.md | 78 ++++++++++++++++++++++++---- src/execnet/_execmodel.py | 4 +- src/execnet/_host.py | 6 ++- src/execnet/_multi.py | 8 +-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_host.py | 6 +-- src/execnet/_trio_worker.py | 22 +++++--- src/execnet/aio.py | 2 +- src/execnet/gateway_base.py | 1 - testing/conftest.py | 3 +- testing/test_aio.py | 9 ++-- testing/test_execmodel_trio.py | 19 +++++++ testing/test_gateway.py | 1 - testing/test_host.py | 16 +++--- 16 files changed, 203 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0a966560..93a52284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,61 @@ 2.2.0 (UNRELEASED) ------------------ +* One namespace per concurrency library you drive execnet from: + ``execnet.sync`` (plain threads; the top-level ``execnet.*`` aliases), + ``execnet.trio``, ``execnet.aio`` and the new ``execnet.gevent``, whose blocking + waits park the calling greenlet instead of its OS thread (needs ``execnet[gevent]``). + + ``execnet.trio`` is the only surface that runs gateways *directly*, as tasks in your + own nursery. The others drive a Trio host thread, so their blocking calls now raise + when made from inside a running asyncio or trio loop -- naming the namespace to use + instead -- rather than stalling that loop. Channels inside a worker are exempt: + exec'd code may run its own event loop and talk to its channel from within it. +* The ``execnet.portal`` namespace is gone. It exposed the ``Wakener`` protocol and the + ``Mailbox``/``OneShot``/``LoopPortal`` primitives but not the registry needed to plug + a ``Wakener`` in, and execnet does not offer third-party event-loop integration: a new + concurrency library gets a namespace of its own, as gevent just did. The primitives + are internal again. +* Gateway groups share one Trio host thread per process instead of starting one each. + Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an + isolated loop with deterministic teardown -- ``Host`` is a context manager and joins + its thread on exit, where the shared one stops at interpreter exit. +* The ``execmodel=`` spec key is now ``profile=``; ``execmodel=`` remains an accepted + alias. It always selected the *worker* profile -- where exec'd code runs relative to + the worker's protocol loop -- while the local execution model it was named after no + longer exists. Accordingly ``Group(execmodel=...)``, ``Group.set_execmodel()``, + ``Group.execmodel`` and ``Group.remote_execmodel`` are deprecated in favour of + ``Group(profile=...)``, ``Group.set_profile()`` and ``Group.profile``; only the + remote default they set has any effect. ``remote_status()`` reports both + ``profile`` and ``execmodel``. +* The ``main_thread_only`` profile is deprecated and now behaves like ``thread``, which + already hands the first ``remote_exec`` the worker's real main thread -- the + GUI/signal-safety property it was added for in 2.1.0. Its other behaviour is gone: + a second concurrent ``remote_exec`` used to close the channel with + ``concurrent remote_exec would cause deadlock``, and now runs on a pool thread. That + guard was a one-second timeout that reported a merely slow predecessor as a deadlock. +* The ``wait=`` spec key added earlier in this release cycle is gone. Which primitive a + blocking wait parks on describes the *caller*, which is what choosing a namespace + already says; a worker's own backend is derived from its profile. +* ``execnet.aio`` now propagates cancellation. Cancelling an awaited ``receive`` (with + ``asyncio.timeout``, say) cancels the host-side operation, where it previously + abandoned only the asyncio side and let the operation consume an item that was then + discarded. ``send``, ``send_eof``, ``aclose`` and ``terminate`` are shielded instead, + so they cannot tear halfway. + + Its classes gained the ``Async`` prefix -- ``AsyncGroup``, ``AsyncGateway``, + ``AsyncChannel`` -- matching ``execnet.trio``, and ``AsyncGroup`` can be driven with + ``start()``/``aclose()`` from application lifespan hooks instead of ``async with``. + ``open_popen_gateway`` is renamed ``open_gateway`` on both async namespaces, since it + always accepted any spec. +* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, now warns once per + process rather than on every access. xdist reaches it from ``serialize_warning_message``, + i.e. from inside pytest's warning-recording hook, so a single ``DeprecationWarning`` + raised in a worker made recording that warning record another, unbounded, wedging the + run. +* ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing + to stdout. It has been a no-operation since execnet 1.2. + * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. @@ -31,7 +86,8 @@ load. Opcode bytes are unchanged for every type that survives. * The supported API is now exactly five namespaces: ``execnet`` (aliases of ``execnet.sync``), ``execnet.sync``, ``execnet.trio``, ``execnet.aio`` and - ``execnet.portal``. The pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, + ``execnet.gevent`` -- one per concurrency library you drive execnet from. The + pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, ``execnet.multi``, ``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` were only ever reachable because ``import execnet`` pulled them in transitively; they are now deprecated forwarding shims that warn on attribute access and will be removed @@ -55,8 +111,7 @@ ``can_send``. * ``execnet.trio`` no longer exports ``ByteStream``, ``RawChannel``, ``RawChannelStream`` or ``serve_gateway``; the raw-channel layer is internal routing - detail. No names were added to ``execnet.sync``, ``execnet.aio`` or - ``execnet.portal``. + detail. No names were added to ``execnet.sync`` or ``execnet.aio``. 2.1.2 (2025-11-11) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 70146c7d..5f108df2 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -128,9 +128,23 @@ port. Phase E (anyio-native core) becomes optional purity/perf work. in the first waiter's hub (carriers may be constructed on the host loop); tests behind the `gevent` dependency group. -Next: Phase C `loop=main` / `exec=task` on the smaller core -(handoff-phase-c-worker-axes.md), and Phase D docs cover the four -namespaces + the axes/presets. +## SUPERSEDED 2026-07-29 by the surface review + +Phase C landed, and reviewing the whole public surface at once (before +Phase D docs froze it) retired several of the decisions above. See +"Surface review" in `handoff-phase-c-worker-axes.md` for what stands. In +short: + +- **The Wakener extension point is gone.** There was never a plan to let + third parties add event loops, and the asyncio wakener sketched here was + deliberately never built (P4 chose per-call bridging). What is left is + two wait backends, threads and gevent, behind a two-branch + `make_wakener`. `execnet.portal` is now private `execnet._portal`. +- **`wait=` is gone as a spec key.** It described the *caller's* + concurrency library, which is what picking a namespace already says. + gevent got the facade it was missing: `execnet.gevent`. +- **The `loop=` / `exec=` axes stayed dropped**; `execmodel=` became + `profile=` (see below). ## Invariants (unchanged, re-mapped) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 102cb78f..ba289844 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -17,6 +17,63 @@ pytest testing/ -n 12` passes (~7s) since the session-attach race fix `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). Known flake: `test_socket_installvia` EOFs rarely under load. +## Surface review — LANDED 2026-07-29 (`7aa17fe..e75cd0a`) + +The public surface had settled commit by commit and was never reviewed as +a whole. Doing that before the Phase D docs froze it produced the +following, which **wins over anything below or in +`handoff-boundary-protocol-rethink.md` that contradicts it**. + +**Namespaces are now one per concurrency library you drive execnet from**: +`execnet.sync` (threads; the top-level aliases), `execnet.trio`, +`execnet.aio`, `execnet.gevent`. `execnet.portal` is gone -- +`execnet._portal` plus the trio-free `execnet._boundary`. + +| was | is | why | +|---|---|---| +| `execnet.portal` public | `execnet._portal` private | it exported `Wakener`/`Mailbox`/`OneShot`/`LoopPortal` but *not* `register_wakener`, so the advertised extension point was unreachable -- and there is no plan to let third parties add event loops at all | +| Wakener registry (`register_wakener`, lazy module table) | two-branch `make_wakener(Literal["thread","gevent"])` | exactly two backends exist; every other library gets a facade | +| `wait=` spec key | gone; `Group._wait_backend`, set by the facade | it described the *caller*, which the namespace already says. Worker-side wait was always derived from the profile | +| gevent via `wait=gevent` | `execnet.gevent.Group` | symmetric with the other surfaces | +| `execmodel=` spec key | `profile=` (`execmodel=` a permanent alias) | the key selects the *worker profile*; the local execution model it was named after no longer exists | +| `Group(execmodel=)`, `set_execmodel`, `group.execmodel` | deprecated; only the remote default survives | they had no behavioural effect. **xdist passes `Group(execmodel=...)` as a keyword** -- that must keep working | +| `main_thread_only` profile | deprecated alias for `thread` | `HybridExec` already gives the first remote_exec the real main thread. Its extra behaviour (refusing a second concurrent remote_exec) was a 1s-timeout deadlock guard, now deleted | +| one `TrioHost` per `Group` | one shared `execnet.Host` per process, `Group(host=...)` to override | a host is a thread and a loop, not something groups need isolated | +| blocking inside a running loop hangs | raises, naming `execnet.aio` / `execnet.trio` | worker-side channels stay exempt: exec'd code may run its own loop | +| `aio.Group`/`Gateway`/`Channel` | `aio.AsyncGroup`/`AsyncGateway`/`AsyncChannel` | matches `execnet.trio`; swapping the import ports the code | +| `open_popen_gateway` | `open_gateway` (both async surfaces) | it always accepted any spec | + +Behaviour changes worth a changelog line: + +- a second concurrent `remote_exec` under `main_thread_only` used to close + the channel with `MAIN_THREAD_ONLY_DEADLOCK_TEXT`; it now runs on a pool + thread. `_executetask_complete`, `MAIN_THREAD_ONLY_ADMIT_TIMEOUT` and + the error text are deleted; `MainExec` became `PrimaryThreadPump`. +- `execnet.aio` cancellation is now real: a cancelled `receive` cancels + the host-side operation instead of consuming and discarding an item. + `send`/`send_eof`/`aclose`/`terminate` are shielded instead. +- the boundary carriers raise `execnet.TimeoutError`, not the builtin; + `OneShot` double-resolve is a `RuntimeError`, not an `assert`. +- `STATUS` answers both `profile` and (legacy) `execmodel`. +- **Latent livelock fixed**: `execnet.dumps` warned on *every* access, and + xdist calls it from `serialize_warning_message` -- i.e. from inside + pytest's warning-recording hook. One DeprecationWarning in a worker + therefore recorded a warning that recorded a warning, unbounded, and + wedged the run. The shim warns once per process + (`execnet._xdist_compat_warned`). Anything that warns in a worker can + hit this class of bug; keep it in mind. + +New tests: `testing/test_host.py` (sharing, explicit `Host`, fork, the +loop guards), `testing/test_boundary.py` (renamed from `test_portal.py`), +aio cancellation contracts in `testing/test_aio.py`. The `execmodel` +fixture parametrization collapsed to a single `profile` fixture, so the +suite is ~540 items rather than ~765. + +Still open from the review, deliberately not done: the async surfaces have +no `remote_status()`, no `MultiChannel`, no group iteration, and no +`RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` +profile (the coordinator's shape does not dictate the worker's). + ## Where the repo stands (2026-07-25, after `a69b844`) One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API @@ -45,12 +102,12 @@ File map (src/execnet/): | file | role | |---|---| -| `gateway_base.py` | `Message` wire protocol + sans-IO `FrameDecoder`, serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`), sync `Channel`/`ChannelFactory` (raw-receiver registry), `BaseGateway`/`WorkerGateway` (`_send` via bridge, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_message.py` / `_serialize.py` / `_channel.py` / `_gateway_base.py` / `_errors.py` / `_execmodel.py` | split by concern since this table was written: wire protocol + sans-IO `FrameDecoder`; serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`); sync `Channel`/`ChannelFactory`; `BaseGateway`/`WorkerGateway`; error types; `WORKER_PROFILES` + `resolve_profile` + the deprecated `ExecModel` xdist shim. `gateway_base.py` is now only a warning shim | | `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | | `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | | `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | | `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | -| `sync.py` / `trio.py` / `portal.py` | the three public namespaces | +| `sync.py` / `trio.py` / `aio.py` / `gevent.py` | the four public namespaces; `_host.py` holds the shared `Host`, `_portal.py`/`_boundary.py` the (private) boundary kit | | `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | ## Semantics that MUST survive (xdist depends on them) @@ -84,20 +141,23 @@ different use-cases get named profiles, `execmodel=` is the public mode key (xdist already passes it), and `wait=` is the only other public knob. Implemented in commits `b2f43c3..cbce183`: -| `execmodel=` | loop thread | exec'd code runs | channel | worker wait | extra deps | +| `profile=` (was `execmodel=`) | loop thread | exec'd code runs | channel | worker wait | extra deps | |---|---|---|---|---|---| | `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | -| `main_thread_only` | side thread | main thread, serialized (deadlock guard) | sync | thread | — | +| ~~`main_thread_only`~~ | *deprecated 2026-07-29, aliases to `thread`* | | | | | | `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | | `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | +(The coordinator-side counterpart of the last row is now `execnet.gevent`, +not `wait=gevent`.) + Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync bridge at all in the trio profile). Subinterpreters: future strategy -slot, not built. `EXECMODEL_PROFILES` (gateway_base) validates -coordinator-side in makegateway. +slot, not built. `WORKER_PROFILES` (`_execmodel.py`) validates +coordinator-side in makegateway, via `resolve_profile`. **Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the @@ -108,9 +168,9 @@ post-start remote_exec setup block are gone. Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a host task) backs makegateway / `SyncIOHandle.wait/kill` / -`Group.terminate` whenever the wait backend is not `thread`, so a gevent -app's management ops park only the calling greenlet. `wait=thread` -keeps the KI-deferred `portal.run` path. +`Group.terminate` whenever the group's wait backend is not `thread`, so a +gevent app's management ops park only the calling greenlet. The default +`thread` backend keeps the KI-deferred `portal.run` path. Still decided/standing: core stays trio-only (anyio/asyncio-core port rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index adb43950..aee2ddcb 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -110,9 +110,7 @@ def resolve_profile(name: str) -> str: ) return replacement if name not in WORKER_PROFILES: - raise ValueError( - f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})" - ) + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") return name diff --git a/src/execnet/_host.py b/src/execnet/_host.py index 56af3b4f..e57511d8 100644 --- a/src/execnet/_host.py +++ b/src/execnet/_host.py @@ -27,8 +27,12 @@ import sys import threading from types import TracebackType +from typing import TYPE_CHECKING from typing import Any +if TYPE_CHECKING: + from typing_extensions import Self + __all__ = ["Host", "check_not_in_event_loop", "default_host"] #: default cap on concurrent threadpool threads running receiver callbacks @@ -142,7 +146,7 @@ def close(self, timeout: float | None = 5.0) -> None: if trio_host is not None: trio_host.stop(timeout=timeout) - def __enter__(self) -> Host: + def __enter__(self) -> Self: return self def __exit__( diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index c739eedc..70559bfa 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -167,9 +167,7 @@ def set_execmodel( DeprecationWarning, stacklevel=2, ) - self.set_profile( - execmodel if remote_execmodel is None else remote_execmodel - ) + self.set_profile(execmodel if remote_execmodel is None else remote_execmodel) def __repr__(self) -> str: idgateways = [gw.id for gw in self] @@ -273,9 +271,7 @@ def _cleanup_atexit(self) -> None: self.terminate(timeout=1.0) if self._async_group is not None: with suppress(Exception): - self._host._ensure_started().call_sync( - self._async_group.shutdown.set - ) + self._host._ensure_started().call_sync(self._async_group.shutdown.set) self._async_group = None def terminate(self, timeout: float | None = None) -> None: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index b1fccbf5..735ee842 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -42,8 +42,8 @@ from ._errors import HostNotFound from ._errors import RemoteError from ._errors import TimeoutError -from ._execmodel import resolve_profile from ._exec_source import normalize_exec_source +from ._execmodel import resolve_profile from ._message import FrameDecoder from ._message import Message from ._message import gateway_info diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed5ee8f0..567851b0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -32,9 +32,12 @@ from ._errors import RemoteError from ._execmodel import ExecModel from ._execmodel import get_execmodel +from ._host import DEFAULT_CALLBACK_THREADS from ._message import FrameDecoder from ._message import Message from ._message import gateway_info +from ._portal import LoopPortal +from ._portal import OneShot from ._serialize import dumps_internal from ._serialize import loads_internal from ._trace import trace @@ -46,9 +49,6 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from ._host import DEFAULT_CALLBACK_THREADS -from ._portal import LoopPortal -from ._portal import OneShot #: bound on how long the endmarker callback may run during host shutdown CONSUMER_ENDMARKER_GRACE = 10.0 diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 0146c00b..9f824412 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -11,12 +11,13 @@ import trio +from ._boundary import Mailbox +from ._boundary import WaitBackend from ._errors import geterrortext from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway from ._serialize import loads_internal from ._trace import trace -from ._boundary import Mailbox if TYPE_CHECKING: from . import _trio_host @@ -426,7 +427,10 @@ def kill(self) -> None: def _build_worker_gateway( - host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" + host: _trio_host.TrioHost, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", ) -> tuple[WorkerGateway, TrioWorkerExec]: """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") @@ -450,7 +454,11 @@ def _build_worker_gateway( def _run_worker( - host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" + host: _trio_host.TrioHost, + io: Any, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", ) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" gateway, trio_exec = _build_worker_gateway(host, id, model, wait) @@ -528,7 +536,9 @@ async def main() -> None: os._exit(0) -def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> None: +def serve_popen_trio( + id: str, profile: str = "thread", wait: WaitBackend = "thread" +) -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -546,7 +556,7 @@ def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> def serve_socket_trio( - id: str, profile: str, socket_fd: int, wait: str = "thread" + id: str, profile: str, socket_fd: int, wait: WaitBackend = "thread" ) -> None: """Serve a WorkerGateway over an inherited socket fd. @@ -641,7 +651,7 @@ def _main() -> None: # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed # coordinator still connects. profile = config.get("profile") or config["execmodel"] - wait = config.get("wait", "thread") + wait: WaitBackend = config.get("wait", "thread") if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 3e3deee6..fb4f6c2a 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -386,7 +386,7 @@ async def _start_host(host: Host) -> Any: def start() -> None: try: trio_host = host._ensure_started() - except BaseException as exc: # noqa: BLE001 + except BaseException as exc: loop.call_soon_threadsafe(_set_future_error, future, exc) else: loop.call_soon_threadsafe(_set_future_result, future, trio_host) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index a82a1c0f..40eea46c 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -34,7 +34,6 @@ "LoadError": "._errors", # execution model presets "ExecModel": "._execmodel", - "EXECMODEL_PROFILES": "._execmodel", "get_execmodel": "._execmodel", # wire protocol "WriteIO": "._message", diff --git a/testing/conftest.py b/testing/conftest.py index a9797ada..29231d27 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -215,7 +215,8 @@ def gw( @pytest.fixture(params=["thread"], scope="session") def profile(request: pytest.FixtureRequest) -> str: """The worker profile gateways in this test run are created with.""" - return request.param + param: str = request.param + return param @pytest.fixture(scope="session") diff --git a/testing/test_aio.py b/testing/test_aio.py index 8c51ecf2..688643f3 100644 --- a/testing/test_aio.py +++ b/testing/test_aio.py @@ -118,7 +118,7 @@ async def main() -> None: def test_cancelled_receive_does_not_consume_an_item() -> None: # The bridge cancels the host-side receive, so the item stays queued - # instead of being consumed and dropped -- asyncio.timeout around a + # instead of being consumed and dropped -- an asyncio timeout around a # receive must behave like passing timeout=. async def main() -> None: async with execnet.aio.open_gateway() as gateway: @@ -130,8 +130,7 @@ async def main() -> None: """ ) with pytest.raises(asyncio.TimeoutError): - async with asyncio.timeout(0.05): - await channel.receive() + await asyncio.wait_for(channel.receive(), 0.05) # release the worker; nothing was consumed by the cancelled wait await channel.send("go") assert [await channel.receive() for _ in range(3)] == [0, 1, 2] @@ -144,9 +143,7 @@ def test_cancelled_send_still_arrives() -> None: # the wire, rather than a frame half-written to the peer. async def main() -> None: async with execnet.aio.open_gateway() as gateway: - channel = await gateway.remote_exec( - "channel.send(channel.receive() * 2)" - ) + channel = await gateway.remote_exec("channel.send(channel.receive() * 2)") task = asyncio.ensure_future(channel.send(21)) await asyncio.sleep(0) task.cancel() diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 2ef456ef..3f624d16 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -134,3 +134,22 @@ async def main() -> None: assert await channel.receive() == 42 trio_lib.run(main) + + +def test_async_coordinator_defaults_to_a_thread_worker() -> None: + # An async coordinator does not imply an async worker: the worker's + # shape is its own choice, so the default stays the thread profile. + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec( + "import threading;" + " channel.send(threading.current_thread() is" + " threading.main_thread())" + ) + with trio_lib.fail_after(TESTTIMEOUT): + # a sync source at all proves this is not the trio profile, + # which rejects them + assert await channel.receive() is True + + trio_lib.run(main) diff --git a/testing/test_gateway.py b/testing/test_gateway.py index e28a98eb..3c6f2fe1 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -17,7 +17,6 @@ import execnet from execnet import Gateway -from execnet import _execmodel from execnet import _trace TESTTIMEOUT = 10.0 # seconds diff --git a/testing/test_host.py b/testing/test_host.py index 686c022f..f5678a31 100644 --- a/testing/test_host.py +++ b/testing/test_host.py @@ -48,7 +48,6 @@ def test_many_groups_run_one_thread(self) -> None: def test_explicit_host_is_isolated_and_closes(self) -> None: host = Host(name="execnet-host-isolated") - assert not host.running group = execnet.Group(host=host) assert group.host is host assert group.host is not default_host() @@ -56,7 +55,6 @@ def test_explicit_host_is_isolated_and_closes(self) -> None: gateway = group.makegateway("popen") channel = gateway.remote_exec("channel.send(6 * 7)") assert channel.receive(TESTTIMEOUT) == 42 - assert host.running assert "execnet-host-isolated" in host_thread_names() finally: group.terminate(timeout=5.0) @@ -78,9 +76,7 @@ def test_starting_is_lazy(self) -> None: assert not host.running assert "execnet-host-lazy" not in host_thread_names() - @pytest.mark.skipif( - not hasattr(os, "fork"), reason="requires os.fork" - ) + @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") def test_forked_child_gets_a_fresh_default_host(self) -> None: # the child inherits a Host object whose thread does not exist # there, so the first use must build a new one @@ -114,14 +110,14 @@ class TestEventLoopGuard: def test_makegateway_inside_asyncio_raises(self) -> None: async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): execnet.Group().makegateway("popen") asyncio.run(main()) def test_makegateway_inside_trio_raises(self) -> None: async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.trio"): + with pytest.raises(RuntimeError, match=r"execnet\.trio"): execnet.Group().makegateway("popen") trio.run(main) @@ -133,11 +129,11 @@ def test_channel_receive_inside_asyncio_raises(self) -> None: channel = gateway.remote_exec("channel.send(1)") async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.receive(TESTTIMEOUT) - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.send(1) - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.waitclose(TESTTIMEOUT) asyncio.run(main()) From 320c89ef7ad93986203c6a3500c46879c516bde2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 07:16:55 +0200 Subject: [PATCH 65/66] ci: run the pytest-xdist test suite against this execnet Our own suite runs xdist as a *tool*, which exercises none of the paths xdist actually drives: crashed-worker replacement, the main-thread worker profile, report and warning serialization. Adding a job that runs xdist's own suite against the execnet built from the branch found 16 real regressions on the first run (baseline: released execnet 2.1.2 gives 220 passed, 0 failed). Both root causes are fixed here. makegateway wrote the *normalized* profile back onto the caller's XSpec. xdist reuses one spec object across gateways and re-reads spec.execmodel to decide whether it still needs its "execmodel=main_thread_only//" prefix, so after normalization it prefixed a second time and built a spec with a duplicate key -- every crashed-worker-replacement test failed. resolve_profile still validates and warns, but its result is no longer assigned to a caller-owned spec; effective_profile maps silently where the value is consumed. Filling in a missing spec value is idempotent and fine; rewriting one the caller set is not. execnet.dumps no longer warns at all. xdist reaches it once per warning a *user's* test raises, from inside pytest's warning-recording hook, so the warning landed in that user's warnings summary attributed to their test -- about a probe only xdist can port. Warning once per process stopped the livelock but not the noise. One expected failure remains, deselected via a documented list: test_remote_inner_argv asserts sys.argv == ["-c"] and documents the old `python -c` bootstrap, which no-source-shipping deliberately replaced. That needs an xdist PR. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 70 ++++++++++++++++++++++++++++++++ .github/xdist-known-failures.txt | 13 ++++++ CHANGELOG.rst | 17 +++++--- handoff-phase-c-worker-axes.md | 27 ++++++++++++ src/execnet/__init__.py | 32 +++++---------- src/execnet/_execmodel.py | 25 +++++++++++- src/execnet/_multi.py | 4 +- src/execnet/_provision.py | 12 ++++-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_worker.py | 4 +- testing/test_execmodel_trio.py | 28 +++++++++++++ testing/test_namespaces.py | 22 +++++----- 12 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 .github/xdist-known-failures.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad25a355..b600032e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,13 @@ on: branches: - "main" + workflow_dispatch: + inputs: + xdist-ref: + description: "pytest-xdist ref to test against" + default: "main" + type: string + # Cancel running jobs for the same workflow and branch. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -64,3 +71,66 @@ jobs: shell: bash run: | tox run -e py --installpkg `find dist/*.tar.gz` + + # pytest-xdist drives execnet's local parallel testing: popen gateways, + # the main-thread worker profile, crashed-worker replacement, and the + # report/warning serialization. Our own suite runs xdist as a *tool*, + # which does not exercise any of that -- so run xdist's own test suite + # against the execnet built from this branch. + # + # To reproduce locally: + # git clone --depth 1 https://github.com/pytest-dev/pytest-xdist + # python -m venv .xdist-venv + # .xdist-venv/bin/pip install ./pytest-xdist[testing] pytest . + # cd pytest-xdist && ../.xdist-venv/bin/python -m pytest + xdist: + + needs: [package] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + with: + path: execnet + + - uses: actions/checkout@v7 + with: + repository: pytest-dev/pytest-xdist + ref: ${{ inputs.xdist-ref || 'main' }} + path: pytest-xdist + + - name: Download Package + uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install pytest-xdist and this execnet + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install "./pytest-xdist[testing]" pytest + # after xdist, so its execnet>=2.1 requirement does not win + python -m pip install --force-reinstall `find dist/*.tar.gz` + python -c "import execnet; print('execnet', execnet.__version__)" + + - name: Run the pytest-xdist test suite + shell: bash + working-directory: pytest-xdist + run: | + deselect=() + # `|| [ -n "$line" ]` so a file without a trailing newline still + # contributes its last entry + while read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="$(echo "$line" | xargs)" + [ -n "$line" ] && deselect+=(--deselect "$line") + done < ../execnet/.github/xdist-known-failures.txt + printf 'deselecting %d known failure(s)\n' $(( ${#deselect[@]} / 2 )) + python -m pytest -ra "${deselect[@]}" diff --git a/.github/xdist-known-failures.txt b/.github/xdist-known-failures.txt new file mode 100644 index 00000000..6dd34d86 --- /dev/null +++ b/.github/xdist-known-failures.txt @@ -0,0 +1,13 @@ +# pytest-xdist tests that are expected to fail against this execnet, with +# the reason. Everything not listed here must pass -- the whole point of +# the `xdist` CI job is that this list stays short and justified. +# +# Blank lines and `#` comments are ignored; every other line is one pytest +# node id, passed to `pytest --deselect`. + +# Asserts `sys.argv == ["-c"]`, and its own docstring says it is +# documenting "the behavior due to execnet using `python -c`". execnet no +# longer ships source over the wire: workers are launched as +# `python -m execnet._trio_worker `, so sys.argv legitimately +# differs. Needs an xdist-side update, not an execnet fix. +testing/test_remote.py::test_remote_inner_argv diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93a52284..49dd647b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,12 @@ Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an isolated loop with deterministic teardown -- ``Host`` is a context manager and joins its thread on exit, where the shared one stops at interpreter exit. +* A spec's ``profile=``/``execmodel=`` value is no longer rewritten in place when it + names a deprecated profile. pytest-xdist reuses one ``XSpec`` across gateways and + re-reads ``spec.execmodel`` to decide whether it still needs prefixing, so normalizing + the value it set made the second use build a spec with a duplicate key -- which broke + crashed-worker replacement. The spec keeps what the caller spelled; the mapping happens + where the value is consumed. * The ``execmodel=`` spec key is now ``profile=``; ``execmodel=`` remains an accepted alias. It always selected the *worker* profile -- where exec'd code runs relative to the worker's protocol loop -- while the local execution model it was named after no @@ -48,11 +54,12 @@ ``start()``/``aclose()`` from application lifespan hooks instead of ``async with``. ``open_popen_gateway`` is renamed ``open_gateway`` on both async namespaces, since it always accepted any spec. -* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, now warns once per - process rather than on every access. xdist reaches it from ``serialize_warning_message``, - i.e. from inside pytest's warning-recording hook, so a single ``DeprecationWarning`` - raised in a worker made recording that warning record another, unbounded, wedging the - run. +* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, no longer warns. + xdist reaches it from ``serialize_warning_message`` -- from inside pytest's + warning-recording hook, once per warning a *user's* test raises. Warning there put a + spurious execnet ``DeprecationWarning`` in that user's warnings summary, attributed to + their test, about a probe only xdist can port; and warning on every access made + recording one warning record another, unbounded, wedging the run. * ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing to stdout. It has been a no-operation since execnet 1.2. diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ba289844..ca036254 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -69,6 +69,33 @@ aio cancellation contracts in `testing/test_aio.py`. The `execmodel` fixture parametrization collapsed to a single `profile` fixture, so the suite is ~540 items rather than ~765. +### D.3 done: the xdist suite runs in CI (2026-07-30) + +The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own +test suite against the execnet built from the branch. **Doing this for +the first time found 16 real regressions** that our suite could not see +(it runs xdist as a *tool*, which exercises none of the crash-replacement +or report-serialization paths). Baseline: released execnet 2.1.2 gives +220 passed / 0 failed. Two root causes, both fixed: + +1. `makegateway` wrote the *normalized* profile back onto the caller's + XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to + decide whether it still needs the `execmodel=main_thread_only//` + prefix, so after normalization it prefixed a second time and built + `execmodel=...//execmodel=...//popen` -> `ValueError: duplicate key`. + Every crashed-worker-replacement test failed. Fix: `resolve_profile` + validates and warns but callers no longer assign its result to a + caller-owned spec; `effective_profile` (no warning) maps at the + consumption points. **Rule: filling in a missing spec value is + idempotent and fine; rewriting one the caller set is not.** +2. the `execnet.dumps` deprecation warning (see above). + +Remaining known failure, deselected via `.github/xdist-known-failures.txt` +(the job is green otherwise): `test_remote_inner_argv` asserts +`sys.argv == ["-c"]` and documents "the behavior due to execnet using +`python -c`" -- which the no-source-shipping worker launch deliberately +changed. That one needs an xdist PR. + Still open from the review, deliberately not done: the async surfaces have no `remote_status()`, no `MultiChannel`, no group iteration, and no `RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 2ed1293f..e88baf6f 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -89,21 +89,21 @@ #: serializability with ``try: execnet.dumps(x) / except execnet.DumpError`` #: before shipping warning args and report attrs. The standalone serializer #: is internal and :func:`can_send` replaces that probe, but dropping the name -#: outright breaks every released xdist, so it stays reachable -- warning, and -#: deliberately absent from ``__all__``. +#: outright breaks every released xdist, so it stays reachable -- deliberately +#: absent from ``__all__`` and from ``dir()``. +#: +#: It does NOT warn, on purpose. xdist reaches it from +#: ``serialize_warning_message``, i.e. from inside pytest's warning-recording +#: hook and once per warning a *user's* test raises. A warning there is +#: attributed to that test, in a run the user cannot change the outcome of +#: (porting the probe is xdist's call, not theirs) -- and warning on every +#: access made recording one warning record another, unbounded, wedging the +#: run. The deprecation lives in the changelog and in the xdist issue. #: #: FOLLOW-UP (after the execnet release): port xdist to ``execnet.can_send``, #: then delete this and its test. Nothing else may be added here. _XDIST_COMPAT = ("dumps",) -#: Warn about the shim at most once per process. xdist reaches it from -#: ``serialize_warning_message``, i.e. from inside pytest's -#: warning-recording hook: warning every time makes recording one warning -#: record another, which records another -- an unbounded loop that wedges -#: the worker. Once is also simply the right amount of noise for a name -#: whose caller is a library, not the user. -_xdist_compat_warned: set[str] = set() - def __getattr__(name: str) -> Any: if name in _LAZY_MODULES: @@ -113,17 +113,5 @@ def __getattr__(name: str) -> Any: if name in _XDIST_COMPAT: import importlib - if name not in _xdist_compat_warned: - import warnings - - _xdist_compat_warned.add(name) - warnings.warn( - f"execnet.{name} is a temporary pytest-xdist compatibility shim" - " and will be removed; the standalone serializer is not public." - " Use execnet.can_send(obj) to test whether a value can cross a" - " channel.", - DeprecationWarning, - stacklevel=2, - ) return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index aee2ddcb..7c25448c 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -96,8 +96,31 @@ def Event(self) -> threading.Event: DEPRECATED_PROFILES = {"main_thread_only": "thread"} +def effective_profile(name: str) -> str: + """Validate a profile name and map deprecated spellings, silently. + + For the places that *consume* a profile (worker config, strategy + lookup). :func:`resolve_profile` is the one that warns, and is called + once where the value enters. + """ + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + return replacement + if name not in WORKER_PROFILES: + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") + return name + + def resolve_profile(name: str) -> str: - """Validate a ``profile=`` value, mapping deprecated spellings.""" + """Validate a ``profile=`` value, warning about deprecated spellings. + + Callers that hold a caller-supplied :class:`~execnet._xspec.XSpec` must + *not* write the result back onto it: pytest-xdist reuses a spec object + across gateways and re-reads ``spec.execmodel`` to decide whether it + still needs prefixing, so normalizing the value it set makes the second + use build a spec with a duplicate key. Validate here, and map with + :func:`effective_profile` where the value is used. + """ replacement = DEPRECATED_PROFILES.get(name) if replacement is not None: warnings.warn( diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 70559bfa..a2b36505 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -229,9 +229,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec = XSpec(spec) self.allocate_id(spec) if spec.profile is None: + # filling in a missing value is idempotent; rewriting one the + # caller set is not -- see resolve_profile's docstring spec.profile = self._profile else: - spec.profile = resolve_profile(spec.profile) + resolve_profile(spec.profile) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 67b02d17..b1892dcf 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,15 +164,21 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet + from ._execmodel import effective_profile + + # the spec keeps whatever the caller spelled; the worker gets what it + # actually has a strategy for. A spec that never went through a Group + # has no profile at all -- the default applies, as it would there. + profile = effective_profile(spec.profile or "thread") config: dict[str, Any] = { "id": f"{spec.id}-worker", - "profile": spec.profile, + "profile": profile, # pre-3.0 spelling, same value: a worker from an older execnet # reads this one. Drop with the XSpec alias. - "execmodel": spec.profile, + "execmodel": profile, # derived, not configurable: the gevent profile parks its greenlets # on gevent wakeners, every other profile is thread-shaped. - "wait": "gevent" if spec.profile == "gevent" else "thread", + "wait": "gevent" if profile == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 735ee842..9f3f06fe 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -915,7 +915,7 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: # exec'd async sources as tasks. spec.profile = "thread" else: - spec.profile = resolve_profile(spec.profile) + resolve_profile(spec.profile) if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 9f824412..b20be7b6 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -14,6 +14,7 @@ from ._boundary import Mailbox from ._boundary import WaitBackend from ._errors import geterrortext +from ._execmodel import effective_profile from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway from ._serialize import loads_internal @@ -439,7 +440,7 @@ def _build_worker_gateway( gateway._wait_backend = wait try: - strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] + strategy_factory = WORKER_EXEC_STRATEGIES[effective_profile(model.backend)] except KeyError: raise ValueError( f"profile {model.backend!r} has no worker exec strategy " @@ -652,6 +653,7 @@ def _main() -> None: # coordinator still connects. profile = config.get("profile") or config["execmodel"] wait: WaitBackend = config.get("wait", "thread") + profile = effective_profile(profile) if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 3f624d16..8fd121d8 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -153,3 +153,31 @@ async def main() -> None: assert await channel.receive() is True trio_lib.run(main) + + +def test_makegateway_does_not_rewrite_the_callers_profile( + makegateway: Callable[[str], Gateway], +) -> None: + # pytest-xdist reuses one XSpec across gateways and re-reads + # spec.execmodel to decide whether it still needs its + # "execmodel=main_thread_only//" prefix. Normalizing the value it set + # made the second use build a spec with a duplicate key, which broke + # crashed-worker replacement. Filling in a *missing* value is fine; + # rewriting one the caller set is not. + spec = execnet.XSpec("execmodel=main_thread_only//popen") + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway(spec) # type: ignore[arg-type] + assert spec.execmodel == "main_thread_only" + assert spec.profile == "main_thread_only" + # ... while the worker still gets a profile it has a strategy for + assert gw.remote_status().profile == "thread" + + +def test_makegateway_fills_in_a_missing_profile() -> None: + group = execnet.Group() + try: + spec = execnet.XSpec("popen") + group.makegateway(spec) + assert spec.profile == "thread" + finally: + group.terminate(timeout=5.0) diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 856268ea..63a707b3 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -109,24 +109,22 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: from execnet import _serialize assert execnet._XDIST_COMPAT == ("dumps",) - execnet._xdist_compat_warned.discard("dumps") - with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): - assert execnet.dumps is _serialize.dumps + assert execnet.dumps is _serialize.dumps # reachable, but never advertised assert "dumps" not in execnet.__all__ assert "dumps" not in dir(execnet) -def test_dumps_shim_warns_only_once() -> None: - # xdist reaches this from serialize_warning_message, i.e. from inside - # pytest's warning-recording hook: warning every time makes recording - # one warning record another, ad infinitum, and the worker wedges. - execnet._xdist_compat_warned.discard("dumps") - with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): - execnet.dumps # noqa: B018 +def test_dumps_shim_does_not_warn() -> None: + # xdist reaches this from serialize_warning_message -- once per warning + # a *user's* test raises, from inside pytest's warning-recording hook. + # A warning there is attributed to that test, which cannot act on it, + # and warning on every access made recording one warning record + # another, unbounded, wedging the run. with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - execnet.dumps # noqa: B018 + warnings.simplefilter("error") + for _ in range(3): + execnet.dumps # noqa: B018 def test_boundary_kit_is_private() -> None: From 17e7c7a95791700793e4aad93da27e5fda872b69 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 07:35:55 +0200 Subject: [PATCH 66/66] ci: split the xdist job into a pinned release and a floating tip The single floating job could go red for reasons that are xdist's to fix, which is not a reason to block an execnet PR -- but a floating job is also the only thing that catches drift early. So run both. The blocking target is pinned: xdist v3.8.0 with pytest<9, a combination checked to be green against *released* execnet 2.1.2 (196 passed, 0 failed). The pytest pin is load-bearing -- xdist 3.8.0 fails two of its own tests under pytest 9 regardless of which execnet is installed, so without it the job would have started life red and meant nothing. Bump the ref and the pin together, after re-checking the new pair against released execnet. The floating target is continue-on-error and leaves `ref` empty so it follows xdist's default branch rather than hardcoding a name -- which also fixes the previous job: that branch is `master`, not `main`, so the checkout would have failed outright. Both verified locally against this execnet: release 195 passed + 1 deselected, default branch 219 passed + 1 deselected. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 46 +++++++++++++++++++++++++------- .github/xdist-known-failures.txt | 10 +++++-- handoff-phase-c-worker-axes.md | 30 ++++++++++++++++----- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b600032e..c28d57ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,11 +11,6 @@ on: - "main" workflow_dispatch: - inputs: - xdist-ref: - description: "pytest-xdist ref to test against" - default: "main" - type: string # Cancel running jobs for the same workflow and branch. concurrency: @@ -78,10 +73,24 @@ jobs: # which does not exercise any of that -- so run xdist's own test suite # against the execnet built from this branch. # + # Two targets: + # + # * `release` is a pinned combination known to be green against the last + # *released* execnet, so a failure there means we broke something. It + # blocks. Bump `ref`/`pytest` together, and only after checking the new + # pair is green against released execnet -- otherwise the signal is + # gone. (xdist 3.8.0 needs pytest<9: with pytest 9 two of its tests + # fail against released execnet too.) + # * `default-branch` is early warning for drift on both sides. It is + # allowed to fail, because it can go red for reasons that are xdist's + # to fix. An empty `ref` follows whatever xdist's default branch is + # (`master` today) rather than hardcoding a name that could change. + # # To reproduce locally: - # git clone --depth 1 https://github.com/pytest-dev/pytest-xdist + # git clone https://github.com/pytest-dev/pytest-xdist + # git -C pytest-xdist checkout v3.8.0 # or stay on the default branch # python -m venv .xdist-venv - # .xdist-venv/bin/pip install ./pytest-xdist[testing] pytest . + # .xdist-venv/bin/pip install ./pytest-xdist[testing] "pytest<9" . # cd pytest-xdist && ../.xdist-venv/bin/python -m pytest xdist: @@ -89,6 +98,23 @@ jobs: runs-on: ubuntu-latest + name: xdist (${{ matrix.name }}) + + continue-on-error: ${{ matrix.experimental }} + + strategy: + fail-fast: false + matrix: + include: + - name: release + ref: "v3.8.0" + pytest: "pytest<9" + experimental: false + - name: default-branch + ref: "" + pytest: "pytest" + experimental: true + steps: - uses: actions/checkout@v7 with: @@ -97,7 +123,7 @@ jobs: - uses: actions/checkout@v7 with: repository: pytest-dev/pytest-xdist - ref: ${{ inputs.xdist-ref || 'main' }} + ref: ${{ matrix.ref }} path: pytest-xdist - name: Download Package @@ -115,10 +141,10 @@ jobs: shell: bash run: | python -m pip install --upgrade pip - python -m pip install "./pytest-xdist[testing]" pytest + python -m pip install "./pytest-xdist[testing]" "${{ matrix.pytest }}" # after xdist, so its execnet>=2.1 requirement does not win python -m pip install --force-reinstall `find dist/*.tar.gz` - python -c "import execnet; print('execnet', execnet.__version__)" + python -c "import execnet, xdist; print('execnet', execnet.__version__)" - name: Run the pytest-xdist test suite shell: bash diff --git a/.github/xdist-known-failures.txt b/.github/xdist-known-failures.txt index 6dd34d86..c3ded559 100644 --- a/.github/xdist-known-failures.txt +++ b/.github/xdist-known-failures.txt @@ -1,9 +1,15 @@ # pytest-xdist tests that are expected to fail against this execnet, with # the reason. Everything not listed here must pass -- the whole point of -# the `xdist` CI job is that this list stays short and justified. +# the `xdist (release)` CI job is that this list stays short and justified. # # Blank lines and `#` comments are ignored; every other line is one pytest -# node id, passed to `pytest --deselect`. +# node id, passed to `pytest --deselect`. The same list is used for both +# the pinned release and the `main` target; a node id that does not exist +# in one of them is silently ignored, so an entry may name a test that +# only one version has. +# +# Do not add an entry to make CI green. Add one only for a test that is +# asserting behaviour execnet deliberately changed, and say so. # Asserts `sys.argv == ["-c"]`, and its own docstring says it is # documenting "the behavior due to execnet using `python -c`". execnet no diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ca036254..98df1166 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -72,11 +72,25 @@ suite is ~540 items rather than ~765. ### D.3 done: the xdist suite runs in CI (2026-07-30) The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own -test suite against the execnet built from the branch. **Doing this for -the first time found 16 real regressions** that our suite could not see -(it runs xdist as a *tool*, which exercises none of the crash-replacement -or report-serialization paths). Baseline: released execnet 2.1.2 gives -220 passed / 0 failed. Two root causes, both fixed: +test suite against the execnet built from the branch, in two variants: + +- **`release`** — pinned `v3.8.0` + `pytest<9`, a combination verified + green against *released* execnet 2.1.2 (196 passed / 0 failed). This + one blocks: a failure means we broke something. Bump the ref and the + pytest pin together, and only after re-checking the new pair against + released execnet — xdist 3.8.0 with pytest 9 fails two of its own tests + regardless of execnet, which is why the pin exists. +- **`default-branch`** — floating, `continue-on-error`. Early warning for + drift on either side, without letting an xdist-side breakage block + execnet PRs. The `ref` is left empty so it follows xdist's default + branch, which is `master`, not `main` — worth knowing before hardcoding + a name. + +**Doing this for the first time found 16 real regressions** that our suite +could not see (it runs xdist as a *tool*, which exercises none of the +crash-replacement or report-serialization paths). Baseline for xdist's +default branch: released execnet 2.1.2 gives 220 passed / 0 failed. Two +root causes, both fixed: 1. `makegateway` wrote the *normalized* profile back onto the caller's XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to @@ -91,10 +105,12 @@ or report-serialization paths). Baseline: released execnet 2.1.2 gives 2. the `execnet.dumps` deprecation warning (see above). Remaining known failure, deselected via `.github/xdist-known-failures.txt` -(the job is green otherwise): `test_remote_inner_argv` asserts +(both jobs are green otherwise): `test_remote_inner_argv` asserts `sys.argv == ["-c"]` and documents "the behavior due to execnet using `python -c`" -- which the no-source-shipping worker launch deliberately -changed. That one needs an xdist PR. +changed. That one needs an xdist PR. The deselect list is shared by both +targets; pytest ignores a `--deselect` that matches nothing, so an entry +may name a test only one xdist version has. Still open from the review, deliberately not done: the async surfaces have no `remote_status()`, no `MultiChannel`, no group iteration, and no