diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..489c3d39 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(uv run pytest:*)" + ] + } +} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad25a355..c28d57ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,8 @@ on: branches: - "main" + workflow_dispatch: + # Cancel running jobs for the same workflow and branch. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -64,3 +66,97 @@ jobs: shell: bash run: | tox run -e py --installpkg `find dist/*.tar.gz` + + # pytest-xdist drives execnet's local parallel testing: popen gateways, + # the main-thread worker profile, crashed-worker replacement, and the + # report/warning serialization. Our own suite runs xdist as a *tool*, + # which does not exercise any of that -- so run xdist's own test suite + # against the execnet built from this branch. + # + # Two targets: + # + # * `release` is a pinned combination known to be green against the last + # *released* execnet, so a failure there means we broke something. It + # blocks. Bump `ref`/`pytest` together, and only after checking the new + # pair is green against released execnet -- otherwise the signal is + # gone. (xdist 3.8.0 needs pytest<9: with pytest 9 two of its tests + # fail against released execnet too.) + # * `default-branch` is early warning for drift on both sides. It is + # allowed to fail, because it can go red for reasons that are xdist's + # to fix. An empty `ref` follows whatever xdist's default branch is + # (`master` today) rather than hardcoding a name that could change. + # + # To reproduce locally: + # git clone https://github.com/pytest-dev/pytest-xdist + # git -C pytest-xdist checkout v3.8.0 # or stay on the default branch + # python -m venv .xdist-venv + # .xdist-venv/bin/pip install ./pytest-xdist[testing] "pytest<9" . + # cd pytest-xdist && ../.xdist-venv/bin/python -m pytest + xdist: + + needs: [package] + + runs-on: ubuntu-latest + + name: xdist (${{ matrix.name }}) + + continue-on-error: ${{ matrix.experimental }} + + strategy: + fail-fast: false + matrix: + include: + - name: release + ref: "v3.8.0" + pytest: "pytest<9" + experimental: false + - name: default-branch + ref: "" + pytest: "pytest" + experimental: true + + steps: + - uses: actions/checkout@v7 + with: + path: execnet + + - uses: actions/checkout@v7 + with: + repository: pytest-dev/pytest-xdist + ref: ${{ matrix.ref }} + path: pytest-xdist + + - name: Download Package + uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install pytest-xdist and this execnet + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install "./pytest-xdist[testing]" "${{ matrix.pytest }}" + # after xdist, so its execnet>=2.1 requirement does not win + python -m pip install --force-reinstall `find dist/*.tar.gz` + python -c "import execnet, xdist; print('execnet', execnet.__version__)" + + - name: Run the pytest-xdist test suite + shell: bash + working-directory: pytest-xdist + run: | + deselect=() + # `|| [ -n "$line" ]` so a file without a trailing newline still + # contributes its last entry + while read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="$(echo "$line" | xargs)" + [ -n "$line" ] && deselect+=(--deselect "$line") + done < ../execnet/.github/xdist-known-failures.txt + printf 'deselecting %d known failure(s)\n' $(( ${#deselect[@]} / 2 )) + python -m pytest -ra "${deselect[@]}" diff --git a/.github/xdist-known-failures.txt b/.github/xdist-known-failures.txt new file mode 100644 index 00000000..c3ded559 --- /dev/null +++ b/.github/xdist-known-failures.txt @@ -0,0 +1,19 @@ +# pytest-xdist tests that are expected to fail against this execnet, with +# the reason. Everything not listed here must pass -- the whole point of +# the `xdist (release)` CI job is that this list stays short and justified. +# +# Blank lines and `#` comments are ignored; every other line is one pytest +# node id, passed to `pytest --deselect`. The same list is used for both +# the pinned release and the `main` target; a node id that does not exist +# in one of them is silently ignored, so an entry may name a test that +# only one version has. +# +# Do not add an entry to make CI green. Add one only for a test that is +# asserting behaviour execnet deliberately changed, and say so. + +# Asserts `sys.argv == ["-c"]`, and its own docstring says it is +# documenting "the behavior due to execnet using `python -c`". execnet no +# longer ships source over the wire: workers are launched as +# `python -m execnet._trio_worker `, so sys.argv legitimately +# differs. Needs an xdist-side update, not an execnet fix. +testing/test_remote.py::test_remote_inner_argv diff --git a/.gitignore b/.gitignore index d734b32f..9f676ac3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ include/ .vagrant.d/ .config/ .local/ +.claude/settings.local.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be4b6e07..91db1f29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,5 +31,6 @@ repos: - id: mypy additional_dependencies: - pytest + - trio + - hypothesis - types-pywin32 - - types-gevent diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7af6272..8431327b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,177 @@ 2.2.0 (UNRELEASED) ------------------ +* New ``execnet`` command line, and it is now the launch contract between a + coordinator and the worker process it starts:: + + execnet worker --protocol-stdio | --protocol-fd FD[,FD] + | --protocol-connect ADDR | --protocol-listen ADDR + --config JSON | --config-fd FD | --config-file PATH + --stdin/--stdout/--stderr DISPOSITION + execnet server [HOST:PORT] [--once] + execnet info + + ``ADDR`` is ``unix:/path`` or ``host:port``. Provisioning emits + ``python -m execnet worker ...`` for a direct interpreter launch and + ``execnet worker ...`` under ``uv run``; both are the same CLI. + ``execnet-socketserver`` still works and forwards to ``execnet server`` + with a ``DeprecationWarning``. ``execnet info`` reports version, trio + availability and supported transports as JSON, and replaces the + ``import execnet, trio`` probe used to decide whether a ``python=`` + interpreter can host a worker directly. +* The protocol no longer has to be the worker's stdin/stdout. A new + ``transport=socket|stdio`` spec key selects; it defaults to ``socket`` on + POSIX and ``stdio`` on Windows, where neither ``pass_fds`` nor ``ssh -R`` + unix-socket forwarding is available. ``socket`` means an inherited + socketpair for ``popen`` and an ``ssh -R``-forwarded unix socket that the + worker dials back on for ``ssh=``/``vagrant_ssh=``. +* **A worker's stdio now belongs to the code it runs.** It used to be + redirected to the null device so it could not corrupt the protocol, which + meant a remote ``print()`` went nowhere at all. With the socket transport + the worker leaves fd 0/1/2 alone entirely; with the stdio transport it + closes stdin and folds stdout onto stderr rather than discarding both. + Remote output that used to vanish now reaches the coordinator -- and a + worker can now also *read* the coordinator's stdin. New ``stdin=``, + ``stdout=`` and ``stderr=`` spec keys (``inherit``/``devnull``, plus + ``close`` for stdin and ``stderr`` for stdout) override any of it, e.g. + ``popen//stdin=devnull``. +* **The worker config no longer travels in the remote command line** for + ssh gateways, which closes an exposure: the config carries ``env:`` + values, and a remote argv is readable by every user on that host through + ``ps``. The socket transport frees ssh's stdin, so the config goes there + (``--config-fd 0``) instead. +* A shipped wheel (dev-version coordinator) is delivered to an ssh remote + over its own connection before the worker is launched, and cached there + by name. The launch command no longer needs ``head -c `` byte + accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the + protocol stream never carries a payload. +* ``main_thread_only`` used to *serialize*, so every sequential + ``remote_exec`` was guaranteed the worker's main thread. The ``thread`` + profile it now maps to releases its claim as an exec finishes, a moment + after the channel close that lets the coordinator send the next request -- + so a coordinator that immediately re-execs can rarely land on a pool + thread instead. The *first* request always gets the main thread; use the + ``trio`` or ``gevent`` profile where placement must never race. + +* One namespace per concurrency library you drive execnet from: + ``execnet.sync`` (plain threads; the top-level ``execnet.*`` aliases), + ``execnet.trio``, ``execnet.aio`` and the new ``execnet.gevent``, whose blocking + waits park the calling greenlet instead of its OS thread (needs ``execnet[gevent]``). + + ``execnet.trio`` is the only surface that runs gateways *directly*, as tasks in your + own nursery. The others drive a Trio host thread, so their blocking calls now raise + when made from inside a running asyncio or trio loop -- naming the namespace to use + instead -- rather than stalling that loop. Channels inside a worker are exempt: + exec'd code may run its own event loop and talk to its channel from within it. +* The ``execnet.portal`` namespace is gone. It exposed the ``Wakener`` protocol and the + ``Mailbox``/``OneShot``/``LoopPortal`` primitives but not the registry needed to plug + a ``Wakener`` in, and execnet does not offer third-party event-loop integration: a new + concurrency library gets a namespace of its own, as gevent just did. The primitives + are internal again. +* Gateway groups share one Trio host thread per process instead of starting one each. + Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an + isolated loop with deterministic teardown -- ``Host`` is a context manager and joins + its thread on exit, where the shared one stops at interpreter exit. +* A spec's ``profile=``/``execmodel=`` value is no longer rewritten in place when it + names a deprecated profile. pytest-xdist reuses one ``XSpec`` across gateways and + re-reads ``spec.execmodel`` to decide whether it still needs prefixing, so normalizing + the value it set made the second use build a spec with a duplicate key -- which broke + crashed-worker replacement. The spec keeps what the caller spelled; the mapping happens + where the value is consumed. +* The ``execmodel=`` spec key is now ``profile=``; ``execmodel=`` remains an accepted + alias. It always selected the *worker* profile -- where exec'd code runs relative to + the worker's protocol loop -- while the local execution model it was named after no + longer exists. Accordingly ``Group(execmodel=...)``, ``Group.set_execmodel()``, + ``Group.execmodel`` and ``Group.remote_execmodel`` are deprecated in favour of + ``Group(profile=...)``, ``Group.set_profile()`` and ``Group.profile``; only the + remote default they set has any effect. ``remote_status()`` reports both + ``profile`` and ``execmodel``. +* The ``main_thread_only`` profile is deprecated and now behaves like ``thread``, which + already hands the first ``remote_exec`` the worker's real main thread -- the + GUI/signal-safety property it was added for in 2.1.0. Its other behaviour is gone: + a second concurrent ``remote_exec`` used to close the channel with + ``concurrent remote_exec would cause deadlock``, and now runs on a pool thread. That + guard was a one-second timeout that reported a merely slow predecessor as a deadlock. +* The ``wait=`` spec key added earlier in this release cycle is gone. Which primitive a + blocking wait parks on describes the *caller*, which is what choosing a namespace + already says; a worker's own backend is derived from its profile. +* ``execnet.aio`` now propagates cancellation. Cancelling an awaited ``receive`` (with + ``asyncio.timeout``, say) cancels the host-side operation, where it previously + abandoned only the asyncio side and let the operation consume an item that was then + discarded. ``send``, ``send_eof``, ``aclose`` and ``terminate`` are shielded instead, + so they cannot tear halfway. + + Its classes gained the ``Async`` prefix -- ``AsyncGroup``, ``AsyncGateway``, + ``AsyncChannel`` -- matching ``execnet.trio``, and ``AsyncGroup`` can be driven with + ``start()``/``aclose()`` from application lifespan hooks instead of ``async with``. + ``open_popen_gateway`` is renamed ``open_gateway`` on both async namespaces, since it + always accepted any spec. +* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, no longer warns. + xdist reaches it from ``serialize_warning_message`` -- from inside pytest's + warning-recording hook, once per warning a *user's* test raises. Warning there put a + spurious execnet ``DeprecationWarning`` in that user's warnings summary, attributed to + their test, about a probe only xdist can port; and warning on every access made + recording one warning record another, unbounded, wedging the run. +* ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing + to stdout. It has been a no-operation since execnet 1.2. + * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. +* Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and + worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. + Other gateway types and greenlet execmodels keep the legacy thread path. +* Removed ``execnet.script.shell``, an interactive remote prompt that injected its own + source into the pre-Trio socket server. The Trio socket server never execs an incoming + source line, so the module could no longer work against it. +* Removed ``execnet.script.quitserver``, which shut a socket server down by sending it + ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. +* Removed ``execnet.script.loop_socketserver``, a restart loop around a sibling + ``socketserver.py`` file. The socket server serves connections in a loop itself + (``--once`` opts out), and the sibling-file path never resolved for an installed + execnet anyway. +* Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. + Wrap the ``execnet-socketserver`` console command with a service host such as NSSM + instead; the socket gateway example documents how. +* Moved the socket server to ``execnet._socketserver`` and removed the now empty + ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged + and remains the supported way to run it. +* Removed ``Gateway.reconfigure`` and ``Channel.reconfigure`` along with the + ``py2str_as_py3str`` / ``py3str_as_py2str`` arguments of ``gateway_base.loads`` and + ``gateway_base.load``. They configured string coercion between Python2 and Python3 + peers, which execnet can no longer have: ``py2str_as_py3str`` was already unreachable + because only a Python2 serializer emits the opcode it gates. The ``RECONFIGURE`` + message code stays reserved but is no longer sent or handled. +* The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed + ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer + load. Opcode bytes are unchanged for every type that survives. +* The supported API is now exactly five namespaces: ``execnet`` (aliases of + ``execnet.sync``), ``execnet.sync``, ``execnet.trio``, ``execnet.aio`` and + ``execnet.gevent`` -- one per concurrency library you drive execnet from. The + pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, + ``execnet.multi``, ``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` + were only ever reachable because ``import execnet`` pulled them in transitively; they + are now deprecated forwarding shims that warn on attribute access and will be removed + in execnet 3.0. Both ``import execnet.gateway_base`` and ``execnet.gateway_base.X`` + after a plain ``import execnet`` keep working for now. Every name they exposed is + available from a public namespace, except the internals listed below. +* ``gateway_base`` was split into private modules grouped by concern: ``_trace``, + ``_errors``, ``_execmodel``, ``_message`` (IO protocols, ``Message``, + ``FrameDecoder``), ``_serialize``, ``_channel`` (``Channel``, ``ChannelFactory``, + the ``ChannelFile`` adapters) and ``_gateway_base`` (``BaseGateway``, + ``WorkerGateway``). +* Removed ``execnet.loads``, ``execnet.dump`` and ``execnet.load``, and dropped + ``execnet.dumps`` from the public surface. The standalone serializer is internal. + Added ``execnet.can_send(obj)``, which answers whether a value can cross a channel, + for callers that previously probed with ``try: execnet.dumps(x) / except DumpError``. + It lives on ``execnet`` only -- the wire contract does not vary by namespace. + + ``execnet.dumps`` itself stays *reachable* for now, warning on access, purely so + released ``pytest-xdist`` keeps working; it is absent from ``__all__`` and from + ``dir(execnet)``. It is scheduled for removal once xdist ports its probe to + ``can_send``. +* ``execnet.trio`` no longer exports ``ByteStream``, ``RawChannel``, + ``RawChannelStream`` or ``serve_gateway``; the raw-channel layer is internal routing + detail. No names were added to ``execnet.sync`` or ``execnet.aio``. + 2.1.2 (2025-11-11) ------------------ diff --git a/doc/basics.rst b/doc/basics.rst index 78672647..8378dbc3 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -43,8 +43,8 @@ Examples for valid gateway specifications ``default`` via SSH through Vagrant's ``vagrant ssh`` command. It supports the same additional parameters as regular SSH connections. -* ``popen//python=python2.7//nice=20`` specification of - a python subprocess using the ``python2.7`` executable which must be +* ``popen//python=python3.13//nice=20`` specification of + a python subprocess using the ``python3.13`` executable which must be discoverable through the system ``PATH``; running with the lowest CPU priority ("nice" level). By default current dir will be the current dir of the instantiator. @@ -57,12 +57,10 @@ Examples for valid gateway specifications same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. -* ``popen//execmodel=eventlet`` specifies a subprocess that uses the - same interpreter as the one it is initiated from but will run the - other side using eventlet for handling IO and dispatching threads. - * ``socket=192.168.1.4:8888`` specifies a Python Socket server - process that listens on ``192.168.1.4:8888`` + process that listens on ``192.168.1.4:8888``. Such a server can be + started with the ``execnet-socketserver`` console command, e.g. run + anywhere with ``uvx --from execnet execnet-socketserver :8888``. .. versionadded:: 1.5 @@ -74,7 +72,7 @@ Examples for valid gateway specifications remote_exec: execute source code remotely =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to execute source code in the instantiated subprocess-interpreter: @@ -88,10 +86,6 @@ a channel object whose symmetric counterpart channel is available to the remotely executing source. -.. method:: Gateway.reconfigure([py2str_as_py3str=True, py3str_as_py2str=False]) - - Reconfigures the string-coercion behaviour of the gateway - .. _`Channel`: .. _`channel-api`: @@ -100,7 +94,7 @@ is available to the remotely executing source. Channels: exchanging data with remote code ======================================================= -.. currentmodule:: execnet.gateway_base +.. currentmodule:: execnet A channel object allows to send and receive data between two asynchronously running programs. @@ -120,7 +114,7 @@ two asynchronously running programs. Grouped Gateways and robust termination =============================================== -.. currentmodule:: execnet.multi +.. currentmodule:: execnet All created gateway instances are part of a group. If you call ``execnet.makegateway`` it actually is forwarded to @@ -138,30 +132,29 @@ processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. -threading models: gevent, eventlet, thread, main_thread_only +threading models: thread, main_thread_only ==================================================================== .. versionadded:: 1.2 (status: experimental!) -execnet supports "main_thread_only", "thread", "eventlet" and "gevent" -as thread models on each of the two sides. You need to decide which -model to use before you create any gateways:: +execnet supports "thread" and "main_thread_only" as thread models on +each of the two sides. You need to decide which model to use before +you create any gateways:: # content of threadmodel.py import execnet - # locally use "eventlet", remotely use "thread" model - execnet.set_execmodel("eventlet", "thread") + # locally use "thread", remotely use "main_thread_only" model + execnet.set_execmodel("thread", "main_thread_only") gw = execnet.makegateway() print (gw) print (gw.remote_status()) print (gw.remote_exec("channel.send(1)").receive()) -You need to have eventlet installed in your environment and then -you can execute this little test file:: +You can execute this little test file:: $ python threadmodel.py - - + + 1 How to execute in the main thread @@ -176,7 +169,7 @@ executions concurrently, they will run in non-main threads. remote_status: get low-level execution info =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to obtain some status information from the remote side. @@ -219,23 +212,64 @@ configure a tracing mechanism: .. _`dumps/loads`: .. _`dumps/loads API`: +.. _`serialization`: -Cross-interpreter serialization of Python objects +Sending objects over a channel ======================================================= -.. versionadded:: 1.1 +A channel carries only **simple builtin data**: ``None``, ``bool``, +``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily nested +``list`` / ``tuple`` / ``set`` / ``frozenset`` / ``dict`` of those -- plus +**channel references**, which arrive as channels on the peer. That is the +entire contract. + +execnet does **not** pickle and does **not** encode rich objects for you: +arbitrary instances, functions, ``datetime``, dataclasses, pydantic models, +numpy arrays, enums, etc. have no wire representation. This is deliberate; +encoded / rich-object channels are out of scope for execnet. + +Sending an unsupported value raises ``DumpError`` (a subclass of +``DataFormatError``); a corrupt or protocol-mismatched payload on receive +raises ``LoadError``. These signal a **caller error to resolve** -- reduce +the value to simple data before sending -- not a transport failure. The +standalone serializer itself is an internal implementation detail and is not +part of the public API. + +To branch *before* sending rather than handling the error, ask: + +.. autofunction:: can_send + +:: + + channel.send(value if execnet.can_send(value) else repr(value)) + +It lives on ``execnet`` itself rather than on any one namespace: the wire +contract is the same whichever surface you drive a gateway from. + +Encode rich objects yourself +------------------------------------------------------- + +Turning a rich object into simple data (and back) is the caller's job. Use +an established encoding mechanism rather than expecting the channel to do it: + +- **pydantic**: ``model.model_dump(mode="json")`` reduces a model to simple + data (``datetime`` -> ISO string, ``UUID`` / ``Enum`` / ``Decimal`` -> + primitives); ``Model.model_validate(...)`` rebuilds it on the other side. + ``TypeAdapter`` covers non-model types. + + :: -Execnet exposes a function pair which you can safely use to -store and load values from different Python interpreters -(e.g. Python2 and Python3, PyPy and Jython). Here is -a basic example:: + channel.send(model.model_dump(mode="json")) + # peer: + model = MyModel.model_validate(channel.receive()) - >>> import execnet - >>> dump = execnet.dumps([1,2,3]) - >>> execnet.loads(dump) - [1,2,3] +- **pytest** does exactly this above execnet: pytest-xdist ships + ``TestReport`` objects with the ``pytest_report_to_serializable`` / + ``pytest_report_from_serializable`` hooks (rich report <-> simple dict) + around ``channel.send`` / ``channel.receive``. -For more examples see :ref:`dumps/loads examples`. +- **stdlib**: ``dataclasses.asdict(obj)``, ``dt.isoformat()`` / + ``datetime.fromisoformat``, or ``json`` with a ``default=`` hook. -.. autofunction:: execnet.dumps(spec) -.. autofunction:: execnet.loads(spec) +Channels are the one non-builtin you *can* send: nested channel references +pass through intact, so callbacks and sub-streams need no encoding. diff --git a/doc/example/hybridpython.rst b/doc/example/hybridpython.rst deleted file mode 100644 index f4ad465a..00000000 --- a/doc/example/hybridpython.rst +++ /dev/null @@ -1,153 +0,0 @@ -Connecting different Python interpreters -========================================== - -.. _`dumps/loads examples`: - -Dumping and loading values across interpreter versions ----------------------------------------------------------- - -.. versionadded:: 1.1 - -Execnet offers a new safe and fast :ref:`dumps/loads API` which you -can use to dump builtin python data structures and load them -later with the same or a different python interpreter (including -between Python2 and Python3). The standard library offers -the pickle and marshal modules but they do not work safely -between different interpreter versions. Using xml/json -requires a mapping of Python objects and is not easy to -get right. Moreover, execnet allows to control handling -of bytecode/strings/unicode types. Here is an example:: - - # using python2 - import execnet - with open("data.py23", "wb") as f: - f.write(execnet.dumps(["hello", "world"])) - - # using Python3 - import execnet - with open("data.py23", "rb") as f: - val = execnet.loads(f.read(), py2str_as_py3str=True) - assert val == ["hello", "world"] - -See the :ref:`dumps/loads API` for more details on string -conversion options. Please note, that you can not dump -user-level instances, only builtin python types. - -Connect to Python2/Numpy from Python3 ----------------------------------------- - -Here we run a Python3 interpreter to connect to a Python2.7 interpreter -that has numpy installed. We send items to be added to an array and -receive back the remote "repr" of the array:: - - import execnet - gw = execnet.makegateway("popen//python=python2.7") - channel = gw.remote_exec(""" - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) - """) - for x in range(10): - channel.send(x) - channel.send(None) - print (channel.receive()) - -will print on the CPython3.1 side:: - - array([1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - -A more refined real-life example of python3/python2 interaction -is the anyvc_ project which uses version-control bindings in -a Python2 subprocess in order to offer Python3-based library -functionality. - -.. _anyvc: http://bitbucket.org/RonnyPfannschmidt/anyvc/overview/ - - -Reconfiguring the string coercion between python2 and python3 -------------------------------------------------------------- - -Sometimes the default configuration of string coercion (2str to 3str, 3str to 2unicode) -is inconvient, thus it can be reconfigured via `gw.reconfigure` and `channel.reconfigure`. Here is an example session on a Python2 interpreter:: - - - >>> import execnet - >>> execnet.makegateway("popen//python=python3.2") - - >>> gw=execnet.makegateway("popen//python=python3.2") - >>> gw.remote_exec("channel.send('hello')").receive() - u'hello' - >>> gw.reconfigure(py3str_as_py2str=True) - >>> gw.remote_exec("channel.send('hello')").receive() - 'hello' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.send('a') - >>> ch.receive() - 'str' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.reconfigure(py2str_as_py3str=False) - >>> ch.send('a') - >>> ch.receive() - u'bytes' - - -Work with Java objects from CPython ----------------------------------------- - -Use your CPython interpreter to connect to a `Jython 2.5.1`_ interpreter -and work with Java types:: - - import execnet - gw = execnet.makegateway("popen//python=jython") - channel = gw.remote_exec(""" - from java.util import Vector - v = Vector() - v.add('aaa') - v.add('bbb') - for val in v: - channel.send(val) - """) - - for item in channel: - print (item) - -will print on the CPython side:: - - aaa - bbb - -.. _`Jython 2.5.1`: http://www.jython.org - -Work with C# objects from CPython ----------------------------------------- - -(Experimental) use your CPython interpreter to connect to a IronPython_ interpreter -which can work with C# classes. Here is an example for instantiating -a CLR Array instance and sending back its representation:: - - import execnet - gw = execnet.makegateway("popen//python=ipy") - - channel = gw.remote_exec(""" - import clr - clr.AddReference("System") - from System import Array - array = Array[float]([1,2]) - channel.send(str(array)) - """) - print (channel.receive()) - -using Mono 2.0 and IronPython-1.1 this will print on the CPython side:: - - System.Double[](1.0, 2.0) - -.. note:: - Using IronPython needs more testing, likely newer versions - will work better. please feedback if you have information. - -.. _IronPython: http://ironpython.net diff --git a/doc/example/py3topy2.py b/doc/example/py3topy2.py deleted file mode 100644 index 3dcf0437..00000000 --- a/doc/example/py3topy2.py +++ /dev/null @@ -1,19 +0,0 @@ -import execnet - -gw = execnet.makegateway("popen//python=python2") -channel = gw.remote_exec( - """ - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) -""" -) -for x in range(10): - channel.send(x) -channel.send(None) -print(channel.receive()) diff --git a/doc/example/test_info.rst b/doc/example/test_info.rst index eefe91b3..7a5ead4f 100644 --- a/doc/example/test_info.rst +++ b/doc/example/test_info.rst @@ -172,15 +172,23 @@ sends back the results. Instantiate gateways through sockets ----------------------------------------------------- -.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/main/src/execnet/script/socketserver.py +In cases where you do not have SSH-access to a machine you need a +bootstrapping-point that listens on a socket. execnet ships one as the +``execnet-socketserver`` console command; run it on the target machine:: -In cases where you do not have SSH-access to a machine -you need to download a small version-independent standalone -`socketserver.py`_ script to provide a remote bootstrapping-point. -You do not need to install the execnet package remotely. -Simply run the script like this:: + execnet-socketserver :8888 # bind to all IPs, port 8888 - python socketserver.py :8888 # bind to all IPs, port 8888 +If execnet is not installed there, uv_ can fetch and run it in one step +without leaving anything behind:: + + uvx --from execnet execnet-socketserver :8888 + +.. _uv: https://docs.astral.sh/uv/ + +The server accepts connections in a loop and serves each one in its own +worker subprocess; pass ``--once`` to serve a single connection and exit. +Passing port ``0`` binds an ephemeral port -- the bound address is printed +on the first line of output either way. You can then instruct execnet on your local machine to bootstrap itself into the remote socket endpoint:: @@ -191,4 +199,47 @@ itself into the remote socket endpoint:: That's it, you can now use the gateway object just like a popen- or SSH-based one. +.. warning:: + + The socket server performs no authentication and its traffic is not + encrypted -- anyone who can reach the port can execute code on that + machine. Bind it to a trusted network only, or tunnel it over SSH. + +Keeping the socket server running ++++++++++++++++++++++++++++++++++ + +``execnet-socketserver`` is an ordinary long-running process, so restarts and +boot-time startup are the job of your platform's service manager. + +On Linux, a systemd unit does it:: + + # /etc/systemd/system/execnet-socketserver.service + [Unit] + Description=execnet socket server + + [Service] + ExecStart=/usr/local/bin/execnet-socketserver :8888 + Restart=always + + [Install] + WantedBy=multi-user.target + +On Windows, wrap the console command with a service host such as NSSM_ or +WinSW_. Use ``where execnet-socketserver`` to find the installed +``execnet-socketserver.exe`` (it lives in the ``Scripts`` directory of the +environment it was installed into), then:: + + nssm install ExecNetSocketServer C:\path\to\Scripts\execnet-socketserver.exe :8888 + nssm set ExecNetSocketServer Start SERVICE_AUTO_START + net start ExecNetSocketServer + +NSSM restarts the process if it exits. Note that ``sc.exe create`` on its own +is not enough: it expects a binary that implements the Windows service control +protocol, which a plain console program does not. If you would rather not +install a service host, a Task Scheduler task with an *At startup* trigger +works too. + +.. _NSSM: https://nssm.cc/ +.. _WinSW: https://github.com/winsw/winsw + .. include:: test_ssh_fileserver.rst diff --git a/doc/examples.rst b/doc/examples.rst index 767fa4fc..3f09197a 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -14,7 +14,6 @@ Note: all examples with `>>>` prompts are automatically tested. example/test_group example/test_proxy example/test_multi - example/hybridpython example/test_debug .. toctree:: diff --git a/doc/implnotes.rst b/doc/implnotes.rst index d13d9e87..eed92232 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -7,7 +7,7 @@ capable of receiving and executing code, and routing data through channels. Gateways operate on InputOutput objects offering -a write and a read(n) method. + a write and a read(n) method. Once bootstrapped a higher level protocol based on Messages is used. Messages are serialized @@ -15,18 +15,60 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -After bootstrapping the BaseGateway opens a receiver thread which -accepts encoded messages and triggers actions to interpret them. -Sending of channel data items happens directly through -write operations to InputOutput objects so there is no -separate thread. - -Code execution messages are put into an execqueue from -which they will be taken for execution. gateway.serve() -will take and execute such items, one by one. This means -that by incoming default execution is single-threaded. - -The receiver thread terminates if the remote side sends -a gateway termination message or if the IO-connection drops. -It puts an end symbol into the execqueue so -that serve() can cleanly finish as well. +Trio host-thread IO +------------------- + +``popen`` and ``ssh`` gateways run their Message protocol IO inside a +dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire; the +worker is launched as ``python -m execnet._trio_worker`` and imports the +installed ``execnet`` + ``trio`` (a rough major/minor version check guards +against an incompatible install). How the worker environment is obtained +depends on the target: + +* Same-interpreter ``popen`` -> ``sys.executable -m execnet._trio_worker``. +* A ``python=`` interpreter that already has execnet -> that interpreter + directly (so ``sys.executable`` is preserved). +* A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via + ``uv`` (``execnet._provision``): ``uv run --with `` where ```` + is ``execnet==`` for a released coordinator or a locally-built, + version-cached wheel for a dev coordinator. For an ``ssh`` remote on a + dev coordinator the wheel is not on the remote filesystem, so the remote + command is a POSIX-sh prelude that reads the wheel bytes from stdin + (``head -c N`` into a temp dir) and ``exec``s ``uv`` against it; the + coordinator streams those bytes before the Message protocol. + +The worker configuration (id, execmodel, coordinator version) is passed as a +single JSON CLI argument (``_provision.worker_cli_arg``), so every launcher +shares one contract instead of scattered positional args. + +Coordinator and worker roles: + +* Coordinator: ``trio.lowlevel.open_process`` (directly, or wrapped in + ``ssh``) plus async framed reader/writer tasks per gateway (one host + thread per ``Group``). It waits for the worker's ``b"1"`` handshake + before starting the Message protocol. +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio streams and + writes the handshake byte; ``remote_exec`` is scheduled from the Trio + nursery (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host +threads wait until the frame is written (so abrupt ``os._exit`` cannot drop +queued data). Sends from the Trio host thread (receiver callbacks) only +enqueue, to avoid deadlocking the writer task. + +``socket`` and ``via`` gateways run on the Trio host too. ``socket`` +connects a Trio TCP stream to an ``execnet-socketserver`` (itself a Trio +listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess +workers). Infrastructure that used to be driven by ``remote_exec``-ing +source is now expressed as native protocol messages handled on the target's +Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, +reply with its address) and ``GATEWAY_START_SUB`` (``via`` -> spawn a +sub-worker — popen, foreign python, ssh, or vagrant — and relay its protocol +over the request channel). + +The Trio host is the only IO path; the legacy thread receiver and the +source-shipping bootstrap have been removed. The receiver task terminates +when the remote side sends a gateway termination message or the +IO-connection drops. diff --git a/doc/index.rst b/doc/index.rst index 19a20eb8..47a918cc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,10 +31,10 @@ Features * Automatic bootstrapping: no manual remote installation. * Safe and simple serialization of Python builtin - types for sending/receiving structured data messages. - (New in 1.1) execnet offers a new :ref:`dumps/loads ` - API which allows cross-interpreter compatible serialization - of Python builtin types. + types for sending/receiving structured data messages; + see :ref:`sending objects over a channel `. + Encoding rich objects is the caller's job (execnet stays + builtin-types-only). * Flexible communication: synchronous send/receive as well as callback/queue mechanisms supported @@ -59,9 +59,6 @@ Known uses project to launch computation processes through ssh. He also compares `disco and execnet`_ in a subsequent post. -* Ronny Pfannschmidt uses it for his `anyvc`_ VCS-abstraction project - to bridge the Python2/Python3 version gap. - * Sysadmins and developers are using it for ad-hoc custom scripting .. _`quora`: http://quora.com @@ -71,7 +68,6 @@ Known uses .. _`distributed testing`: https://pypi.python.org/pypi/pytest-xdist .. _`Distributed NTLK with execnet`: http://streamhacker.com/2009/11/29/distributed-nltk-execnet/ .. _`disco and execnet`: http://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ -.. _`anyvc`: http://bitbucket.org/RonnyPfannschmidt/anyvc/ Project status -------------------------- diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md new file mode 100644 index 00000000..5f108df2 --- /dev/null +++ b/handoff-boundary-protocol-rethink.md @@ -0,0 +1,163 @@ +# Handoff: host boundary protocol rethink (P1–P5, precedes Phase C proper) + +Decided with Ronny 2026-07-26. This supersedes the *ordering* of +`handoff-phase-c-worker-axes.md` (still valid for C content): the boundary +rethink lands first, then C's `loop=`/`exec=` work continues on the +smaller core. The wire protocol (Message framing) is untouched — this is +about how things cross between the trio host loop and its consumers. + +## Rationale + +Consumer→loop is already universal: `LoopPortal.post` rides +`TrioToken.run_sync_soon`, thread-safe from any foreign context (plain +thread, gevent greenlet, asyncio callback, another trio loop). One +ingress, strict FIFO — unchanged. + +Loop→consumer is hardwired to threading primitives (`Channel._items` +execmodel queue, `threading.Event` for waitclose/join/write-acks, +`SyncReceiver`), which is the only reason `ExecModel` still exists. The +fix: the loop never blocks and never knows who listens — it fires a +thread-safe wakeup callable the consumer supplied. Every event loop has +exactly one such primitive: + +| consumer | wakeup primitive | +|---|---| +| plain thread | `threading.Event.set` | +| asyncio | `loop.call_soon_threadsafe` | +| gevent | `hub.loop.async_()` watcher — `send()` is gevent's one documented thread-safe op | +| foreign trio loop | `TrioToken.run_sync_soon` | + +## The boundary kit (grows in `execnet.portal`) + +1. **`Wakener`** — protocol, one thread-safe method `notify()`. The + entire integration surface for a new event loop (eventlet, Qt, …) + is implementing this; no more ExecModel. +2. **`Mailbox[T]`** — unbounded deque + Wakener. `put()` from the loop + (append + notify, never blocks the loop); `get(timeout)` blocking for + sync backends — generalizes the KI-safe `SyncReceiver` drain pattern + (Event.wait + drain, re-check after clear) — and `await`-able for + asyncio/trio wakeners. Replaces `Channel._items`, `SyncReceiver`, + `MultiChannel`'s queue. +3. **`OneShot[T]`** — single-result future on the same wakener. + Replaces the per-send write-ack `threading.Event`; adds + `portal.start(async_fn) -> OneShot` so asyncio callers await + management ops instead of blocking a thread in `portal.run`. + +## Channel unification (decided: full rebase) + +`AsyncGateway._dispatch` becomes the ONLY message router. `RawChannel` +grows a consumer hook: a bound facade diverts payloads into a `Mailbox` +instead of the internal memory channel (callbacks: invoked on the loop +thread, preserving current semantics; moving them later is just posting +via the wakener). + +Sync `Channel` becomes a genuine facade over (raw id, mailbox, portal): + +- `receive()` = `mailbox.get(timeout)` + `loads_internal` at the call + site — deserialization moves off the loop thread; `DataFormatError` + now surfaces at `receive()`. +- `send()` = portal-posted frame + `OneShot` write-ack (non-loop threads + keep block-until-written, 120s → OSError, KI-safe). +- `waitclose()`/`join()` = closed-events on the wakener. +- `__del__` best-effort close keeps `portal.post` (post_message path). +- Unknown inbound ids still auto-create (`_channel_for`). + +Deleted outright: `_receivelock`, `ChannelFactory` dispatch paths +(`_local_receive`/`_local_close`), `SyncBridgeGateway._dispatch`, the +`Message.received` handler-table use, the duplicate close state machine. +`CHANNEL_EXEC`/`GATEWAY_START_*` become per-gateway hooks on the async +core. `TrioWorkerExec` FIFO pump survives unchanged; exec'd code gets +facade channels. + +## Execmodels: retired as machinery, preserved as presets + +execmodel names become presets over three orthogonal axes (spec + worker +CLI config; decided: **both** coordinator and worker side): + +| axis | values | meaning | +|---|---|---| +| `loop` | `main` \| `thread` | where the trio host runs (Phase C) | +| `exec` | `thread` \| `main` \| `task` | where remote_exec code runs (Phase C) | +| `wait` | `thread` \| `gevent` \| `asyncio` | which Wakener blocking facade waits park on (new) | + +Presets: `execmodel=thread` → `loop=main, exec=thread, wait=thread`; +`main_thread_only` → `loop=thread, exec=main, wait=thread`; `gevent` +returns as `loop=thread, exec=thread, wait=gevent` (greenlet channel +waits park the greenlet, not the hub; sends already greenlet-safe via +the portal). eventlet stays dead (third parties can implement Wakener). + +Dies: `ExecModel` ABC, `WorkerPool`, `Reply` (`multi.safe_terminate` +moves to plain threads or delegates to `FacadeAsyncGroup.terminate`). +Stays as deprecated preset-mappers: `get_execmodel`, `set_execmodel`, +`group.execmodel`, `spec.execmodel` (xdist compat). + +## asyncio (decided: land now) + +`execnet.aio` namespace mirroring `execnet.trio`'s surface; every op is +portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full +asyncio-native API over the trio host loop, all transports, no anyio +port. Phase E (anyio-native core) becomes optional purity/perf work. + +## Staging — ALL LANDED 2026-07-26 (commits 2c78416..b01d009) + +1. **P1 — boundary kit** (`2c78416`): Wakener/Mailbox/OneShot + + ThreadWakener; SyncReceiver, write-acks, `_done_sync` ported. +2. **P2 — channel unification** (`2a40a55`): sync Channel onto + RawChannel(+set_consumer)+Mailbox, classic dispatch deleted. The kit + lives in trio-free `execnet._boundary` (portal re-exports) so + `import execnet` stays trio-free. Fix worth knowing: an unconsumed + remotely-closed RawChannel stays registered until a consumer claims + it — call-site deserialization means a passed channel can bind after + its data AND close arrived (was a payload-loss race, latent in the + async core's `open_channel(id)` too). +3. **P3 — retirement** (`811fd99`): ExecModel ABC/WorkerPool/Reply gone; + one deprecated ExecModel preset KEEPS the stdlib-delegate members + because pytest-xdist's remote worker calls + `channel.gateway.execmodel.RLock()/Event()`; `wait=` axis end to end + (spec validation, worker CLI config, `BaseGateway._new_wakener`, + `_boundary` registry + Flag); safe_terminate on daemon threads; + test_threadpool.py deleted. +4. **P4 — `execnet.aio`** (`845510e`): implemented as a per-call + _HostBridge (host task + `loop.call_soon_threadsafe` future), NOT the + originally sketched AsyncioWakener/Mailbox — real awaitables over the + trio-native AsyncGroup/AsyncChannel, simpler and semantically exact. + The "asyncio" wait= registry entry is therefore unused/unregistered. + v1 limitation: cancelling a bridged await abandons it aio-side only. +5. **P5 — gevent wakener** (`b01d009`): `execnet._gevent_support`, + lazily imported by `make_wakener("gevent")`; hub-bound pieces created + in the first waiter's hub (carriers may be constructed on the host + loop); tests behind the `gevent` dependency group. + +## SUPERSEDED 2026-07-29 by the surface review + +Phase C landed, and reviewing the whole public surface at once (before +Phase D docs froze it) retired several of the decisions above. See +"Surface review" in `handoff-phase-c-worker-axes.md` for what stands. In +short: + +- **The Wakener extension point is gone.** There was never a plan to let + third parties add event loops, and the asyncio wakener sketched here was + deliberately never built (P4 chose per-call bridging). What is left is + two wait backends, threads and gevent, behind a two-branch + `make_wakener`. `execnet.portal` is now private `execnet._portal`. +- **`wait=` is gone as a spec key.** It described the *caller's* + concurrency library, which is what picking a namespace already says. + gevent got the facade it was missing: `execnet.gevent`. +- **The `loop=` / `exec=` axes stayed dropped**; `execmodel=` became + `profile=` (see below). + +## Invariants (unchanged, re-mapped) + +- Non-loop-thread sends block until the frame hit the OS write (OneShot + write-ack; 120s → OSError); loop-thread sends only enqueue; all sends + one portal FIFO. +- After close: `OSError("cannot send (already closed?)")`. +- exec admission order = message arrival order (TrioWorkerExec._pump). +- Channel callbacks run off the loop thread: a per-channel consumer task + drains into a threadpool call (moved 2026-07-26; the kit made it cheap, + as predicted). See `handoff-phase-c-worker-axes.md` "Callback consumer + tasks". +- `Group.terminate(timeout)` never hangs (~2×timeout). +- Sync blocking waits stay KeyboardInterrupt-interruptible on the main + thread (thread Wakener keeps the Event.wait + drain pattern). +- No source shipping; `import execnet` must not import trio. diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md new file mode 100644 index 00000000..d3a69ff6 --- /dev/null +++ b/handoff-phase-b-async-core.md @@ -0,0 +1,182 @@ +# Handoff: Phase B — invert execnet onto an async-native Trio core + +**SUPERSEDED by `handoff-phase-c-worker-axes.md`** — kept as the Phase B +record. **Phase B is complete (B.1–B.6)** on branch +`feat/trio-host-thread-io`; work continues at **Phase C**. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). ssh paths have a real local harness +in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-24, after B.5) + +Trio is the only IO path; no source is shipped over the wire (workers run +`python -m execnet._trio_worker `, foreign/remote interpreters +are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All +transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` +sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). + +**The inversion landed (B.5)**: there is one protocol engine — +`AsyncGateway` — and the sync API is a facade over it. `ProtocolSession` +is gone. Coordinator and worker sessions are `SyncBridgeGateway` +(an `AsyncGateway` subclass) whose `_dispatch` runs the classic sync +`Message.received` handlers under `_receivelock`; the sync +`Channel`/`ChannelFactory` state machine in `gateway_base` is unchanged +and remains the semantic layer for blocking users (and the worker's +exec'd code). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; `makegateway` and the bounded process shutdown delegate to +`AsyncGroup`, while `terminate` still calls each member's +`exit()`/`join()` (pinned by `test_basic_group`). + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (raw-receiver registry: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway` (`_send` via bridge session, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core**: `ByteStream` Protocol, `RawChannel`, `AsyncChannel`, `AsyncGateway` (single serve task; outbound queue items are `(frame, on_written)` so sync senders can wait for the OS write; writer fails pending frames on shutdown; `_finalize` hook for subclass shutdown), `AsyncGroup` (async CM owning the nursery; **all transports**: popen/`python=`/ssh/vagrant_ssh/socket(+installvia)/`via=`; overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`; per-process reaper tasks; terminate = tunneled first, GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream`, `open_popen_gateway`, transport helpers (`connect_command_worker` incl. ssh-255→HostNotFound, `connect_socket_worker`, `start_socketserver_via` (async), `ssh_transport_args`, `popen_worker_argv`) | +| `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread; `start_session` returns a bridge), **`SyncBridgeGateway`** (sync dispatch + portal-FIFO `enqueue_message`/`post_message`/`request_close_write`, threading-`Event` `wait_done`), **`FacadeAsyncGroup`** (bridge-building AsyncGroup; via/installvia through the sync master), `makegateway_trio`, `SyncIOHandle` (remoteaddress/wait/kill/close_write), `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers (`_start_sub_and_relay` frame-native) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds`; serves on a `SyncBridgeGateway` | +| `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | +| `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | +| `multi.py` | sync `Group` facade (`_ensure_async_group`, terminate via `AsyncGroup`), `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | +| `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | + +Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair +protocol tests + popen/via integration inside `trio.run`). + +Semantics that MUST survive (xdist depends on them): + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; sends from + the loop thread only enqueue. After close: `OSError("cannot send (already + closed?)")`; `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (see `TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now. + +## Phase B goal + +Three public namespaces, all pure Trio (**no anyio/asyncio** — an +`execnet.anyio` backend is deferred Phase E; keep idioms portable: neutral +stream protocol, sans-IO protocol logic, no gratuitous trio-only constructs): + +``` +execnet.sync – today's blocking API, rebuilt as a facade (top-level + execnet.Group etc. stay as compatibility aliases) +execnet.trio – trio-native AsyncGroup/AsyncGateway/AsyncChannel, awaited + directly inside the user's own trio.run +execnet.portal – the communicating API (exists as portal.py; public + exposure happens in B.6) +``` + +## Remaining work + +### B.4 Async-native core — DONE (commits `0eb6cc5`..`8286b77`) + +Everything lives in `_trio_gateway.py` (see file map). Landed: +RawChannel + AsyncChannel two-level model, AsyncGateway single-task +dispatch (no `_receivelock` on the async path), AsyncGroup with bounded +nursery-scoped termination, async popen/`python=`/`via=` gateways with +`remote_exec` inside the user's own `trio.run`, and the frame-native via +tunnel (raw-receiver registry in the sync `ChannelFactory`; +`ChannelByteStream` is gone). + +Leftovers deliberately deferred: + +- **ChannelFile / makefile over RawChannel** — do together with the B.5 + facade (the current `ChannelFileRead` is str-based; decide there whether + a text wrapper stays for backward compat). +- rsync data plane / wheel shipping over raw channels — later. +- async ssh/socket gateways — the async `makegateway` only does + popen-style and `via=`; other transports stay on the sync host path + until B.5/B.6 need them. +- worker-side CHANNEL_EXEC on an AsyncGateway is rejected with a + RemoteError ("unsupported message") — async exec is Phase C + (`exec=task`). + +### B.5 Rebuild the sync API as a facade — DONE (commit `f932084`) + +Public surface stayed byte-for-byte; the whole sync suite passes +unchanged (508 passed). What landed, and the decisions taken: + +- One engine: `SyncBridgeGateway(AsyncGateway)` serves both coordinator + and worker; `ProtocolSession` deleted. Sync dispatch still runs the + `gateway_base` Message handlers, so the sync `Channel`/`ChannelFactory` + semantics (queues, callbacks, ENDMARKER, weakref drop) are untouched — + and remain what the worker's exec'd code sees. +- Send invariant kept: `enqueue_message` posts every frame through the + portal (one global FIFO); non-loop threads wait on the frame's + `on_written` ack (120s → OSError), the loop thread only enqueues. +- `Channel.__del__` now goes through `_send_nonblocking` → + `SyncBridgeGateway.post_message` (portal post, never waits; falls back + to `_send` for duck-typed test gateways). +- KI decision: blocking data paths (send-ack, receive, waitclose, join) + wait on `threading.Event`/`queue.Queue` — KI-interruptible as before. + Management ops (makegateway, terminate) run inside `portal.run` + (`trio.from_thread.run`) where KI is deferred; accepted for now. +- `Group.terminate` still calls member `exit()`/`join()` (contract pinned + by `test_basic_group`), with process wait/kill delegated to + `AsyncGroup.terminate` (tunneled-first, ~2×timeout bound). +- Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel + (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's + serve task never finishes and terminate hangs. +- Post-B fix (`51b9053`): the session-attach startup race — the bridge + must attach itself to the sync gateway in `__init__`, before serving + starts, or a first-message reply goes through the IO stub and kills + the fresh gateway (predated B.5; surfaced by `pytest -n 12`). The + suite now runs xdist-clean. +- `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` + classes as-is for backward compat (they only use channel send/receive). +- NOT yet retired: `ExecModel` internals and `WorkerPool` (still the + worker STATUS duck-type shape and used by `multi.safe_terminate`, + `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at + the end of the phase (B.6 or later) if the tests pinning them move. + +### B.6 Namespace split — DONE (commit `4f84335`) + +`execnet.sync` (blocking facade re-exports; top-level `execnet.*` aliases +into it), `execnet.trio` (AsyncGroup/AsyncGateway/AsyncChannel, raw +channels, stream helpers, shared serialization + errors), and +`execnet.portal` (LoopPortal/SyncReceiver). `trio`/`portal` load lazily +via module `__getattr__`, so `import execnet` still does not import the +trio event loop (pinned by `testing/test_namespaces.py`). Trio-native +protocol/integration tests already live in `testing/test_trio_gateway.py` +(memory_stream_pair + real popen gateways inside `trio.run`). + +### End-of-phase leftovers (deferred into C/D) + +- Retire `ExecModel` internals and `WorkerPool`: still pinned by + `multi.safe_terminate`, `testing/test_threadpool.py`, and conftest's + `pool` fixture, and `get_execmodel`/backend names feed Phase C's + `execmodel=` compat mapping — retire once C lands the new worker + config axes. +- ChannelFile / makefile over RawChannel (async makefile) — untouched; + the sync str-based wrappers stay for backward compat. +- rsync data plane / wheel shipping over raw channels — later. +- Async remote_exec on the worker (`exec=task`) — Phase C. + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; rough + major/minor version check (`_trio_worker._check_version`) warns. +- Wheel-on-demand for `GATEWAY_START_SUB` is a recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the sub-spec + has `python=`/`ssh=`). +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `Group.terminate(timeout)` must never hang (bounded even when kill is + stuck). + +## After Phase B (context, don't build now) + +- **C**: worker config axes `loop=thread|main` × `exec=thread|main|task` + (compat: `execmodel=thread` → loop=main+exec=thread; `main_thread_only` → + loop=thread+exec=main); `exec=task` later enables async remote_exec. +- **D**: docs + trio-surface tests, pytest-xdist verification. +- **E** (deferred): `execnet.anyio` — coordinator core on anyio, + `backend="asyncio"` knob for the facade. `FrameDecoder` and the B.4 + channel layers are IO-free by construction, so they port as-is; anyio's + `StapledByteStream` satisfies the `ByteStream` protocol structurally. diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md new file mode 100644 index 00000000..4e91d32e --- /dev/null +++ b/handoff-phase-c-worker-axes.md @@ -0,0 +1,398 @@ +# Handoff: Phase C — worker config axes (`loop=` × `exec=`) + Phase D + +For a fresh session on branch `feat/trio-host-thread-io`. Phase B (the +inversion onto the async-native Trio core) is **complete**; this doc +supersedes `handoff-phase-b-async-core.md` (kept for the B record). +Plan context lives in session memory (`trio-port-plan`), but everything +needed is restated here. + +The rework is up as **draft PR pytest-dev/execnet#422** (branch pushed +to the `origin` fork). Keep it updated as C/D land; undraft once the +docs overhaul (D.1) at least covers migration notes. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). The suite is xdist-clean: `uv run +pytest testing/ -n 12` passes (~7s) since the session-attach race fix +(`51b9053`) — keep it that way. ssh paths have a real local harness in +`testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Surface review — LANDED 2026-07-29 (`7aa17fe..e75cd0a`) + +The public surface had settled commit by commit and was never reviewed as +a whole. Doing that before the Phase D docs froze it produced the +following, which **wins over anything below or in +`handoff-boundary-protocol-rethink.md` that contradicts it**. + +**Namespaces are now one per concurrency library you drive execnet from**: +`execnet.sync` (threads; the top-level aliases), `execnet.trio`, +`execnet.aio`, `execnet.gevent`. `execnet.portal` is gone -- +`execnet._portal` plus the trio-free `execnet._boundary`. + +| was | is | why | +|---|---|---| +| `execnet.portal` public | `execnet._portal` private | it exported `Wakener`/`Mailbox`/`OneShot`/`LoopPortal` but *not* `register_wakener`, so the advertised extension point was unreachable -- and there is no plan to let third parties add event loops at all | +| Wakener registry (`register_wakener`, lazy module table) | two-branch `make_wakener(Literal["thread","gevent"])` | exactly two backends exist; every other library gets a facade | +| `wait=` spec key | gone; `Group._wait_backend`, set by the facade | it described the *caller*, which the namespace already says. Worker-side wait was always derived from the profile | +| gevent via `wait=gevent` | `execnet.gevent.Group` | symmetric with the other surfaces | +| `execmodel=` spec key | `profile=` (`execmodel=` a permanent alias) | the key selects the *worker profile*; the local execution model it was named after no longer exists | +| `Group(execmodel=)`, `set_execmodel`, `group.execmodel` | deprecated; only the remote default survives | they had no behavioural effect. **xdist passes `Group(execmodel=...)` as a keyword** -- that must keep working | +| `main_thread_only` profile | deprecated alias for `thread` | `HybridExec` already gives the first remote_exec the real main thread. Its extra behaviour (refusing a second concurrent remote_exec) was a 1s-timeout deadlock guard, now deleted | +| one `TrioHost` per `Group` | one shared `execnet.Host` per process, `Group(host=...)` to override | a host is a thread and a loop, not something groups need isolated | +| blocking inside a running loop hangs | raises, naming `execnet.aio` / `execnet.trio` | worker-side channels stay exempt: exec'd code may run its own loop | +| `aio.Group`/`Gateway`/`Channel` | `aio.AsyncGroup`/`AsyncGateway`/`AsyncChannel` | matches `execnet.trio`; swapping the import ports the code | +| `open_popen_gateway` | `open_gateway` (both async surfaces) | it always accepted any spec | + +Behaviour changes worth a changelog line: + +- a second concurrent `remote_exec` under `main_thread_only` used to close + the channel with `MAIN_THREAD_ONLY_DEADLOCK_TEXT`; it now runs on a pool + thread. `_executetask_complete`, `MAIN_THREAD_ONLY_ADMIT_TIMEOUT` and + the error text are deleted; `MainExec` became `PrimaryThreadPump`. +- `execnet.aio` cancellation is now real: a cancelled `receive` cancels + the host-side operation instead of consuming and discarding an item. + `send`/`send_eof`/`aclose`/`terminate` are shielded instead. +- the boundary carriers raise `execnet.TimeoutError`, not the builtin; + `OneShot` double-resolve is a `RuntimeError`, not an `assert`. +- `STATUS` answers both `profile` and (legacy) `execmodel`. +- **Latent livelock fixed**: `execnet.dumps` warned on *every* access, and + xdist calls it from `serialize_warning_message` -- i.e. from inside + pytest's warning-recording hook. One DeprecationWarning in a worker + therefore recorded a warning that recorded a warning, unbounded, and + wedged the run. The shim warns once per process + (`execnet._xdist_compat_warned`). Anything that warns in a worker can + hit this class of bug; keep it in mind. + +New tests: `testing/test_host.py` (sharing, explicit `Host`, fork, the +loop guards), `testing/test_boundary.py` (renamed from `test_portal.py`), +aio cancellation contracts in `testing/test_aio.py`. The `execmodel` +fixture parametrization collapsed to a single `profile` fixture, so the +suite is ~540 items rather than ~765. + +### D.3 done: the xdist suite runs in CI (2026-07-30) + +The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own +test suite against the execnet built from the branch, in two variants: + +- **`release`** — pinned `v3.8.0` + `pytest<9`, a combination verified + green against *released* execnet 2.1.2 (196 passed / 0 failed). This + one blocks: a failure means we broke something. Bump the ref and the + pytest pin together, and only after re-checking the new pair against + released execnet — xdist 3.8.0 with pytest 9 fails two of its own tests + regardless of execnet, which is why the pin exists. +- **`default-branch`** — floating, `continue-on-error`. Early warning for + drift on either side, without letting an xdist-side breakage block + execnet PRs. The `ref` is left empty so it follows xdist's default + branch, which is `master`, not `main` — worth knowing before hardcoding + a name. + +**Doing this for the first time found 16 real regressions** that our suite +could not see (it runs xdist as a *tool*, which exercises none of the +crash-replacement or report-serialization paths). Baseline for xdist's +default branch: released execnet 2.1.2 gives 220 passed / 0 failed. Two +root causes, both fixed: + +1. `makegateway` wrote the *normalized* profile back onto the caller's + XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to + decide whether it still needs the `execmodel=main_thread_only//` + prefix, so after normalization it prefixed a second time and built + `execmodel=...//execmodel=...//popen` -> `ValueError: duplicate key`. + Every crashed-worker-replacement test failed. Fix: `resolve_profile` + validates and warns but callers no longer assign its result to a + caller-owned spec; `effective_profile` (no warning) maps at the + consumption points. **Rule: filling in a missing spec value is + idempotent and fine; rewriting one the caller set is not.** +2. the `execnet.dumps` deprecation warning (see above). + +Remaining known failure, deselected via `.github/xdist-known-failures.txt` +(both jobs are green otherwise): `test_remote_inner_argv` asserts +`sys.argv == ["-c"]` and documents "the behavior due to execnet using +`python -c`" -- which the no-source-shipping worker launch deliberately +changed. That one needs an xdist PR. The deselect list is shared by both +targets; pytest ignores a `--deselect` that matches nothing, so an entry +may name a test only one xdist version has. + +Still open from the review, deliberately not done: the async surfaces have +no `remote_status()`, no `MultiChannel`, no group iteration, and no +`RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` +profile (the coordinator's shape does not dictate the worker's). + +## CLI + transports — LANDED 2026-07-30 + +The protocol used to *be* the worker's stdin/stdout. `execnet worker` now +names the transport, and the CLI is the launch contract (every launcher +emits `python -m execnet worker ...`, or `execnet worker ...` under +`uv run`). `_trio_worker` keeps the serving/profile layer; argument +parsing lives in `_cli`. + +| transport | who uses it | how | +|---|---|---| +| `--protocol-fd N` / `R,W` | popen (`transport=socket`), socketserver, `installvia` | inherited socketpair via `pass_fds`; a single fd must be a socket (checked with `S_ISSOCK`), a pipe needs the pair form | +| `--protocol-connect ADDR` | ssh / vagrant_ssh (`transport=socket`) | coordinator binds a private unix socket, `ssh -R` forwards it, the worker dials back | +| `--protocol-listen ADDR` | nothing yet — for trampolines | binds, prints `{"listening": ...}`, serves the first connection | +| `--protocol-stdio` | Windows, `transport=stdio`, and the `via` tunnel (which relays over the sub's stdio by construction) | the classic pipe pair | + +`transport=socket|stdio` is a spec key defaulting per platform +(`_provision.resolve_transport`); Windows stays on stdio because +`subprocess` rejects `pass_fds` there and `ssh -R` cannot forward a unix +socket. **Still unverified: whether the pre-existing socket/`installvia` +path (which already used `pass_fds`) works on the Windows CI job at all.** + +Things this fixed, each verified while doing it: + +- remote `print()` used to go to `/dev/null`; a worker's stdio is now the + user's, with `stdin=`/`stdout=`/`stderr=` spec keys to override. On the + stdio transport the fallback is close stdin + stdout→stderr, so output + is visible without a second stream. +- the ssh worker config carried `env:` values **in the remote argv**, + readable by every user on that host via `ps`. The socket transport + frees ssh's stdin, so it goes there (`--config-fd 0`). +- the dev-coordinator wheel was framed in-band on the protocol pipe with + `head -c `; it now travels on its own ssh connection before the + launch, cached remotely under `"$HOME"/.cache/execnet/wheels`. Two + traps found writing that: `shlex.quote("~/...")` creates a directory + literally named `~` (use `"$HOME"`), and the cached-skip branch must + still drain stdin or the coordinator gets EPIPE. + +`HostNotFound` needed rework for the dial-back: with no shared pipe there +is no EOF to observe, so `_accept_or_diagnose` races the accept against +`process.wait()` and still maps ssh's exit 255. + +The asyncssh harness in `testing/test_ssh_local.py` needed a +`server_factory` whose `unix_server_requested` returns True — asyncssh +refuses `-R` unix forwards otherwise, which is what a dial-back needs. + +Known semantic loss, documented on `HybridExec` and in the changelog: +`main_thread_only` serialized, so *every* sequential `remote_exec` got the +main thread. The `thread` profile releases its claim just after the +channel close that lets the coordinator send the next request, so an +immediate re-exec can rarely land on a pool thread. The first request is +still deterministic (FIFO admission). This surfaced as a `-n 12` flake, +not by reasoning — worth remembering that the hybrid claim is best-effort. + +## Where the repo stands (2026-07-25, after `a69b844`) + +One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API +is a facade over it — coordinator and worker sessions are +`SyncBridgeGateway` (an `AsyncGateway` subclass in `_trio_host.py`) whose +`_dispatch` runs the classic sync `Message.received` handlers under +`_receivelock`; `SyncBridgeGateway.__init__` attaches itself to the sync +gateway *before* serving starts (the fix for the startup race where a +STATUS/CHANNEL_EXEC arriving first replied through the IO stub and killed +the session). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; makegateway and bounded process shutdown delegate to +`AsyncGroup`, while `terminate` keeps the member `exit()`/`join()` +contract (pinned by `test_basic_group`). `AsyncGroup` supports every +transport (popen, `python=`, ssh, vagrant_ssh, socket with installvia, +`via=`). Public namespaces: `execnet.sync` (top-level `execnet.*` +aliases into it), `execnet.trio`, `execnet.portal` (trio/portal lazy via +module `__getattr__`; `import execnet` must not import trio — pinned by +`testing/test_namespaces.py`). + +No source is shipped over the wire: workers run +`python -m execnet._trio_worker `; foreign/remote +interpreters are uv-provisioned via `_provision.py`; dev coordinators +ship a wheel. + +File map (src/execnet/): + +| file | role | +|---|---| +| `_message.py` / `_serialize.py` / `_channel.py` / `_gateway_base.py` / `_errors.py` / `_execmodel.py` | split by concern since this table was written: wire protocol + sans-IO `FrameDecoder`; serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`); sync `Channel`/`ChannelFactory`; `BaseGateway`/`WorkerGateway`; error types; `WORKER_PROFILES` + `resolve_profile` + the deprecated `ExecModel` xdist shim. `gateway_base.py` is now only a warning shim | +| `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | +| `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | +| `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | +| `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | +| `sync.py` / `trio.py` / `aio.py` / `gevent.py` | the four public namespaces; `_host.py` holds the shared `Host`, `_portal.py`/`_boundary.py` the (private) boundary kit | +| `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | + +## Semantics that MUST survive (xdist depends on them) + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; loop + thread sends only enqueue. All sends go through one portal-posted FIFO. +- After close: `OSError("cannot send (already closed?)")`; + `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (`TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks now run in a threadpool thread driven by a per-channel + consumer *task* on the loop (RESOLVED 2026-07-26, was the "revisit in D" + open decision — moved off the loop). A slow callback no longer blocks the + reader; per-channel order is still strict, and `waitclose()` still returns + only after every callback (incl. the endmarker) has run. See "Callback + consumer tasks" below. +- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues + #43/#221). +- Sync blocking waits (send-ack, receive, waitclose, join) stay on + `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt + them; `portal.run` (KI-deferred) is only for management ops. + +## Phase C — LANDED: use-case worker profiles on `execmodel=` (2026-07-26) + +Where this conflicts with anything below or elsewhere, this section +wins. The axes framing (`loop=`/`exec=` as spec keys, a reserved +`backend=` axis) was **dropped** in the final rethink with Ronny: +different use-cases get named profiles, `execmodel=` is the public mode +key (xdist already passes it), and `wait=` is the only other public +knob. Implemented in commits `b2f43c3..cbce183`: + +| `profile=` (was `execmodel=`) | loop thread | exec'd code runs | channel | worker wait | extra deps | +|---|---|---|---|---|---| +| `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | +| ~~`main_thread_only`~~ | *deprecated 2026-07-29, aliases to `thread`* | | | | | +| `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | +| `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | + +(The coordinator-side counterpart of the last row is now `execnet.gevent`, +not `wait=gevent`.) + +Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating +to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: +PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec +serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync +bridge at all in the trio profile). Subinterpreters: future strategy +slot, not built. `WORKER_PROFILES` (`_execmodel.py`) validates +coordinator-side in makegateway, via `resolve_profile`. + +**Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) +answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the +worker config JSON and apply at startup. Coordinator bookkeeping can no +longer claim an exec slot (previously an info call could occupy the main +thread so pytest landed on a worker thread) — `rinfo_source` and the +post-start remote_exec setup block are gone. + +Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a +host task) backs makegateway / `SyncIOHandle.wait/kill` / +`Group.terminate` whenever the group's wait backend is not `thread`, so a +gevent app's management ops park only the calling greenlet. The default +`thread` backend keeps the KI-deferred `portal.run` path. + +Still decided/standing: core stays trio-only (anyio/asyncio-core port +rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd +code may start its own loop in every profile except `trio`. + +## Phase C — original gap analysis (pre-decision record, superseded) + +## Phase C — original gap analysis (pre-decision record) + +Target axes: `loop=thread|main` × `exec=thread|main|task`. +Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; +`execmodel=main_thread_only` → `loop=thread` + `exec=main`. + +1. **Spec surface**: `xspec.py` only knows `execmodel=`. Add `loop=` / + `exec=` keys, combination validation, and the compat mapping. Same + for the worker CLI config JSON (`_provision.worker_cli_arg` still + ships `{"id", "execmodel", "coordinator_version"}`). +2. **`loop=main` does not exist**: the worker always starts `TrioHost` + on a dedicated thread (`serve_popen_trio`), main thread parked in + `gateway.join()` or integrated as exec primary. The compat mapping + means plain `execmodel=thread` workers should end up with `trio.run` + on the main thread — a structural change to `_run_worker`, not a + flag. +3. **`exec=main`** is today's `main_thread_only` machinery + (`integrate_as_primary_thread` + `_executetask_complete` deadlock + guard) — keep behavior, re-home under the new naming. +4. **`exec=task` (async remote_exec)**: `CHANNEL_EXEC` on a pure + `AsyncGateway` is still rejected ("unsupported message" in + `_trio_gateway._dispatch`). Needs a task-based exec scheduler + handing exec'd code an `AsyncChannel`, preserving FIFO admission, + plus explicit cancellation semantics. +5. **STATUS / `remote_status()`** reports `execmodel`; must speak the + new axes compatibly. +6. **Retire `ExecModel`/`WorkerPool`** (deferred from B): replace + internals with plain `threading`/`queue`; deprecated shims for + `group.execmodel`/`set_execmodel`/`spec.execmodel`. Pinned today by + `multi.safe_terminate`, `testing/test_threadpool.py`, conftest's + `pool` fixture, `Channel._items` (`execmodel.queue`), and the worker + STATUS duck-type — gated on the compat mapping landing first. +7. **Wheel-on-demand for `GATEWAY_START_SUB`**: recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the + sub-spec has `python=`/`ssh=`). + +## Phase D — what is missing + +1. **Docs are entirely stale**: `doc/` still describes source + bootstrapping and execmodels; nothing on uv provisioning, the + no-source-shipping/version-compat policy, the three namespaces, or + `execnet.trio`/`execnet.portal` APIs. `doc/implnotes.rst` references + pre-inversion internals. Changelog entries for the whole branch. +2. **Trio-surface test gaps**: async ssh/socket/vagrant transport tests + (the asyncssh harness only exercises the sync facade; both share + `connect_command_worker`, so a direct `AsyncGroup` ssh test is + cheap), cancellation mid-`remote_exec`/`receive`, `send_eof` and + reconfigure against real workers, B.5 engine paths (write-ack + failure, `enqueue_message` after close). +3. **pytest-xdist verification**: run pytest-xdist's own suite against + this branch plus real `-n` smoke runs (crash/endmarker tests, + `main_thread_only` GUI case). Our own suite running `-n 12` green is + necessary but not sufficient. +4. **Open decision CLOSED (2026-07-26)**: channel callbacks moved off the + loop thread — see "Callback consumer tasks" below. +5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 + consistent XPASSes were investigated — trio's single-loop dispatch + + FIFO admission makes them pass reliably when idle, but under + sustained load `test_gateway_status_busy` (numexecuting race: + `_track_start` runs in a separately scheduled task) and + `test_popen_stderr_tracing` (capfd race) still fail, so the + `flakytest` marks stay. `test_safe_terminate2`'s xpass is CPython + dummy-thread accounting, unrelated to trio. To retire the status + marks for real: retry-poll for `numexecuting == 2` like the tests + already do for `== 0`. + +Recommended order: C.1+C.2 first (spec axes + loop placement) since the +compat mapping unblocks the ExecModel retirement, then `exec=task`; run +the xdist verification (D.3) mid-C as an early canary rather than only +at the end. + +## Callback consumer tasks (`setcallback`, LANDED 2026-07-26) + +`setcallback` no longer delivers inline on the loop thread and no longer +pins the channel in a `ChannelFactory._callback_channels` registry. Instead +`Channel.setcallback` → `BaseGateway._start_channel_consumer` → +`SyncBridgeGateway.attach_consumer`, which on the loop: + +- moves any already-buffered mailbox items into a fresh per-channel inbox + (a `trio` memory channel), diverts future raw payloads there + (`raw.set_consumer(inbox.send_nowait, on_close)`), and nulls `_mailbox`; +- starts a consumer *task* on the host root nursery (`_run_consumer`) that is + handed a **strong** reference to the sync `Channel` — so the channel's + lifecycle is now bound to the task (and to GC once the stream closes), + replacing the strong-ref registry. + +`_run_consumer` does `async for data in inbox:` and runs each callback via +`trio.to_thread.run_sync(..., limiter=host.callback_limiter)` — off the loop, +one thread of a bounded pool (`DEFAULT_CALLBACK_THREADS=40`), sequential per +channel (order preserved), concurrent across channels. A raising callback +(or a `loads_internal` failure) → `CHANNEL_CLOSE_ERROR` + local close +(`_consumer_failed`). On EOF/close the endmarker fires (shielded, bounded by +`CONSUMER_ENDMARKER_GRACE`) and the task sets `channel._consumer_done`. + +Key invariant preserved: `waitclose()` waits on `_consumer_done` (not +`_receiveclosed`) for callback channels, so it still returns only after every +callback — including the endmarker — has run (`test_waiting_for_callbacks`, +`test_channel_endmarker_callback`). Deleted along the way: `_callback`, +`_endmarker`, `_fire_endmarker`, `_callback_channels`, +`_register_callback_channel`, and the callback branch of `_deliver_payload` +(now uniformly `mailbox.put`). `__del__`'s `CHANNEL_LAST_MESSAGE` case folds +away (a callback channel is never GC'd while open). + +Stress coverage: `testing/test_channel_stress.py` (Hypothesis) with a +`--stress=N` pytest option (profiles registered in +`testing/conftest.py::pytest_configure`; default quick profile). Hypothesis +also surfaced and we fixed a latent serializer bug: `_save_integral` only +bounds-checked the upper int4 limit, so a negative int below `-2**31` +overflowed `struct.pack('!i', …)` instead of taking the long path +(`FOUR_BYTE_INT_MIN` added; regression `test_serializer.test_int_boundaries`). + +`hypothesis` was added to the `testing` extra in `pyproject.toml`. + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; + rough major/minor version check (`_trio_worker._check_version`) warns. +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `import execnet` must not import the trio event loop. +- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO + `FrameDecoder`) — `execnet.anyio` is deferred Phase E. diff --git a/pyproject.toml b/pyproject.toml index fcd2b4b5..e321346c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ description = "execnet: rapid multi-Python deployment" readme = {"file" = "README.rst", "content-type" = "text/x-rst"} license = "MIT" requires-python = ">=3.10" +dependencies = [ + "trio>=0.32", +] authors = [ { name = "holger krekel and others" }, ] @@ -34,18 +37,33 @@ classifiers = [ "Topic :: System :: Networking", ] +[project.scripts] +execnet = "execnet._cli:main" +# deprecated alias for `execnet server` +execnet-socketserver = "execnet._cli:socketserver_main" + [project.optional-dependencies] +gevent = [ + "gevent", +] testing = [ "pre-commit", "pytest>8.0", "pytest-timeout", + "hypothesis", "tox", "hatch", "uv", - "gevent", + "asyncssh", ] [dependency-groups] +dev = [ + "pytest-xdist>=3.8.0", +] +gevent = [ + "gevent>=26.7.0", +] testing = [ "execnet[testing]", ] @@ -121,8 +139,5 @@ disallow_untyped_defs = false disallow_incomplete_defs = false [[tool.mypy.overrides]] -module = [ - "eventlet.*", - "gevent.thread.*", -] +module = ["asyncssh.*"] ignore_missing_imports = true diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 1edf3254..e88baf6f 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,29 +4,46 @@ pure python lib for connecting to local and remote Python Interpreters. +One namespace per concurrency library you drive execnet from: + +* :mod:`execnet.sync` — the blocking API for plain threads; the top-level + ``execnet.*`` names below are aliases into it. +* :mod:`execnet.trio` — the trio-native API, awaited inside your own + ``trio.run``. +* :mod:`execnet.aio` — the asyncio-native API. +* :mod:`execnet.gevent` — the blocking API with greenlet-parking waits. + +:mod:`execnet.trio` is the only one that runs gateways *directly* as tasks +in your own nursery. The other three drive a shared Trio host thread, so +their blocking calls must not be made from inside a running event loop. + +``can_send`` sits here rather than on any one of them: the wire-format +contract is the same whichever surface you drive a gateway from. + (c) 2012, Holger Krekel and others """ +from typing import Any + +from ._serialize import can_send from ._version import version as __version__ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads -from .gateway_bootstrap import HostNotFound -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from .sync import Channel +from .sync import DataFormatError +from .sync import DumpError +from .sync import Gateway +from .sync import Group +from .sync import Host +from .sync import HostNotFound +from .sync import LoadError +from .sync import MultiChannel +from .sync import RemoteError +from .sync import RSync +from .sync import TimeoutError +from .sync import XSpec +from .sync import default_group +from .sync import makegateway +from .sync import set_execmodel +from .sync import set_profile __all__ = [ "Channel", @@ -34,6 +51,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", @@ -42,11 +60,58 @@ "TimeoutError", "XSpec", "__version__", + "can_send", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", + "set_profile", ] + + +#: resolved lazily so ``import execnet`` does not load the trio event loop +#: machinery, plus the deprecated pre-Trio module names -- those used to be +#: reachable here only because the import chain pulled them in, and callers +#: that still do ``execnet.gateway_base.X`` must reach the warning shim. +_LAZY_MODULES = ( + "aio", + "gevent", + "trio", + "gateway", + "gateway_base", + "multi", + "rsync", + "rsync_remote", + "xspec", +) + + +#: TEMPORARY pytest-xdist compatibility. ``xdist/remote.py`` probes +#: serializability with ``try: execnet.dumps(x) / except execnet.DumpError`` +#: before shipping warning args and report attrs. The standalone serializer +#: is internal and :func:`can_send` replaces that probe, but dropping the name +#: outright breaks every released xdist, so it stays reachable -- deliberately +#: absent from ``__all__`` and from ``dir()``. +#: +#: It does NOT warn, on purpose. xdist reaches it from +#: ``serialize_warning_message``, i.e. from inside pytest's warning-recording +#: hook and once per warning a *user's* test raises. A warning there is +#: attributed to that test, in a run the user cannot change the outcome of +#: (porting the probe is xdist's call, not theirs) -- and warning on every +#: access made recording one warning record another, unbounded, wedging the +#: run. The deprecation lives in the changelog and in the xdist issue. +#: +#: FOLLOW-UP (after the execnet release): port xdist to ``execnet.can_send``, +#: then delete this and its test. Nothing else may be added here. +_XDIST_COMPAT = ("dumps",) + + +def __getattr__(name: str) -> Any: + if name in _LAZY_MODULES: + import importlib + + return importlib.import_module(f".{name}", __name__) + if name in _XDIST_COMPAT: + import importlib + + return getattr(importlib.import_module("._serialize", __name__), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/__main__.py b/src/execnet/__main__.py new file mode 100644 index 00000000..83640e8d --- /dev/null +++ b/src/execnet/__main__.py @@ -0,0 +1,11 @@ +"""``python -m execnet`` -- the same CLI as the ``execnet`` console script. + +Provisioning emits this form for a direct interpreter launch, where the +console script's location on the target is not knowable; under ``uv run`` +the bare ``execnet`` command resolves inside the provisioned environment. +""" + +from ._cli import main + +if __name__ == "__main__": + main() diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py new file mode 100644 index 00000000..fcd4bae6 --- /dev/null +++ b/src/execnet/_boundary.py @@ -0,0 +1,230 @@ +"""Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. + +Importable without loading any event loop (``import execnet`` must not +import trio); :mod:`execnet._portal` re-exports these next to LoopPortal. + +All of this is internal. There are exactly two wait backends -- OS +threads and gevent greenlets -- and no plan to let third parties add +more: every other concurrency library gets a facade of its own +(:mod:`execnet.trio`, :mod:`execnet.aio`) rather than a wakener. +""" + +from __future__ import annotations + +import queue +import threading +import time +from typing import Any +from typing import Generic +from typing import Literal +from typing import Protocol +from typing import TypeVar +from typing import cast + +from ._errors import TimeoutError + +__all__ = [ + "Flag", + "Mailbox", + "OneShot", + "ThreadWakener", + "WaitBackend", + "Wakener", + "make_wakener", +] + +T = TypeVar("T") + +#: which primitive a facade's blocking waits park on +WaitBackend = Literal["thread", "gevent"] + + +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. + + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener). + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. + """ + + def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wakener = ThreadWakener() if wakener is None else wakener + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() + + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout + while True: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None + try: + return self._items.get_nowait() + except queue.Empty: + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + if self._done: + raise RuntimeError("OneShot already resolved") + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + if self._done: + raise RuntimeError("OneShot already resolved") + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) + + +class Flag: + """An idempotent event on a wakener: may be set any number of times. + + Each Flag owns its wakener exclusively -- sharing one wakener between + carriers would lose wakeups (another carrier's ``clear`` can swallow + this one's ``notify``). + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._flag = False + + def is_set(self) -> bool: + return self._flag + + def set(self) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._flag = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> bool: + """Block until set; returns whether the flag is set.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._flag: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + break + return self._flag + + +def make_wakener(backend: WaitBackend) -> Wakener: + """Create a fresh wakener for a wait backend. + + Each carrier owns its wakener exclusively (see :class:`Flag`), so this + always returns a new instance. gevent is imported lazily -- it is an + optional dependency of the :mod:`execnet.gevent` facade. + """ + if backend == "thread": + return ThreadWakener() + if backend == "gevent": + from ._gevent_support import GeventWakener + + return GeventWakener() + raise ValueError(f"unknown wait backend {backend!r}") diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py new file mode 100644 index 00000000..611dbe2f --- /dev/null +++ b/src/execnet/_channel.py @@ -0,0 +1,502 @@ +"""The blocking :class:`Channel` and its registry and file adapters. + +A ``Channel`` is a facade over the async core: the gateway's Trio session +diverts inbound payloads for a channel id into a :class:`~execnet._boundary.Mailbox` +(or a registered callback), and ``receive()`` deserializes at the call site, +off the loop thread. +""" + +from __future__ import annotations + +import enum +import threading +import weakref +from collections.abc import Callable +from collections.abc import Iterator +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import cast +from typing import overload + +from ._boundary import Flag +from ._boundary import Mailbox +from ._errors import RemoteError +from ._errors import TimeoutError +from ._message import Message +from ._serialize import dumps_internal +from ._serialize import loads_internal + +if TYPE_CHECKING: + from ._gateway_base import BaseGateway + + +class NoEndmarker(enum.Enum): + """Type of the "no endmarker wanted" sentinel. + + An enum rather than a bare ``object()`` so it is nameable in an + annotation: an endmarker may be *any* object, so the only thing that + distinguishes "none wanted" from a legitimate endmarker is identity, + and a second module inventing its own ``object()`` for it would be + silently wrong. ``endmarker: object | Literal[NoEndmarker.NOT_WANTED]`` + says which sentinel a caller has to hand back. + """ + + NOT_WANTED = enum.auto() + + +NO_ENDMARKER_WANTED = NoEndmarker.NOT_WANTED + +#: what an ``endmarker=`` parameter accepts: any object to deliver at the +#: end, or the sentinel meaning "do not deliver one" +Endmarker = object | Literal[NoEndmarker.NOT_WANTED] + + +class Channel: + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + _INTERNALWAKEUP = 1000 + _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; this module stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None + + def __init__(self, gateway: BaseGateway, id: int) -> None: + """:private:""" + assert isinstance(id, int) + assert not isinstance(gateway, type) + self.gateway = gateway + self.id = id + # serialized payloads (or ENDMARKER); None once a consumer is attached + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) + self._closed = False + self._receiveclosed = Flag(gateway._new_wakener()) + self._remoteerrors: list[RemoteError] = [] + + def _trace(self, *msg: object) -> None: + self.gateway._trace(self.id, *msg) + + def setcallback( + self, + callback: Callable[[Any], Any], + endmarker: Endmarker = NO_ENDMARKER_WANTED, + ) -> None: + """Set a callback function for receiving items. + + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. + """ + self.gateway._start_channel_consumer(self, callback, endmarker) + + def __repr__(self) -> str: + flag = (self.isclosed() and "closed") or "open" + return "" % (self.id, flag) + + def __del__(self) -> None: + if self.gateway is None: # can be None in tests + return # type: ignore[unreachable] + + self._trace("channel.__del__") + # no multithreading issues here, because we have the last ref to 'self' + if self._closed: + # state transition "closed" --> "deleted" + for error in self._remoteerrors: + error.warn() + elif self._receiveclosed.is_set(): + # state transition "sendonly" --> "deleted" + # the remote channel is already in "deleted" state, nothing to do + pass + else: + # state transition "opened" --> "deleted" + # check if we are in the middle of interpreter shutdown + # in which case the process will go away and we probably + # don't need to try to send a closing or last message + # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. + if Message is not None: + with suppress(OSError, ValueError): # ignore problems with sending + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(Message.CHANNEL_CLOSE, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) + + def _getremoteerror(self): + try: + return self._remoteerrors.pop(0) + except IndexError: + try: + return self.gateway._error + except AttributeError: + pass + return None + + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ + if self._closed: + return # late data for a locally closed channel: drop + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ + if remoteerror: + self._remoteerrors.append(remoteerror) + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + # + # public API for channel objects + # + def isclosed(self) -> bool: + """Return True if the channel is closed. + + A closed channel may still hold items. + """ + return self._closed + + @overload + def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: + pass + + @overload + def makefile( + self, + mode: Literal["w"] = ..., + proxyclose: bool = ..., + ) -> ChannelFileWrite: + pass + + def makefile( + self, + mode: Literal["r", "w"] = "w", + proxyclose: bool = False, + ) -> ChannelFileWrite | ChannelFileRead: + """Return a file-like object. + + mode can be 'w' or 'r' for writeable/readable files. + If proxyclose is true, file.close() will also close the channel. + """ + if mode == "w": + return ChannelFileWrite(channel=self, proxyclose=proxyclose) + elif mode == "r": + return ChannelFileRead(channel=self, proxyclose=proxyclose) + raise ValueError(f"mode {mode!r} not available") + + def close(self, error=None) -> None: + """Close down this channel with an optional error message. + + Note that closing of a channel tied to remote_exec happens + automatically at the end of execution and cannot + be done explicitly. + """ + if self._executing: + raise OSError("cannot explicitly close channel within remote_exec") + if self._closed: + self.gateway._trace(self, "ignoring redundant call to close()") + if not self._closed: + # state transition "opened/sendonly" --> "closed" + # threads warning: the channel might be closed under our feet, + # but it's never damaging to send too many CHANNEL_CLOSE messages + # however, if the other side triggered a close already, we + # do not send back a closed message. + if not self._receiveclosed.is_set(): + put = self.gateway._send + if error is not None: + put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) + else: + put(Message.CHANNEL_CLOSE, self.id) + self._trace("sent channel close message") + if isinstance(error, RemoteError): + self._remoteerrors.append(error) + self._closed = True # --> "closed" + self._receiveclosed.set() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) + + def waitclose(self, timeout: float | None = None) -> None: + """Wait until this channel is closed (or the remote side + otherwise signalled that no more data was being sent). + + The channel may still hold receiveable items, but not receive + any more after waitclose() has returned. + + Exceptions from executing code on the other side are reraised as local + channel.RemoteErrors. + + EOFError is raised if the reading-connection was prematurely closed, + which often indicates a dying process. + + self.TimeoutError is raised after the specified number of seconds + (default is None, i.e. wait indefinitely). + """ + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + self.gateway._check_event_loop("channel.waitclose()") + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): + raise self.TimeoutError("Timeout after %r seconds" % timeout) + error = self._getremoteerror() + if error: + raise error + + def send(self, item: object) -> None: + """Sends the given item to the other side of the channel, + possibly blocking if the sender queue is full. + + The item must be a simple Python type and will be + copied to the other side by value. + + OSError is raised if the write pipe was prematurely closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + self.gateway._check_event_loop("channel.send()") + self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) + + def receive(self, timeout: float | None = None) -> Any: + """Receive a data item that was sent from the other side. + + timeout: None [default] blocked waiting. A positive number + indicates the number of seconds after which a channel.TimeoutError + exception will be raised if no item was received. + + Note that exceptions from the remotely executing code will be + reraised as channel.RemoteError exceptions containing + a textual representation of the remote traceback. + """ + mailbox = self._mailbox + if mailbox is None: + raise OSError("cannot receive(), channel has receiver callback") + self.gateway._check_event_loop("channel.receive()") + x = mailbox.get(timeout) + if x is ENDMARKER: + mailbox.put(x) # for other receivers + raise self._getremoteerror() or EOFError() + else: + return loads_internal(x, self) + + def __iter__(self) -> Iterator[Any]: + return self + + def next(self) -> Any: + try: + return self.receive() + except EOFError: + raise StopIteration from None + + __next__ = next + + +ENDMARKER = object() + + +class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. + """ + + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: + self._channels: weakref.WeakValueDictionary[int, Channel] = ( + weakref.WeakValueDictionary() + ) + self._writelock = threading.Lock() + self.gateway = gateway + self.count = startcount + self.finished = False + self._list = list # needed during interp-shutdown + + def new(self, id: int | None = None) -> Channel: + """Create a new Channel with 'id' (or create new id if None).""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + if id is None: + id = self.count + self.count += 2 + try: + channel = self._channels[id] + except KeyError: + channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) + return channel + + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def channels(self) -> list[Channel]: + return self._list(self._channels.values()) + + # + # internal methods, called from the loop thread (or local close paths) + # + def _no_longer_opened(self, id: int) -> None: + self._channels.pop(id, None) + + def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + """Close ``id`` as if the peer had closed it (no message is sent).""" + channel = self._channels.get(id) + if channel is None: + # channel already in "deleted" state + if remoteerror: + remoteerror.warn() + self._no_longer_opened(id) + else: + channel._close_from_remote(remoteerror, sendonly=sendonly) + + def _finished_receiving(self) -> None: + with self._writelock: + self.finished = True + for id in self._list(self._channels): + self._local_close(id, sendonly=True) + + +class ChannelFile: + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + self.channel = channel + self._proxyclose = proxyclose + + def isatty(self) -> bool: + return False + + def close(self) -> None: + if self._proxyclose: + self.channel.close() + + def __repr__(self) -> str: + state = (self.channel.isclosed() and "closed") or "open" + return "" % (self.channel.id, state) + + +class ChannelFileWrite(ChannelFile): + def write(self, out: bytes) -> None: + self.channel.send(out) + + def flush(self) -> None: + pass + + +class ChannelFileRead(ChannelFile): + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + super().__init__(channel, proxyclose) + self._buffer: str | None = None + + def read(self, n: int) -> str: + try: + if self._buffer is None: + self._buffer = cast(str, self.channel.receive()) + while len(self._buffer) < n: + self._buffer += cast(str, self.channel.receive()) + except EOFError: + self.close() + if self._buffer is None: + ret = "" + else: + ret = self._buffer[:n] + self._buffer = self._buffer[n:] + return ret + + def readline(self) -> str: + if self._buffer is not None: + i = self._buffer.find("\n") + if i != -1: + return self.read(i + 1) + line = self.read(len(self._buffer) + 1) + else: + line = self.read(1) + while line and line[-1] != "\n": + c = self.read(1) + if not c: + break + line += c + return line diff --git a/src/execnet/_cli.py b/src/execnet/_cli.py new file mode 100644 index 00000000..0b4e86cf --- /dev/null +++ b/src/execnet/_cli.py @@ -0,0 +1,266 @@ +"""The ``execnet`` command line. + +Three subcommands, reachable both as the ``execnet`` console script and as +``python -m execnet`` (the latter is what provisioning emits for a direct +interpreter launch, where the script's location is not knowable):: + + execnet worker ... # serve one gateway over a chosen transport + execnet server ... # accept gateway connections on a socket + execnet info # what this interpreter's execnet can do, as JSON + +``execnet worker`` is the launch contract between a coordinator and the +process it starts. Naming the protocol transport explicitly is what makes +ssh socket redirects and trampoline processes expressible: the protocol no +longer has to be the process's stdin/stdout, so a worker's stdio can belong +to the code it runs. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any + +__all__ = ["main"] + +#: what the worker may do with each standard fd once the protocol has a +#: transport of its own. ``stderr`` for stdout points fd 1 at fd 2, which +#: keeps remote output visible without needing a second stream. +STDIN_DISPOSITIONS = ("inherit", "close", "devnull") +STDOUT_DISPOSITIONS = ("inherit", "devnull", "stderr") +STDERR_DISPOSITIONS = ("inherit", "devnull") + + +def _protocol_fd(value: str) -> tuple[int, ...]: + """``N`` (a bidirectional socket) or ``R,W`` (a pipe pair).""" + try: + fds = tuple(int(part) for part in value.split(",")) + except ValueError: + raise argparse.ArgumentTypeError( + f"expected FD or READFD,WRITEFD, got {value!r}" + ) from None + if len(fds) not in (1, 2): + raise argparse.ArgumentTypeError( + f"expected FD or READFD,WRITEFD, got {value!r}" + ) + return fds + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="execnet", + description="Serve and inspect execnet gateways.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + worker = sub.add_parser( + "worker", + help="serve a single gateway to the coordinator that launched us", + description=( + "Serve one gateway over the given protocol transport. Normally" + " launched by a coordinator, not by hand." + ), + ) + transport = worker.add_mutually_exclusive_group() + transport.add_argument( + "--protocol-stdio", + dest="protocol", + action="store_const", + const=("stdio", None), + help="run the protocol over this process's stdin/stdout (default)", + ) + transport.add_argument( + "--protocol-fd", + metavar="FD[,FD]", + type=_protocol_fd, + help="run the protocol over an inherited socket fd, or pipe read,write pair", + ) + transport.add_argument( + "--protocol-connect", + metavar="ADDR", + help="dial out to ADDR (unix:/path or host:port) and serve there", + ) + transport.add_argument( + "--protocol-listen", + metavar="ADDR", + help="listen on ADDR (unix:/path or host:port) for one connection", + ) + + config = worker.add_mutually_exclusive_group() + config.add_argument("--config", metavar="JSON", help="worker config as JSON") + config.add_argument( + "--config-fd", + metavar="FD", + type=int, + help="read the worker config as JSON from FD until EOF", + ) + config.add_argument( + "--config-file", metavar="PATH", help="read the worker config as JSON from PATH" + ) + + worker.add_argument( + "--stdin", choices=STDIN_DISPOSITIONS, default=None, help="what to do with fd 0" + ) + worker.add_argument( + "--stdout", + choices=STDOUT_DISPOSITIONS, + default=None, + help="what to do with fd 1", + ) + worker.add_argument( + "--stderr", + choices=STDERR_DISPOSITIONS, + default=None, + help="what to do with fd 2", + ) + worker.set_defaults(func=_run_worker) + + server = sub.add_parser( + "server", + help="accept gateway connections on a socket", + description=( + "Listen for coordinator connections and hand each one to a fresh" + " worker subprocess. No code is executed in this process." + ), + ) + server.add_argument( + "hostport", + nargs="?", + default=":8888", + help="address to bind as HOST:PORT or :PORT (default: :8888)", + ) + server.add_argument( + "--once", + action="store_true", + help="serve a single connection and exit instead of looping", + ) + server.set_defaults(func=_run_server) + + info = sub.add_parser( + "info", + help="report this execnet's version and capabilities as JSON", + description=( + "Print a JSON object describing this interpreter's execnet." + " Coordinators use it to decide whether a target interpreter can" + " host a worker directly or has to be provisioned." + ), + ) + info.set_defaults(func=_run_info) + + return parser + + +def _load_config(ns: argparse.Namespace) -> dict[str, Any]: + """The worker config, from whichever source was named. + + ``--config-fd`` is what a coordinator should use for anything sensitive: + a config passed in argv is visible in ``ps`` to every user on the host, + and it carries ``env:`` values. + """ + if ns.config is not None: + raw = ns.config + elif ns.config_fd is not None: + # read a dup, so ``--config-fd 0`` leaves fd 0 itself open (at EOF). + # Closing it would free the slot for the next os.open, and anything + # then writing to "stdin" would land in an unrelated file. + with os.fdopen(os.dup(ns.config_fd), "r", encoding="utf-8") as stream: + raw = stream.read() + elif ns.config_file is not None: + with open(ns.config_file, encoding="utf-8") as stream: + raw = stream.read() + else: + raise SystemExit("execnet worker: one of --config/--config-fd/--config-file") + config: dict[str, Any] = json.loads(raw) + return config + + +def _run_worker(ns: argparse.Namespace) -> None: + from . import _trio_worker + + if ns.protocol_fd is not None: + transport: _trio_worker.Transport = _trio_worker.FdTransport(ns.protocol_fd) + elif ns.protocol_connect is not None: + transport = _trio_worker.ConnectTransport(ns.protocol_connect) + elif ns.protocol_listen is not None: + transport = _trio_worker.ListenTransport(ns.protocol_listen) + else: + transport = _trio_worker.StdioTransport() + # The stdio transport owns fd 0/1, so it has to claim them before the + # config is read (--config-fd 0 would be the very fd we are moving). + transport.prepare() + config = _load_config(ns) + _trio_worker.serve_worker( + config, + transport, + stdin=ns.stdin, + stdout=ns.stdout, + stderr=ns.stderr, + ) + + +def _run_server(ns: argparse.Namespace) -> None: + import trio + + from . import _socketserver + + trio.run(_socketserver.serve, ns.hostport, ns.once) + + +def _run_info(ns: argparse.Namespace) -> None: + json.dump(interpreter_info(), sys.stdout) + sys.stdout.write("\n") + + +def interpreter_info() -> dict[str, Any]: + """What a coordinator needs to know before launching a worker here. + + Distinct from ``_message.gateway_info``, which answers the in-protocol + ``GATEWAY_INFO`` request on an established gateway; this one is what a + coordinator can learn *before* connecting. + """ + from ._version import version + + try: + import trio + + trio_version: str | None = trio.__version__ + except Exception: + trio_version = None + + return { + "execnet": version, + "trio": trio_version, + "python": ".".join(str(part) for part in sys.version_info[:3]), + "executable": sys.executable, + "platform": sys.platform, + "protocols": _supported_protocols(), + } + + +def _supported_protocols() -> list[str]: + """Transports this platform can actually serve.""" + protocols = ["stdio", "listen", "connect"] + if hasattr(os, "dup"): + protocols.append("fd") + return sorted(protocols) + + +def main(argv: list[str] | None = None) -> None: + """Console entry point (``execnet``) and ``python -m execnet``.""" + parser = _build_parser() + ns = parser.parse_args(argv) + ns.func(ns) + + +def socketserver_main(argv: list[str] | None = None) -> None: + """Deprecated ``execnet-socketserver`` entry point; use ``execnet server``.""" + import warnings + + warnings.warn( + "execnet-socketserver is deprecated; use `execnet server` instead.", + DeprecationWarning, + stacklevel=2, + ) + main(["server", *(sys.argv[1:] if argv is None else argv)]) diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py new file mode 100644 index 00000000..276fbb5e --- /dev/null +++ b/src/execnet/_errors.py @@ -0,0 +1,106 @@ +"""Exception types and the error texts that cross the wire. + +The channel-facing errors (:class:`RemoteError`, :class:`TimeoutError`, +:class:`HostNotFound`, and the :class:`DataFormatError` family) are the one +part of this module that is public: every namespace -- :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` -- re-exports them, because the same +errors are raised whichever surface you drive a gateway from. +""" + +from __future__ import annotations + +import os +import sys +import traceback + +#: exceptions that must never be swallowed by a broad ``except`` +sysex = (KeyboardInterrupt, SystemExit) + +INTERRUPT_TEXT = "keyboard-interrupted" + + +class GatewayReceivedTerminate(Exception): + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" + + +def geterrortext( + exc: BaseException, + format_exception=traceback.format_exception, + sysex: tuple[type[BaseException], ...] = sysex, +) -> str: + try: + # In py310, can change this to: + # l = format_exception(exc) + l = format_exception(type(exc), exc, exc.__traceback__) + errortext = "".join(l) + except sysex: + raise + except BaseException: + errortext = f"{type(exc).__name__}: {exc}" + return errortext + + +class RemoteError(Exception): + """Exception containing a stringified error from the other side.""" + + def __init__(self, formatted: str) -> None: + super().__init__() + self.formatted = formatted + + def __str__(self) -> str: + return self.formatted + + def __repr__(self) -> str: + return f"{self.__class__.__name__}: {self.formatted}" + + def warn(self) -> None: + if self.formatted != INTERRUPT_TEXT: + # XXX do this better + sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") + + +class TimeoutError(IOError): + """Exception indicating that a timeout was reached.""" + + +class DataFormatError(Exception): + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ + + +class DumpError(DataFormatError): + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ + + +class LoadError(DataFormatError): + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ diff --git a/src/execnet/_exec_source.py b/src/execnet/_exec_source.py new file mode 100644 index 00000000..10c55795 --- /dev/null +++ b/src/execnet/_exec_source.py @@ -0,0 +1,87 @@ +"""Normalize ``remote_exec`` sources (string / function / module) to code. + +Shared by the sync coordinator ``Gateway`` and the trio-native +``AsyncGateway`` so both accept the same source kinds with identical +restrictions (pure functions taking ``channel`` first, no closures, no +non-builtin globals). +""" + +from __future__ import annotations + +import inspect +import linecache +import textwrap +import types +from collections.abc import Callable + + +def normalize_exec_source( + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + kwargs: dict[str, object], +) -> tuple[str, str | None, str | None]: + """Return ``(source, file_name, call_name)`` for a CHANNEL_EXEC payload.""" + call_name = None + file_name = None + if isinstance(source, types.ModuleType): + file_name = inspect.getsourcefile(source) + linecache.updatecache(file_name) # type: ignore[arg-type] + source = inspect.getsource(source) + elif isinstance(source, types.FunctionType): + call_name = source.__name__ + file_name = inspect.getsourcefile(source) + source = _source_of_function(source) + else: + source = textwrap.dedent(str(source)) + + if not call_name and kwargs: + raise TypeError("can't pass kwargs to non-function remote_exec") + return source, file_name, call_name + + +def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: + import ast + import builtins + + vars = dict.fromkeys(codeobj.co_varnames) + return [ + node.id + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Name) + and node.id not in vars + and node.id not in builtins.__dict__ + ] + + +def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: + if function.__name__ == "": + raise ValueError("can't evaluate lambda functions'") + # XXX: we dont check before remote instantiation + # if arguments are used properly + try: + sig = inspect.getfullargspec(function) + except AttributeError: + args = inspect.getargspec(function)[0] + else: + args = sig.args + if not args or args[0] != "channel": + raise ValueError("expected first function argument to be `channel`") + + closure = function.__closure__ + codeobj = function.__code__ + + if closure is not None: + raise ValueError("functions with closures can't be passed") + + try: + source = inspect.getsource(function) + except OSError as e: + raise ValueError("can't find source file for %s" % function) from e + + source = textwrap.dedent(source) # just for inner functions + + used_globals = _find_non_builtin_globals(source, codeobj) + if used_globals: + raise ValueError("the use of non-builtin globals isn't supported", used_globals) + + leading_ws = "\n" * (codeobj.co_firstlineno - 1) + return leading_ws + source diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py new file mode 100644 index 00000000..7c25448c --- /dev/null +++ b/src/execnet/_execmodel.py @@ -0,0 +1,146 @@ +"""Worker profiles, and the deprecated ExecModel shim. + +The machinery behind "execution models" was retired during the Trio port. +What the name actually selected -- where exec'd code runs relative to the +worker's protocol loop -- survives as the ``profile=`` spec key and +:data:`WORKER_PROFILES`. + +:class:`ExecModel` itself is kept only because pytest-xdist's remote worker +builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. +""" + +from __future__ import annotations + +import os +import threading +import warnings + + +class ExecModel: + """Deprecated preset name for an execution model. + + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ + + def __init__(self, backend: str) -> None: + self.backend = backend + + def __repr__(self) -> str: + return "" % self.backend + + @property + def queue(self): + import queue + + return queue + + @property + def subprocess(self): + import subprocess + + return subprocess + + @property + def socket(self): + import socket + + return socket + + def get_ident(self) -> int: + import _thread + + return _thread.get_ident() + + def sleep(self, delay: float) -> None: + import time + + time.sleep(delay) + + def start(self, func, args=()) -> None: + import _thread + + _thread.start_new_thread(func, args) + + def fdopen(self, fd, mode, bufsize=1, closefd=True): + return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) + + def Lock(self): + return threading.RLock() + + def RLock(self): + return threading.RLock() + + def Event(self) -> threading.Event: + return threading.Event() + + +#: worker profiles: where exec'd code runs relative to the protocol loop +WORKER_PROFILES = ( + "thread", # hybrid: primary on the main thread, overflow on pool threads + "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec +) + + +#: profiles kept as accepted spellings, mapped to what they now select. +#: ``main_thread_only`` predates the restored hybrid ``thread`` profile, +#: which already hands the first remote_exec the real main thread -- the +#: GUI/signal property it existed for. What it additionally did was refuse +#: a *second* concurrent remote_exec instead of overflowing to a pool +#: thread; that guard is gone. +DEPRECATED_PROFILES = {"main_thread_only": "thread"} + + +def effective_profile(name: str) -> str: + """Validate a profile name and map deprecated spellings, silently. + + For the places that *consume* a profile (worker config, strategy + lookup). :func:`resolve_profile` is the one that warns, and is called + once where the value enters. + """ + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + return replacement + if name not in WORKER_PROFILES: + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") + return name + + +def resolve_profile(name: str) -> str: + """Validate a ``profile=`` value, warning about deprecated spellings. + + Callers that hold a caller-supplied :class:`~execnet._xspec.XSpec` must + *not* write the result back onto it: pytest-xdist reuses a spec object + across gateways and re-reads ``spec.execmodel`` to decide whether it + still needs prefixing, so normalizing the value it set makes the second + use build a spec with a duplicate key. Validate here, and map with + :func:`effective_profile` where the value is used. + """ + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + warnings.warn( + f"the {name!r} worker profile is deprecated and now behaves like" + f" {replacement!r}, which already runs the first remote_exec on" + " the worker's main thread. A second concurrent remote_exec no" + " longer fails -- it runs on a pool thread.", + DeprecationWarning, + stacklevel=3, + ) + return replacement + if name not in WORKER_PROFILES: + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") + return name + + +def get_execmodel(backend: str | ExecModel) -> ExecModel: + """Deprecated: build the xdist-facing shim for a profile name.""" + if isinstance(backend, ExecModel): + return backend + if backend in WORKER_PROFILES or backend in DEPRECATED_PROFILES: + return ExecModel(backend) + raise ValueError(f"unknown profile {backend!r}") diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py new file mode 100644 index 00000000..8f992e09 --- /dev/null +++ b/src/execnet/_gateway.py @@ -0,0 +1,163 @@ +"""Gateway code for initiating popen, socket and ssh connections. + +(c) 2004-2013, Holger Krekel and others +""" + +from __future__ import annotations + +import types +import warnings +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any + +from ._channel import Channel +from ._exec_source import normalize_exec_source +from ._gateway_base import BaseGateway +from ._message import IO +from ._message import Message +from ._multi import Group +from ._serialize import dumps_internal +from ._xspec import XSpec + +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", +] + + +class Gateway(BaseGateway): + """Gateway to a local or remote Python Interpreter.""" + + _group: Group + _guard_event_loop = True + + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ + super().__init__(io=io, id=spec.id, _startcount=1) + self.spec = spec + + @property + def remoteaddress(self) -> str: + # Only defined for remote IO types. + return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] + + def __repr__(self) -> str: + """A string representing gateway type and status.""" + try: + r: str = (self.hasreceiver() and "receive-live") or "not-receiving" + i = str(len(self._channelfactory.channels())) + except AttributeError: + r = "uninitialized" + i = "no" + return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.spec.profile} profile, {i} active channels>" + + def exit(self) -> None: + """Trigger gateway exit. + + Defer waiting for finishing of receiver-thread and subprocess activity + to when group.terminate() is called. + """ + self._trace("gateway.exit() called") + if self not in self._group: + self._trace("gateway already unregistered with group") + return + self._group._unregister(self) + try: + self._trace("--> sending GATEWAY_TERMINATE") + self._send(Message.GATEWAY_TERMINATE) + self._trace("--> io.close_write") + self._io.close_write() + except (ValueError, EOFError, OSError) as exc: + self._trace("io-error: could not send termination sequence") + self._trace(" exception: %r" % exc) + + def _rinfo(self, update: bool = False) -> RInfo: + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ + if update or not hasattr(self, "_cache_rinfo"): + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return self._cache_rinfo + + def hasreceiver(self) -> bool: + """Whether gateway is able to receive data.""" + session = self._trio_session + return session is not None and bool(session.is_alive()) + + def remote_status(self) -> RemoteStatus: + """Obtain information about the remote execution status.""" + channel = self.newchannel() + self._send(Message.STATUS, channel.id) + statusdict = channel.receive() + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return RemoteStatus(statusdict) + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Return channel object and connect it to a remote + execution thread where the given ``source`` executes. + + * ``source`` is a string: execute source string remotely + with a ``channel`` put into the global namespace. + * ``source`` is a pure function: serialize source and + call function with ``**kwargs``, adding a + ``channel`` object to the keyword arguments. + * ``source`` is a pure module: execute source of module + with a ``channel`` in its global namespace. + + In all cases the binding ``__name__='__channelexec__'`` + will be available in the global namespace of the remotely + executing code. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.newchannel() + self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + + def remote_init_threads(self, num: int | None = None) -> None: + """DEPRECATED. Is currently a NO-OPERATION already.""" + warnings.warn( + "remote_init_threads() has been a no-operation since execnet 1.2" + " and will be removed; drop the call.", + DeprecationWarning, + stacklevel=2, + ) + + +class RInfo: + def __init__(self, kwargs) -> None: + self.__dict__.update(kwargs) + + def __repr__(self) -> str: + info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) + return "" % info + + if TYPE_CHECKING: + + def __getattr__(self, name: str) -> Any: ... + + +RemoteStatus = RInfo diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py new file mode 100644 index 00000000..773ba314 --- /dev/null +++ b/src/execnet/_gateway_base.py @@ -0,0 +1,243 @@ +"""The gateway base classes shared by coordinator and worker. + +:class:`BaseGateway` owns the channel factory, the send path and the handle on +the Trio session that does the actual Message IO; :class:`WorkerGateway` adds +the worker-side ``remote_exec`` scheduling and shutdown. + +:copyright: 2004-2015 +:authors: + - Holger Krekel + - Armin Rigo + - Benjamin Peterson + - Ronny Pfannschmidt + - many others +""" + +from __future__ import annotations + +import os +import sys +from _thread import interrupt_main +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +from ._boundary import WaitBackend +from ._boundary import Wakener +from ._boundary import make_wakener +from ._channel import Channel +from ._channel import ChannelFactory +from ._channel import Endmarker +from ._errors import INTERRUPT_TEXT +from ._errors import geterrortext +from ._errors import sysex +from ._message import IO +from ._message import Message +from ._trace import trace + + +class BaseGateway: + _sysex = sysex + id = "" + _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None + #: which primitive this gateway's blocking waits park on (channels, + #: write-acks, join). Inherited from the facade coordinator-side, and + #: derived from the worker profile worker-side. + _wait_backend: WaitBackend = "thread" + #: whether blocking operations refuse to run inside a foreign event + #: loop. Only coordinator-side: exec'd code in a worker may legitimately + #: run its own loop and talk to its channel from inside it. + _guard_event_loop = False + + def _check_event_loop(self, what: str) -> None: + if self._guard_event_loop: + from ._host import check_not_in_event_loop + + check_not_in_event_loop(what) + + def __init__(self, io: IO, id, _startcount: int = 2) -> None: + self.execmodel = io.execmodel + self._io = io + self.id = id + self._channelfactory = ChannelFactory(self, _startcount) + # globals may be NONE at process-termination + self.__trace = trace + self._geterrortext = geterrortext + self._trio_session = None + + def _trace(self, *msg: object) -> None: + self.__trace(self.id, *msg) + + def _attach_trio_session(self, session: Any) -> None: + """Attach the Trio bridge session doing the Message IO.""" + self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier.""" + return make_wakener(self._wait_backend) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) + + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: Endmarker, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + + def _terminate_execution(self) -> None: + pass + + def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return + try: + message.to_io(self._io) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + # ValueError might be because the IO is already closed + raise OSError("cannot send (already closed?)") from e + + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + channel.close("execution disallowed") + + # _____________________________________________________________________ + # + # High Level Interface + # _____________________________________________________________________ + # + def newchannel(self) -> Channel: + """Return a new independent channel.""" + return self._channelfactory.new() + + def join(self, timeout: float | None = None) -> None: + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + + +class WorkerGateway(BaseGateway): + _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") + return + trio_exec.schedule(channel, sourcetask) + + def _terminate_execution(self) -> None: + # called from receiverthread + self._trace("shutting down execution pool") + self._execpool.trigger_shutdown() + if not self._execpool.waitall(5.0): + self._trace("execution ongoing after 5 secs, trying interrupt_main") + # We try hard to terminate execution based on the assumption + # that there is only one gateway object running per-process. + if sys.platform != "win32": + self._trace("sending ourselves a SIGINT") + os.kill(os.getpid(), 2) # send ourselves a SIGINT + elif interrupt_main is not None: + self._trace("calling interrupt_main()") + interrupt_main() + if not self._execpool.waitall(10.0): + self._trace( + "execution did not finish in another 10 secs, calling os._exit()" + ) + os._exit(1) + + def executetask( + self, + item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], + ) -> None: + try: + channel, (source, file_name, call_name, kwargs) = item + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") + channel._executing = True + try: + co = compile(source + "\n", file_name or "", "exec") + exec(co, loc) + if call_name: + self._trace("calling %s(**%60r)" % (call_name, kwargs)) + function = loc[call_name] + function(channel, **kwargs) + finally: + channel._executing = False + self._trace("execution finished") + except KeyboardInterrupt: + channel.close(INTERRUPT_TEXT) + raise + except EOFError: + self._trace("ignoring EOFError because receiving finished") + + except BaseException as exc: + if not channel.gateway._channelfactory.finished: + self._trace(f"got exception: {exc!r}") + errortext = self._geterrortext(exc) + channel.close(errortext) + return + channel.close() diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py new file mode 100644 index 00000000..17cb7993 --- /dev/null +++ b/src/execnet/_gevent_support.py @@ -0,0 +1,71 @@ +"""The gevent wait backend behind :mod:`execnet.gevent`. + +The boundary kit imports this lazily (``make_wakener("gevent")``) so +gevent stays an optional dependency. + +A blocking wait parks only the calling greenlet: the carrier's +event lives in the waiting greenlet's hub, and the loop side's +``notify()`` crosses threads through a ``loop.async_`` watcher -- the +one libev/libuv primitive gevent documents as safe to use from other +threads. +""" + +from __future__ import annotations + +import threading +from typing import Any + +import gevent.event +from gevent.hub import get_hub + + +class GeventWakener: + """Wakener parking greenlets instead of OS threads. + + ``notify()`` is thread-safe and non-blocking (called from the trio + host thread). The hub-bound pieces (event + async watcher) are + created lazily in the first waiter's hub, so construction is safe + from any thread -- including the host loop, which creates channels + for inbound ids. Waiters must share one hub (one gevent thread per + carrier), which is the ordinary gevent setup. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._notified = False + self._event: gevent.event.Event | None = None + self._watcher: Any = None + + def notify(self) -> None: + with self._lock: + self._notified = True + watcher = self._watcher + if watcher is not None: + watcher.send() + + def _ensure(self) -> gevent.event.Event: + """Create the hub-bound event/watcher in the waiter's context.""" + event = self._event + if event is None: + event = gevent.event.Event() + watcher = get_hub().loop.async_() + watcher.start(event.set) + with self._lock: + self._event = event + self._watcher = watcher + return event + + def wait(self, timeout: float | None = None) -> bool: + event = self._ensure() + with self._lock: + # a notify may have fired before the watcher existed + if self._notified and not event.is_set(): + event.set() + return bool(event.wait(timeout)) + + def clear(self) -> None: + with self._lock: + self._notified = False + event = self._event + if event is not None: + event.clear() diff --git a/src/execnet/_host.py b/src/execnet/_host.py new file mode 100644 index 00000000..e57511d8 --- /dev/null +++ b/src/execnet/_host.py @@ -0,0 +1,188 @@ +"""The Trio host thread that the blocking surfaces drive. + +:mod:`execnet.trio` runs gateways *directly*, as tasks in the caller's own +nursery. Every other surface -- :mod:`execnet.sync`, :mod:`execnet.aio`, +:mod:`execnet.gevent` -- has no loop of its own to put them on, so protocol +IO runs on a :class:`Host`: one OS thread running ``trio.run``. + +There is one shared host per process by default, because a host is a +thread and a trio loop, not a resource groups need isolated from each +other. Pass an explicit :class:`Host` when you do want isolation or +deterministic teardown:: + + with execnet.Host() as host: + group = execnet.Group(host=host) + ... + # the thread is joined here, rather than at interpreter exit + +This module stays free of ``import trio`` so ``import execnet`` does not +load the event loop machinery: the real :class:`~execnet._trio_host.TrioHost` +is built on first use. +""" + +from __future__ import annotations + +import atexit +import os +import sys +import threading +from types import TracebackType +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from typing_extensions import Self + +__all__ = ["Host", "check_not_in_event_loop", "default_host"] + +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 + + +def _running_event_loop() -> str | None: + """``"asyncio"`` / ``"trio"`` when called from inside one, else None. + + Both checks go through ``sys.modules`` first, so a program that never + imported asyncio or trio pays two dict lookups. asyncio is probed with + the private ``_get_running_loop`` because it *returns* None rather than + raising -- this sits in front of every blocking channel operation, and + building an exception per send is not free. + """ + asyncio = sys.modules.get("asyncio") + if asyncio is not None: + get_running = getattr(asyncio.events, "_get_running_loop", None) + if get_running is not None: + if get_running() is not None: + return "asyncio" + else: # pragma: no cover - every supported CPython has the private one + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + return "asyncio" + trio = sys.modules.get("trio") + if trio is not None: + try: + trio.lowlevel.current_trio_token() + except RuntimeError: + pass + else: + return "trio" + return None + + +#: which namespace to point a caller at, per detected loop +_SURFACE_FOR_LOOP = { + "asyncio": "execnet.aio (AsyncGroup)", + "trio": "execnet.trio (AsyncGroup)", +} + + +def check_not_in_event_loop(what: str) -> None: + """Raise if ``what`` is about to block a running event loop's thread. + + The blocking surfaces park the calling thread on a wakener, which + inside a running loop stalls every task on it -- usually as a hang, so + it is worth turning into an error that names the right namespace. + """ + loop = _running_event_loop() + if loop is None: + return + raise RuntimeError( + f"{what} blocks the calling thread and you are inside a running" + f" {loop} event loop, which would stall every task on it." + f" Use {_SURFACE_FOR_LOOP[loop]} instead, or run this in a" + " worker thread." + ) + + +class Host: + """A Trio event loop on a dedicated thread, shared by gateway groups. + + Starting is lazy: constructing a Host costs nothing, and the thread + comes up when a group first needs it. + """ + + def __init__( + self, + name: str = "execnet-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: + self.name = name + self.callback_threads = callback_threads + self._lock = threading.Lock() + self._trio_host: Any = None + + def __repr__(self) -> str: + state = "running" if self._trio_host is not None else "idle" + return f"" + + @property + def running(self) -> bool: + return self._trio_host is not None + + def _ensure_started(self) -> Any: + """The started :class:`~execnet._trio_host.TrioHost` (internal).""" + with self._lock: + if self._trio_host is None: + from . import _trio_host + + trio_host = _trio_host.TrioHost( + name=self.name, callback_threads=self.callback_threads + ) + trio_host.start() + self._trio_host = trio_host + return self._trio_host + + def close(self, timeout: float | None = 5.0) -> None: + """Stop the loop and join the thread (no-op when not running). + + Gateways served by this host must already be terminated; closing + does not terminate them for you. + """ + with self._lock: + trio_host, self._trio_host = self._trio_host, None + if trio_host is not None: + trio_host.stop(timeout=timeout) + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + +_default_lock = threading.Lock() +_default: Host | None = None +#: the pid _default was created in; a forked child inherits a dead thread +_default_pid: int | None = None + + +def default_host() -> Host: + """The process-wide host, started lazily and stopped at interpreter exit. + + After ``os.fork()`` the child inherits a Host whose thread does not + exist there, so the first use in a child builds a fresh one. + """ + global _default, _default_pid + with _default_lock: + pid = os.getpid() + if _default is None or _default_pid != pid: + _default = Host() + _default_pid = pid + return _default + + +def _close_default_atexit() -> None: + host = _default + if host is not None and _default_pid == os.getpid(): + host.close(timeout=1.0) + + +atexit.register(_close_default_atexit) diff --git a/src/execnet/_message.py b/src/execnet/_message.py new file mode 100644 index 00000000..4d31339f --- /dev/null +++ b/src/execnet/_message.py @@ -0,0 +1,174 @@ +"""The wire protocol: IO protocols, Message framing and its decoder. + +A frame is a 9-byte header (``!bii``: message code, channel id, payload +length) followed by the payload. Nothing here dispatches -- routing lives in +``AsyncGateway._dispatch`` and the sync bridge session; this module only packs, +unpacks and buffers. +""" + +from __future__ import annotations + +import os +import struct +import sys +from collections.abc import Iterator +from typing import TYPE_CHECKING +from typing import Protocol + +if TYPE_CHECKING: + from ._execmodel import ExecModel + + +class WriteIO(Protocol): + def write(self, data: bytes, /) -> None: ... + + +class ReadIO(Protocol): + def read(self, numbytes: int, /) -> bytes: ... + + +class IO(Protocol): + execmodel: ExecModel + + def read(self, numbytes: int, /) -> bytes: ... + + def write(self, data: bytes, /) -> None: ... + + def close_read(self) -> None: ... + + def close_write(self) -> None: ... + + def wait(self) -> int | None: ... + + def kill(self) -> None: ... + + +class Message: + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 + + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", + } + + def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + self.msgcode = msgcode + self.channelid = channelid + self.data = data + + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + + @staticmethod + def from_io(io: ReadIO) -> Message: + try: + header = io.read(9) # type 1, channel 4, payload 4 + if not header: + raise EOFError("empty read") + except EOFError as e: + raise EOFError("couldn't load message header, " + e.args[0]) from None + msgtype, channel, payload = Message.from_header(header) + return Message(msgtype, channel, io.read(payload)) + + def to_io(self, io: WriteIO) -> None: + io.write(self.pack()) + + def __repr__(self) -> str: + name = self._types[self.msgcode] + return f"" + + +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py new file mode 100644 index 00000000..293be4da --- /dev/null +++ b/src/execnet/_multi.py @@ -0,0 +1,465 @@ +""" +Managing Gateway Groups and interactions with multiple channels. + +(c) 2008-2014, Holger Krekel and others +""" + +from __future__ import annotations + +import atexit +import queue +import threading +import time +import types +import warnings +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import suppress +from threading import Lock +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import TypeAlias +from typing import overload + +from ._boundary import WaitBackend +from ._channel import NO_ENDMARKER_WANTED +from ._channel import Channel +from ._channel import Endmarker +from ._execmodel import ExecModel +from ._execmodel import get_execmodel +from ._execmodel import resolve_profile +from ._host import Host +from ._host import check_not_in_event_loop +from ._host import default_host +from ._trace import trace +from ._xspec import XSpec + +if TYPE_CHECKING: + from ._gateway import Gateway + + +class Group: + """Gateway Group.""" + + defaultspec = "popen" + + #: which primitive this group's blocking waits park on. Set by the + #: facade, not by a spec: it describes the *caller's* concurrency + #: library, which is exactly what picking a namespace already says. + #: ``execnet.gevent.Group`` overrides it. + _wait_backend: WaitBackend = "thread" + + def __init__( + self, + xspecs: Iterable[XSpec | str | None] = (), + profile: str | None = None, + *, + host: Host | None = None, + execmodel: str | None = None, + ) -> None: + """Initialize a group and make gateways as specified. + + ``profile`` is the default worker profile for gateways created + without an explicit ``profile=`` in their spec. ``host`` is the + Trio host thread to serve this group's protocol IO on; it defaults + to the process-wide one. ``execmodel`` is the deprecated spelling + of ``profile`` (pytest-xdist still passes it). + """ + if execmodel is not None: + if profile is not None: + raise TypeError("pass either profile= or execmodel=, not both") + warnings.warn( + "Group(execmodel=...) is deprecated; use Group(profile=...)." + " execnet has no local execution model any more -- the value" + " only selects the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + profile = execmodel + self._gateways: list[Gateway] = [] + self._autoidcounter = 0 + self._autoidlock = Lock() + self._gateways_to_join: list[Gateway] = [] + self._host = default_host() if host is None else host + self._async_group: Any = None + self.set_profile("thread" if profile is None else profile) + for xspec in xspecs: + self.makegateway(xspec) + atexit.register(self._cleanup_atexit) + + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host + + def _ensure_trio_host(self) -> Any: + return self._host._ensure_started() + + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + + @property + def profile(self) -> str: + """Default worker profile for gateways created by this group.""" + return self._profile + + def set_profile(self, profile: str) -> None: + """Set the default worker profile for newly created gateways. + + NOTE: only settable before any gateway is created. + """ + if self._gateways: + raise ValueError( + "can not set the profile if gateways have been created already" + ) + self._profile = resolve_profile(profile) + + @property + def execmodel(self) -> ExecModel: + """Deprecated: there is no local execution model any more.""" + warnings.warn( + "Group.execmodel is deprecated: execnet has no local execution" + " model. Use Group.profile for the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) + + @property + def remote_execmodel(self) -> ExecModel: + """Deprecated alias for :attr:`profile`, as an ExecModel shim.""" + warnings.warn( + "Group.remote_execmodel is deprecated; use Group.profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) + + def set_execmodel( + self, execmodel: str, remote_execmodel: str | None = None + ) -> None: + """Deprecated alias for :meth:`set_profile`. + + The *local* execution model it used to set no longer exists -- all + protocol IO runs on the Trio host -- so only the worker profile is + taken from these arguments (``remote_execmodel`` when given, else + ``execmodel``). + """ + warnings.warn( + "Group.set_execmodel is deprecated; use Group.set_profile(profile)." + " execnet has no local execution model, so only the remote value" + " has an effect.", + DeprecationWarning, + stacklevel=2, + ) + self.set_profile(execmodel if remote_execmodel is None else remote_execmodel) + + def __repr__(self) -> str: + idgateways = [gw.id for gw in self] + return "" % idgateways + + def __getitem__(self, key: int | str | Gateway) -> Gateway: + if isinstance(key, int): + return self._gateways[key] + for gw in self._gateways: + if gw == key or gw.id == key: + return gw + raise KeyError(key) + + def __contains__(self, key: str) -> bool: + try: + self[key] + return True + except KeyError: + return False + + def __len__(self) -> int: + return len(self._gateways) + + def __iter__(self) -> Iterator[Gateway]: + return iter(list(self._gateways)) + + def makegateway(self, spec: XSpec | str | None = None) -> Gateway: + """Create and configure a gateway to a Python interpreter. + + The ``spec`` string encodes the target gateway type + and configuration information. The general format is:: + + key1=value1//key2=value2//... + + If you leave out the ``=value`` part a True value is assumed. + Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. + Valid configuration:: + + id= specifies the gateway id + python= specifies which python interpreter to execute + profile=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' + (default; the first remote_exec claims the + worker main thread, further ones overflow to + pool threads), 'trio' (async sources as tasks, + single-threaded) or 'gevent' (a greenlet per + remote_exec). Spelled 'execmodel=' before + execnet 3.0; that spelling still works. + chdir= specifies to which directory to change + nice= specifies process priority of new process + env:NAME=value specifies a remote environment variable setting. + + If no spec is given, self.defaultspec is used. + """ + check_not_in_event_loop("Group.makegateway()") + if not spec: + spec = self.defaultspec + if not isinstance(spec, XSpec): + spec = XSpec(spec) + self.allocate_id(spec) + if spec.profile is None: + # filling in a missing value is idempotent; rewriting one the + # caller set is not -- see resolve_profile's docstring + spec.profile = self._profile + else: + resolve_profile(spec.profile) + from . import _trio_host + + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): + raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) + gw.spec = spec + self._register(gw) + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. + return gw + + def allocate_id(self, spec: XSpec) -> None: + """(re-entrant) allocate id for the given xspec object.""" + if spec.id is None: + with self._autoidlock: + id = "gw" + str(self._autoidcounter) + self._autoidcounter += 1 + if id in self: + raise ValueError(f"already have gateway with id {id!r}") + spec.id = id + + def _register(self, gateway: Gateway) -> None: + assert not hasattr(gateway, "_group") + assert gateway.id + assert gateway.id not in self + self._gateways.append(gateway) + gateway._group = self + + def _unregister(self, gateway: Gateway) -> None: + self._gateways.remove(gateway) + self._gateways_to_join.append(gateway) + + def _cleanup_atexit(self) -> None: + # The host is shared and stops itself at exit; a group only owns + # its gateways and the async group task running on that host. + trace(f"=== atexit cleanup {self!r} ===") + self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._host._ensure_started().call_sync(self._async_group.shutdown.set) + self._async_group = None + + def terminate(self, timeout: float | None = None) -> None: + """Trigger exit of member gateways and wait for termination + of member gateways and associated subprocesses. + + After waiting timeout seconds try to to kill local sub processes of + popen- and ssh-gateways. + + Timeout defaults to None meaning open-ended waiting and no kill + attempts. + """ + while self or self._gateways_to_join: + vias: set[str] = set() + for gw in self: + if gw.spec.via: + vias.add(gw.spec.via) + for gw in self: + if gw.id not in vias: + gw.exit() + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._host_terminate(timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: + gw.join() + self._gateways_to_join[:] = [] + + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for the wait backend. + + A non-thread backend implies the caller may be a greenlet: wait on + a OneShot instead of blocking the OS thread (which would stall the + hub for the whole grace). + """ + trio_host = self._host._ensure_started() + if self._wait_backend == "thread": + trio_host.call(self._async_group.terminate, timeout) + return + from ._boundary import make_wakener + + trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(self._wait_backend), + ).wait() + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs, + ) -> MultiChannel: + """remote_exec source on all member gateways and return + a MultiChannel connecting to all sub processes.""" + channels = [] + for gw in self: + channels.append(gw.remote_exec(source, **kwargs)) + return MultiChannel(channels) + + +class MultiChannel: + def __init__(self, channels: Sequence[Channel]) -> None: + self._channels = channels + + def __len__(self) -> int: + return len(self._channels) + + def __iter__(self) -> Iterator[Channel]: + return iter(self._channels) + + def __getitem__(self, key: int) -> Channel: + return self._channels[key] + + def __contains__(self, chan: Channel) -> bool: + return chan in self._channels + + def send_each(self, item: object) -> None: + for ch in self._channels: + ch.send(item) + + @overload + def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: + pass + + @overload + def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: + pass + + def receive_each( + self, withchannel: bool = False + ) -> list[tuple[Channel, Any]] | list[Any]: + assert not hasattr(self, "_queue") + l: list[object] = [] + for ch in self._channels: + obj = ch.receive() + if withchannel: + l.append((ch, obj)) + else: + l.append(obj) + return l + + def make_receive_queue(self, endmarker: Endmarker = NO_ENDMARKER_WANTED): + try: + return self._queue # type: ignore[has-type] + except AttributeError: + self._queue: queue.Queue[tuple[Channel, Any]] | None = None + for ch in self._channels: + if self._queue is None: + self._queue = queue.Queue() + + def putreceived(obj, channel: Channel = ch) -> None: + self._queue.put((channel, obj)) # type: ignore[union-attr] + + ch.setcallback(putreceived, endmarker=endmarker) + return self._queue + + def waitclose(self) -> None: + first = None + for ch in self._channels: + try: + ch.waitclose() + except ch.RemoteError as exc: + if first is None: + first = exc + if first: + raise first + + +TermKillFunc: TypeAlias = Callable[[], object] +TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] + + +def safe_terminate( + execmodel: ExecModel, + timeout: float | None, + list_of_paired_functions: Sequence[TermKillPair], +) -> None: + """Run terminate/kill pairs in parallel with a hard wait bound. + + Each termfunc is given ``timeout``. If it does not finish, killfunc runs. + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). + """ + errors: list[BaseException] = [] + + def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): + killfunc() + return + if term_errors: + errors.append(term_errors[0]) + + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions + ] + for thread in threads: + thread.start() + # Allow term timeout plus a kill attempt; never block indefinitely. + wait_timeout = None if timeout is None else timeout * 2 + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] + + +default_group = Group() +makegateway = default_group.makegateway +set_profile = default_group.set_profile +#: deprecated alias, see Group.set_execmodel +set_execmodel = default_group.set_execmodel diff --git a/src/execnet/_portal.py b/src/execnet/_portal.py new file mode 100644 index 00000000..6c1253c4 --- /dev/null +++ b/src/execnet/_portal.py @@ -0,0 +1,72 @@ +"""Cross-thread / cross-loop communication primitives — the boundary kit. + +Internal. A :class:`LoopPortal` is a handle to a running trio loop that +foreign threads use to run functions on the loop or push work into it (the +consumer -> loop direction). Because each direction only needs the +*receiving* loop's token, two trio loops in two threads can communicate by +holding each other's portal. + +The loop -> consumer direction never blocks the loop and never knows who +is listening: the loop fires a :class:`Wakener`, a single thread-safe +``notify()`` supplied by the consumer. There are exactly two wakeners -- +OS threads and gevent greenlets -- because every other concurrency library +gets a facade of its own (:mod:`execnet.trio`, :mod:`execnet.aio`) instead. +On top of the wakener sit the two carriers: + +* :class:`Mailbox` -- an item stream (channel payloads, exec requests), +* :class:`OneShot` -- a single result (write acknowledgements, call + results a consumer wants to await instead of block on). +""" + +from __future__ import annotations + +from collections.abc import Awaitable +from collections.abc import Callable +from typing import Any +from typing import TypeVar + +import trio + +from ._boundary import Mailbox +from ._boundary import OneShot +from ._boundary import ThreadWakener +from ._boundary import Wakener + +__all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] + +T = TypeVar("T") + + +class LoopPortal: + """Handle to a running trio loop, usable from foreign threads. + + Must be constructed on the loop's own thread (it captures the current + trio token). + """ + + def __init__(self) -> None: + self._token = trio.lowlevel.current_trio_token() + + def is_loop_thread(self) -> bool: + """Whether the calling thread is running this portal's loop.""" + try: + return trio.lowlevel.current_trio_token() is self._token + except RuntimeError: + return False + + def run(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Run ``await async_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def run_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + """Run ``sync_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + def post(self, sync_fn: Callable[..., object], *args: Any) -> None: + """Schedule ``sync_fn(*args)`` on the loop without waiting. + + Thread-safe and callable from the loop thread itself; all posts run + in strict FIFO order (``TrioToken.run_sync_soon``). Raises + ``trio.RunFinishedError`` once the loop has shut down. + """ + self._token.run_sync_soon(sync_fn, *args) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py new file mode 100644 index 00000000..22427b1a --- /dev/null +++ b/src/execnet/_provision.py @@ -0,0 +1,559 @@ +"""Coordinator-side worker provisioning via ``uv``. + +Non-same-interpreter Trio workers (foreign-python popen, later ssh/socket) are +launched inside an environment that ``uv`` provisions with a matching execnet + +trio. The worker itself is always ``python -m execnet worker`` and does the +usual ``b"1"`` handshake on whichever protocol transport it was given; only +the launch prefix differs. + +Delivery of execnet into that environment is version-aware: + +* released coordinator (``X.Y.Z``) -> ``uv run --with execnet==X.Y.Z`` +* dev coordinator (``X.Y.Z.devN+g...``) -> build a wheel from the editable + install's source tree, cache it keyed by version, and ``uv run --with `` + +trio is pulled transitively as an execnet dependency. +""" + +from __future__ import annotations + +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from functools import cache +from pathlib import Path +from typing import Any + +_RELEASED_RE = re.compile(r"^\d+\.\d+\.\d+$") + +#: ``transport=`` values: where a worker's protocol stream lives. +#: +#: ``socket`` keeps the protocol off the worker's stdio -- an inherited +#: socketpair for popen, an ``ssh -R``-forwarded unix socket for ssh -- so +#: the worker's stdin/stdout/stderr belong to the code it runs. ``stdio`` +#: is the classic shape, and the only one available where the machinery +#: ``socket`` needs is missing. +TRANSPORTS = ("socket", "stdio") + + +def socket_transport_available() -> bool: + """Whether this coordinator can drive the socket transport. + + ``subprocess`` refuses ``pass_fds`` on Windows and ``ssh -R`` cannot + forward a unix socket there either, so Windows coordinators stay on + stdio unless someone asks for otherwise and accepts the consequences. + """ + return not sys.platform.startswith("win") + + +def resolve_transport(spec: Any) -> str: + """The transport for ``spec``: explicit if given, else the platform default.""" + requested: str | None = getattr(spec, "transport", None) + if requested is None: + return "socket" if socket_transport_available() else "stdio" + if requested not in TRANSPORTS: + raise ValueError(f"unknown transport {requested!r} (known: {list(TRANSPORTS)})") + return requested + + +def uv_available() -> bool: + """Whether the ``uv`` launcher is on PATH.""" + return shutil.which("uv") is not None + + +def shell_split_path(path: str) -> list[str]: + """Split a ``python=`` value into argv tokens with shell lexing. + + Takes care to handle Windows' ``\\`` correctly. + """ + if sys.platform.startswith("win"): + # replace \\ by / otherwise shlex will strip them out + path = path.replace("\\", "/") + return shlex.split(path) + + +@cache +def target_info(python: str) -> dict[str, Any] | None: + """``execnet info`` from interpreter ``python``, or None if it cannot run. + + One probe answers everything provisioning wants to know before it + connects: whether execnet is importable at all, which version it is, + whether trio is there, and which transports it can serve. An execnet + too old to have the CLI fails the probe and gets uv-provisioned, which + is the right outcome. + """ + argv = [*shell_split_path(python), "-m", "execnet", "info"] + try: + completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + try: + info: dict[str, Any] = json.loads(completed.stdout) + except ValueError: + return None + return info + + +def target_has_execnet(python: str) -> bool: + """Whether ``python`` can host a worker directly (execnet + trio present). + + When true the worker is launched on that interpreter as-is (preserving + ``sys.executable``); otherwise it must be uv-provisioned. + """ + info = target_info(python) + return info is not None and info.get("trio") is not None + + +def _version_slug(version: str) -> str: + """Filesystem-safe slug for a version string (may contain ``+``/``.``).""" + return re.sub(r"[^0-9A-Za-z]+", "_", version) + + +def _wheel_cache_dir() -> Path: + d = Path(tempfile.gettempdir()) / "execnet-bootstrap-wheels" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _editable_source_root() -> Path | None: + """Source tree of an editable execnet install, via PEP 610 ``direct_url.json``. + + Returns ``None`` when execnet is not installed editable (nothing to build). + """ + from importlib.metadata import PackageNotFoundError + from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname + + try: + dist = distribution("execnet") + except PackageNotFoundError: + return None + raw = dist.read_text("direct_url.json") + if not raw: + return None + info = json.loads(raw) + if not info.get("dir_info", {}).get("editable"): + return None + url = info.get("url", "") + if not url.startswith("file:"): + return None + return Path(url2pathname(urlparse(url).path)) + + +_BUILT_RE = re.compile(r"^Successfully built (?P.+\.whl)\s*$", re.MULTILINE) + + +def _parse_built_wheel(uv_build_stderr: str) -> Path | None: + """Extract the exact wheel path from ``uv build``'s ``Successfully built`` line. + + Building the filename ourselves is unsafe: the build-time version (dirty tree + -> ``.dYYYYMMDD``/differing dev count) can differ from the import-time + ``__version__``, so we trust the path uv reports. + """ + matches = _BUILT_RE.findall(uv_build_stderr) + if len(matches) != 1: + return None + return Path(matches[0].strip()) + + +def _build_wheel(version: str) -> Path: + """Build (and cache) a wheel of the editable execnet source for ``version``.""" + version_dir = _wheel_cache_dir() / _version_slug(version) + if version_dir.exists(): + cached = sorted(version_dir.glob("*.whl")) + if len(cached) == 1: + return cached[0] + + root = _editable_source_root() + if root is None: + raise RuntimeError( + f"cannot provision dev execnet {version!r}: no editable source tree " + "found (install a released execnet or an editable checkout)" + ) + version_dir.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + ["uv", "build", "--wheel", "-o", str(version_dir), str(root)], + check=True, + capture_output=True, + text=True, + ) + wheel = _parse_built_wheel(proc.stderr) + if wheel is None or not wheel.exists(): + raise RuntimeError( + f"could not determine wheel path from uv build for {version!r}:\n" + f"{proc.stderr}" + ) + return wheel + + +def coordinator_requirement() -> str: + """A ``uv --with`` requirement that installs this coordinator's execnet.""" + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return f"execnet=={version}" + return str(_build_wheel(version)) + + +def worker_cli_arg(spec: Any) -> str: + """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). + + Passed to ``python -m execnet worker`` by every launcher (popen, uv, ssh) + so the worker config lives in one place rather than scattered positional + args. + """ + import execnet + + from ._execmodel import effective_profile + + # the spec keeps whatever the caller spelled; the worker gets what it + # actually has a strategy for. A spec that never went through a Group + # has no profile at all -- the default applies, as it would there. + profile = effective_profile(spec.profile or "thread") + config: dict[str, Any] = { + "id": f"{spec.id}-worker", + "profile": profile, + # pre-3.0 spelling, same value: a worker from an older execnet + # reads this one. Drop with the XSpec alias. + "execmodel": profile, + # derived, not configurable: the gevent profile parks its greenlets + # on gevent wakeners, every other profile is thread-shaped. + "wait": "gevent" if profile == "gevent" else "thread", + "coordinator_version": execnet.__version__, + } + # Startup setup applied by the worker before serving (never through + # remote_exec: an exec slot must not be claimed by bookkeeping). + if spec.chdir: + config["chdir"] = spec.chdir + if spec.nice: + config["nice"] = int(spec.nice) + if spec.env: + config["env"] = spec.env + return json.dumps(config) + + +def stdio_tokens(spec: Any) -> list[str]: + """``--stdin/--stdout/--stderr`` tokens for whatever ``spec`` asked for. + + A worker inherits the coordinator's stdio by default now that the + protocol has a transport of its own, which also means remote code can + *consume* the coordinator's stdin. These keys are how a caller says + otherwise, e.g. ``popen//stdin=devnull``. + """ + tokens = [] + for name in ("stdin", "stdout", "stderr"): + value = getattr(spec, name, None) + if value: + tokens += [f"--{name}", value] + return tokens + + +def _worker_tokens(config: str | None, *protocol: str) -> list[str]: + """``python -u -m execnet worker`` tokens for a launch. + + The literal ``python`` token is resolved by uv (inside the provisioned + environment) or the remote shell. ``config`` of None means the config + arrives on stdin (``--config-fd 0``), which is what remote launches use: + a config in argv is visible in ``ps`` to every user on that host, and it + carries ``env:`` values. + """ + tokens = ["python", "-u", "-m", "execnet", "worker", *protocol] + if config is None: + return [*tokens, "--config-fd", "0"] + return [*tokens, "--config", config] + + +def worker_module_tokens(spec: Any, *protocol: str) -> list[str]: + """``python -u -m execnet worker`` tokens for ``spec``.""" + return _worker_tokens(worker_cli_arg(spec), *protocol) + + +def _uv_tokens(python: str | None) -> list[str]: + # --no-project keeps the surrounding execnet checkout from being synced. + prefix = ["uv", "run", "--no-project"] + if python: + prefix += ["--python", python] + return prefix + + +def _extra_with_tokens(config: str) -> list[str]: + """Additional ``--with`` requirements the worker env needs. + + Derived from the worker config itself so every uv launcher (popen, + ssh, via sub-spawn) provisions the same: the gevent profile / wait + backend needs gevent importable in the worker. + """ + parsed = json.loads(config) + if parsed.get("profile") == "gevent" or parsed.get("wait") == "gevent": + return ["--with", "gevent"] + return [] + + +def uv_worker_argv(spec: Any, *protocol: str) -> list[str]: + """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + config = worker_cli_arg(spec) + return [ + *_uv_tokens(spec.python), + "--with", + coordinator_requirement(), + *_extra_with_tokens(config), + *_worker_tokens(config, *protocol), + ] + + +#: where a shipped wheel lands on the remote, keyed by name (which carries +#: the version) so repeated gateways to one host reuse it. A *shell +#: fragment*, not a path: ``$HOME`` is expanded remotely, and quoting it as +#: a literal would create a directory actually called ``~``. +REMOTE_WHEEL_DIR = '"$HOME"/.cache/execnet/wheels' + + +def remote_wheel_path(wheel: Path) -> str: + """Shell fragment for the remote path a shipped wheel is delivered to.""" + return f"{REMOTE_WHEEL_DIR}/{shlex.quote(wheel.name)}" + + +def wheel_delivery_command(wheel: Path) -> str: + """Remote sh command that receives ``wheel`` on stdin, unless already there. + + Out of band: run over its *own* ssh connection before the worker + launch, so the protocol stream never has to carry a payload and the + launch command needs neither ``head -c `` byte accounting nor + ``exec`` to keep an fd alive. + + Both branches consume stdin -- the coordinator streams the wheel + unconditionally, and a remote that skipped the write without draining + would hand it an EPIPE. + """ + path = remote_wheel_path(wheel) + return ( + f"mkdir -p {REMOTE_WHEEL_DIR} || exit 1; " + f"if [ -s {path} ]; then cat > /dev/null; else cat > {path}.tmp" + f" && mv {path}.tmp {path}; fi" + ) + + +def _remote_shell_command( + python: str | None, + config: str, + *protocol: str, + requirement: str | None = None, + wheel: Path | None = None, + config_on_stdin: bool = False, +) -> str: + """Remote sh command launching the worker via uv. + + ``requirement`` installs from an index; ``wheel`` uses one already + delivered by :func:`wheel_delivery_command`. + """ + worker = _worker_tokens(None if config_on_stdin else config, *protocol) + uv = [*_uv_tokens(python), *_extra_with_tokens(config)] + if wheel is None: + assert requirement is not None + return shlex.join([*uv, "--with", requirement, *worker]) + # the wheel path is remote-side and may contain ~, so it is not quoted + # by shlex.join -- splice it in after quoting the rest + return " ".join( + [shlex.join([*uv, "--with"]), remote_wheel_path(wheel), shlex.join(worker)] + ) + + +def ssh_remote_command(spec: Any, *protocol: str, config_on_stdin: bool = False) -> str: + """Remote shell command launching the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …``. Dev + coordinator -> ``uv run --with …``; delivering the + wheel is a separate step (:func:`wheel_delivery_command`). + """ + import execnet + + version = execnet.__version__ + config = worker_cli_arg(spec) + kwargs: dict[str, Any] = {"config_on_stdin": config_on_stdin} + if _RELEASED_RE.match(version): + kwargs["requirement"] = f"execnet=={version}" + else: + kwargs["wheel"] = _build_wheel(version) + return _remote_shell_command(spec.python, config, *protocol, **kwargs) + + +def ssh_wheel(spec: Any) -> Path | None: + """The wheel this coordinator must deliver before launching, if any.""" + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return None + return _build_wheel(version) + + +def ssh_argv( + ssh: str, + ssh_config: str | None, + remote_command: str, + options: list[str] | None = None, +) -> list[str]: + """``ssh`` client argv running ``remote_command`` on host ``ssh``.""" + args = ["ssh", "-C"] + if ssh_config: + args += ["-F", ssh_config] + if options: + args += options + args += ssh.split() + args.append(remote_command) + return args + + +def vagrant_ssh_argv( + machine: str, + ssh_config: str | None, + remote_command: str, + options: list[str] | None = None, +) -> list[str]: + """``vagrant ssh`` argv running ``remote_command`` on the named VM. + + Everything after ``--`` is passed through to the underlying ssh client, + mirroring ``ssh_argv``. + """ + args = ["vagrant", "ssh", machine, "--", "-C"] + if ssh_config: + args += ["-F", ssh_config] + if options: + args += options + args.append(remote_command) + return args + + +def spawn_request(spec: Any) -> dict[str, Any]: + """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. + + Carries the sub-spec essentials plus provisioning material when the sub + may need it (ssh or foreign python): a released coordinator sends a pip + requirement; a dev coordinator ships its wheel bytes for the master to + materialize into its local wheel cache. + + TODO: the wheel is shipped eagerly because only the master can tell + whether the target interpreter already has execnet; a wheel-on-demand + round-trip would avoid the transfer in the common provisioned case. + """ + import execnet + + request: dict[str, Any] = { + "config": worker_cli_arg(spec), + "python": spec.python or None, + "ssh": spec.ssh or None, + "vagrant_ssh": spec.vagrant_ssh or None, + "ssh_config": spec.ssh_config or None, + } + if spec.ssh or spec.vagrant_ssh or spec.python: + version = execnet.__version__ + if _RELEASED_RE.match(version): + request["requirement"] = f"execnet=={version}" + else: + wheel = _build_wheel(version) + request["wheel"] = (wheel.name, wheel.read_bytes()) + return request + + +def materialize_wheel(name: str, data: bytes) -> Path: + """Write shipped wheel bytes into the local wheel cache (idempotent).""" + target = _wheel_cache_dir() / name + if not target.exists(): + tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") + tmp.write_bytes(data) + tmp.replace(target) + return target + + +def _requested_requirement(request: dict[str, Any]) -> tuple[str | None, Path | None]: + """(uv requirement, local wheel path) from a spawn request's material.""" + requirement = request.get("requirement") + if isinstance(requirement, str): + return requirement, None + shipped = request.get("wheel") + if shipped is not None: + name, data = shipped + path = materialize_wheel(name, data) + return str(path), path + return None, None + + +#: an out-of-band step to run before a sub-worker launch: ``(argv, stdin)`` +DeliveryStep = tuple[list[str], bytes] + + +def sub_spawn_argv( + request: dict[str, Any], +) -> tuple[list[str], DeliveryStep | None]: + """(argv, wheel delivery) spawning a requested sub-worker on this host. + + Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs + this interpreter's worker module, a foreign ``python`` runs directly when + it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps + the remote uv command. A shipped wheel is delivered by the returned + step -- its own ssh connection, run before the launch -- rather than + framed into the launch command's stdin. + + The sub's protocol is relayed over its stdio by the master, so it always + gets the stdio transport. + """ + config = request["config"] + assert isinstance(config, str) + python = request.get("python") + ssh = request.get("ssh") + vagrant = request.get("vagrant_ssh") + if ssh or vagrant: + requirement, wheel = _requested_requirement(request) + if requirement is None: + raise RuntimeError("remote spawn request without provisioning material") + ssh_config = request.get("ssh_config") + + def wrap(command: str, options: list[str] | None = None) -> list[str]: + if ssh: + assert isinstance(ssh, str) + return ssh_argv(ssh, ssh_config, command, options) + assert isinstance(vagrant, str) + return vagrant_ssh_argv(vagrant, ssh_config, command, options) + + delivery: DeliveryStep | None = None + if wheel is not None: + delivery = (wrap(wheel_delivery_command(wheel)), wheel.read_bytes()) + command = _remote_shell_command(python, config, wheel=wheel) + else: + command = _remote_shell_command(python, config, requirement=requirement) + return wrap(command), delivery + if python: + assert isinstance(python, str) + if target_has_execnet(python): + argv = [*shell_split_path(python), "-u", "-m", "execnet", "worker"] + return [*argv, "--config", config], None + requirement, _ = _requested_requirement(request) + if requirement is None or not uv_available(): + raise RuntimeError( + f"cannot provision sub-worker for python={python!r}: " + "uv and provisioning material required" + ) + return [ + *_uv_tokens(python), + "--with", + requirement, + *_worker_tokens(config), + ], None + return [ + sys.executable, + "-u", + "-m", + "execnet", + "worker", + "--config", + config, + ], None diff --git a/src/execnet/_rsync.py b/src/execnet/_rsync.py new file mode 100644 index 00000000..d14825f6 --- /dev/null +++ b/src/execnet/_rsync.py @@ -0,0 +1,248 @@ +""" +1:N rsync implementation on top of execnet. + +(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Callable +from hashlib import md5 +from queue import Queue +from typing import Literal + +from execnet import _rsync_remote +from execnet._channel import Channel +from execnet._gateway import Gateway +from execnet._gateway_base import BaseGateway + + +class RSync: + """This class allows to send a directory structure (recursively) + to one or multiple remote filesystems. + + There is limited support for symlinks, which means that symlinks + pointing to the sourcetree will be send "as is" while external + symlinks will be just copied (regardless of existence of such + a path on remote side). + """ + + def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: + self._sourcedir = str(sourcedir) + self._verbose = verbose + assert callback is None or callable(callback) + self._callback = callback + self._channels: dict[Channel, Callable[[], None] | None] = {} + self._receivequeue: Queue[ + tuple[ + Channel, + ( + None + | tuple[Literal["send"], tuple[list[str], bytes]] + | tuple[Literal["list_done"], None] + | tuple[Literal["ack"], str] + | tuple[Literal["links"], None] + | tuple[Literal["done"], None] + ), + ] + ] = Queue() + self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] + + def filter(self, path: str) -> bool: + return True + + def _end_of_channel(self, channel: Channel) -> None: + if channel in self._channels: + # too early! we must have got an error + channel.waitclose() + # or else we raise one + raise OSError(f"connection unexpectedly closed: {channel.gateway} ") + + def _process_link(self, channel: Channel) -> None: + for link in self._links: + channel.send(link) + # completion marker, this host is done + channel.send(42) + + def _done(self, channel: Channel) -> None: + """Call all callbacks.""" + finishedcallback = self._channels.pop(channel) + if finishedcallback: + finishedcallback() + channel.waitclose() + + def _list_done(self, channel: Channel) -> None: + # sum up all to send + if self._callback: + s = sum([self._paths[i] for i in self._to_send[channel]]) + self._callback("list", s, channel) + + def _send_item( + self, + channel: Channel, + modified_rel_path_components: list[str], + checksum: bytes, + ) -> None: + """Send one item.""" + modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) + try: + with open(modified_path, "rb") as fp: + data = fp.read() + except OSError: + data = None + + # provide info to progress callback function + modified_rel_path = "/".join(modified_rel_path_components) + if data is not None: + self._paths[modified_rel_path] = len(data) + else: + self._paths[modified_rel_path] = 0 + if channel not in self._to_send: + self._to_send[channel] = [] + self._to_send[channel].append(modified_rel_path) + # print "sending", modified_rel_path, data and len(data) or 0, checksum + + if data is not None: + if checksum is not None and checksum == md5(data).digest(): + data = None # not really modified + else: + self._report_send_file(channel.gateway, modified_rel_path) + channel.send(data) + + def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: + if self._verbose: + print(f"{gateway} <= {modified_rel_path}") + + def send(self, raises: bool = True) -> None: + """Sends a sourcedir to all added targets. + + raises indicates whether to raise an error or return in case of lack of + targets. + """ + if not self._channels: + if raises: + raise OSError( + "no targets available, maybe you are trying call send() twice?" + ) + return + # normalize a trailing '/' away + self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) + # send directory structure and file timestamps/sizes + self._send_directory_structure(self._sourcedir) + + # paths and to_send are only used for doing + # progress-related callbacks + self._paths: dict[str, int] = {} + self._to_send: dict[Channel, list[str]] = {} + + # send modified file to clients + while self._channels: + channel, req = self._receivequeue.get() + if req is None: + self._end_of_channel(channel) + else: + if req[0] == "links": + self._process_link(channel) + elif req[0] == "done": + self._done(channel) + elif req[0] == "ack": + if self._callback: + self._callback("ack", self._paths[req[1]], channel) + elif req[0] == "list_done": + self._list_done(channel) + elif req[0] == "send": + self._send_item(channel, req[1][0], req[1][1]) + else: + assert "Unknown command %s" % req[0] # type: ignore[unreachable] + + def add_target( + self, + gateway: Gateway, + destdir: str | os.PathLike[str], + finishedcallback: Callable[[], None] | None = None, + **options, + ) -> None: + """Add a remote target specified via a gateway and a remote destination + directory.""" + for name in options: + assert name in ("delete",) + + def itemcallback(req) -> None: + self._receivequeue.put((channel, req)) + + channel = gateway.remote_exec(_rsync_remote) + channel.setcallback(itemcallback, endmarker=None) + channel.send((str(destdir), options)) + self._channels[channel] = finishedcallback + + def _broadcast(self, msg: object) -> None: + for channel in self._channels: + channel.send(msg) + + def _send_link( + self, + linktype: Literal["linkbase", "link"], + basename: str, + linkpoint: str, + ) -> None: + self._links.append((linktype, basename, linkpoint)) + + def _send_directory(self, path: str) -> None: + # dir: send a list of entries + names = [] + subpaths = [] + for name in os.listdir(path): + p = os.path.join(path, name) + if self.filter(p): + names.append(name) + subpaths.append(p) + mode = os.lstat(path).st_mode + self._broadcast([mode, *names]) + for p in subpaths: + self._send_directory_structure(p) + + def _send_link_structure(self, path: str) -> None: + sourcedir = self._sourcedir + basename = path[len(self._sourcedir) + 1 :] + linkpoint = os.readlink(path) + # On Windows, readlink returns an extended path (//?/) for + # absolute links, but relpath doesn't like mixing extended + # and non-extended paths. So fix it up ourselves. + if ( + os.path.__name__ == "ntpath" + and linkpoint.startswith("\\\\?\\") + and not self._sourcedir.startswith("\\\\?\\") + ): + sourcedir = "\\\\?\\" + self._sourcedir + try: + relpath = os.path.relpath(linkpoint, sourcedir) + except ValueError: + relpath = None + if ( + relpath is not None + and relpath not in (os.curdir, os.pardir) + and not relpath.startswith(os.pardir + os.sep) + ): + self._send_link("linkbase", basename, relpath) + else: + # relative or absolute link, just send it + self._send_link("link", basename, linkpoint) + self._broadcast(None) + + def _send_directory_structure(self, path: str) -> None: + try: + st = os.lstat(path) + except OSError: + self._broadcast((None, 0, 0)) + return + if stat.S_ISREG(st.st_mode): + # regular file: send a mode/timestamp/size pair + self._broadcast((st.st_mode, st.st_mtime, st.st_size)) + elif stat.S_ISDIR(st.st_mode): + self._send_directory(path) + elif stat.S_ISLNK(st.st_mode): + self._send_link_structure(path) + else: + raise ValueError(f"cannot sync {path!r}") diff --git a/src/execnet/_rsync_remote.py b/src/execnet/_rsync_remote.py new file mode 100644 index 00000000..bc2e4f41 --- /dev/null +++ b/src/execnet/_rsync_remote.py @@ -0,0 +1,126 @@ +""" +(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Literal +from typing import cast + +if TYPE_CHECKING: + from execnet._channel import Channel + + +def serve_rsync(channel: Channel) -> None: + import os + import shutil + import stat + from hashlib import md5 + + destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) + modifiedfiles = [] + + def remove(path: str) -> None: + assert path.startswith(destdir) + try: + os.unlink(path) + except OSError: + # assume it's a dir + shutil.rmtree(path, True) + + def receive_directory_structure(path: str, relcomponents: list[str]) -> None: + try: + st = os.lstat(path) + except OSError: + st = None + msg = channel.receive() + if isinstance(msg, list): + if st and not stat.S_ISDIR(st.st_mode): + os.unlink(path) + st = None + if not st: + os.makedirs(path) + mode = msg.pop(0) + if mode: + # Ensure directories are writable, otherwise a + # permission denied error (EACCES) would be raised + # when attempting to receive read-only directory + # structures. + os.chmod(path, mode | 0o700) + entrynames = {} + for entryname in msg: + destpath = os.path.join(path, entryname) + receive_directory_structure(destpath, [*relcomponents, entryname]) + entrynames[entryname] = True + if options.get("delete"): + for othername in os.listdir(path): + if othername not in entrynames: + otherpath = os.path.join(path, othername) + remove(otherpath) + elif msg is not None: + assert isinstance(msg, tuple) + checksum = None + if st: + if stat.S_ISREG(st.st_mode): + msg_mode, msg_mtime, msg_size = msg + if msg_size != st.st_size: + pass + elif msg_mtime != st.st_mtime: + with open(path, "rb") as fp: + checksum = md5(fp.read()).digest() + elif msg_mode and msg_mode != st.st_mode: + os.chmod(path, msg_mode | 0o700) + return + else: + return # already fine + else: + remove(path) + channel.send(("send", (relcomponents, checksum))) + modifiedfiles.append((path, msg)) + + receive_directory_structure(destdir, []) + + STRICT_CHECK = False # seems most useful this way for py.test + channel.send(("list_done", None)) + + for path, (mode, time, size) in modifiedfiles: + data = cast(bytes, channel.receive()) + channel.send(("ack", path[len(destdir) + 1 :])) + if data is not None: + if STRICT_CHECK and len(data) != size: + raise OSError(f"file modified during rsync: {path!r}") + with open(path, "wb") as fp: + fp.write(data) + try: + if mode: + os.chmod(path, mode) + os.utime(path, (time, time)) + except OSError: + pass + del data + channel.send(("links", None)) + + msg = channel.receive() + while msg != 42: + # we get symlink + _type, relpath, linkpoint = cast( + "tuple[Literal['linkbase', 'link'], str, str]", msg + ) + path = os.path.join(destdir, relpath) + with suppress(OSError): + remove(path) + + if _type == "linkbase": + src = os.path.join(destdir, linkpoint) + else: + assert _type == "link", _type + src = linkpoint + os.symlink(src, path) + msg = channel.receive() + channel.send(("done", None)) + + +if __name__ == "__channelexec__": + serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/_serialize.py b/src/execnet/_serialize.py new file mode 100644 index 00000000..3151efe3 --- /dev/null +++ b/src/execnet/_serialize.py @@ -0,0 +1,439 @@ +"""The wire serializer for execnet's simple builtin data format. + +Internal: no public namespace re-exports ``dumps``/``loads``. Callers that +need to know whether a value can cross a channel use +:func:`execnet.can_send`, which is exported from every public namespace. + +The channel layer is *not* imported here -- ``Unserializer`` resolves a +channel or gateway argument by duck-typing -- so ``_channel`` may depend on +this module and not the other way round. +""" + +from __future__ import annotations + +import struct +from collections.abc import Callable +from io import BytesIO +from typing import TYPE_CHECKING +from typing import Any + +from ._errors import DumpError +from ._errors import LoadError + +if TYPE_CHECKING: + from ._channel import Channel + from ._gateway_base import BaseGateway + from ._message import ReadIO + + +def bchr(n: int) -> bytes: + return bytes([n]) + + +DUMPFORMAT_VERSION = bchr(2) + +FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 + +FLOAT_FORMAT = "!d" +FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) +COMPLEX_FORMAT = "!dd" +COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) + + +class _Stop(Exception): + pass + + +class opcode: + """Container for name -> num mappings.""" + + BUILDTUPLE = b"@" + BYTES = b"A" + CHANNEL = b"B" + FALSE = b"C" + FLOAT = b"D" + FROZENSET = b"E" + INT = b"F" + LONG = b"G" + LONGINT = b"H" + LONGLONG = b"I" + NEWDICT = b"J" + NEWLIST = b"K" + NONE = b"L" + STRING = b"N" + SET = b"O" + SETITEM = b"P" + STOP = b"Q" + TRUE = b"R" + COMPLEX = b"T" + + +class Unserializer: + num2func: dict[bytes, Callable[[Unserializer], None]] = {} + + def __init__( + self, + stream: ReadIO, + channel_or_gateway: Channel | BaseGateway | None = None, + ) -> None: + # A channel -- sync or trio-native -- resolves through its gateway; a + # gateway is already the right object. Duck-typed so the serializer + # stays independent of the channel layer. + gw: Any = getattr(channel_or_gateway, "gateway", channel_or_gateway) + self.stream = stream + if gw is None: + self.channelfactory = None + else: + self.channelfactory = gw._channelfactory + + def load(self, versioned: bool = False) -> Any: + if versioned: + ver = self.stream.read(1) + if ver != DUMPFORMAT_VERSION: + raise LoadError("wrong dumpformat version %r" % ver) + self.stack: list[object] = [] + try: + while True: + opcode = self.stream.read(1) + if not opcode: + raise EOFError + try: + loader = self.num2func[opcode] + except KeyError: + raise LoadError( + f"unknown opcode {opcode!r} - wire protocol corruption?" + ) from None + loader(self) + except _Stop: + if len(self.stack) != 1: + raise LoadError("internal unserialization error") from None + return self.stack.pop(0) + else: + raise LoadError("didn't get STOP") + + def load_none(self) -> None: + self.stack.append(None) + + num2func[opcode.NONE] = load_none + + def load_true(self) -> None: + self.stack.append(True) + + num2func[opcode.TRUE] = load_true + + def load_false(self) -> None: + self.stack.append(False) + + num2func[opcode.FALSE] = load_false + + def load_int(self) -> None: + i = self._read_int4() + self.stack.append(i) + + num2func[opcode.INT] = load_int + + def load_longint(self) -> None: + s = self._read_byte_string() + self.stack.append(int(s)) + + num2func[opcode.LONGINT] = load_longint + + load_long = load_int + num2func[opcode.LONG] = load_long + load_longlong = load_longint + num2func[opcode.LONGLONG] = load_longlong + + def load_float(self) -> None: + binary = self.stream.read(FLOAT_FORMAT_SIZE) + self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) + + num2func[opcode.FLOAT] = load_float + + def load_complex(self) -> None: + binary = self.stream.read(COMPLEX_FORMAT_SIZE) + self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) + + num2func[opcode.COMPLEX] = load_complex + + def _read_int4(self) -> int: + value: int = struct.unpack("!i", self.stream.read(4))[0] + return value + + def _read_byte_string(self) -> bytes: + length = self._read_int4() + as_bytes = self.stream.read(length) + return as_bytes + + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) + + num2func[opcode.STRING] = load_string + + def load_bytes(self) -> None: + s = self._read_byte_string() + self.stack.append(s) + + num2func[opcode.BYTES] = load_bytes + + def load_newlist(self) -> None: + length = self._read_int4() + self.stack.append([None] * length) + + num2func[opcode.NEWLIST] = load_newlist + + def load_setitem(self) -> None: + if len(self.stack) < 3: + raise LoadError("not enough items for setitem") + value = self.stack.pop() + key = self.stack.pop() + self.stack[-1][key] = value # type: ignore[index] + + num2func[opcode.SETITEM] = load_setitem + + def load_newdict(self) -> None: + self.stack.append({}) + + num2func[opcode.NEWDICT] = load_newdict + + def _load_collection(self, type_: type) -> None: + length = self._read_int4() + if length: + res = type_(self.stack[-length:]) + del self.stack[-length:] + self.stack.append(res) + else: + self.stack.append(type_()) + + def load_buildtuple(self) -> None: + self._load_collection(tuple) + + num2func[opcode.BUILDTUPLE] = load_buildtuple + + def load_set(self) -> None: + self._load_collection(set) + + num2func[opcode.SET] = load_set + + def load_frozenset(self) -> None: + self._load_collection(frozenset) + + num2func[opcode.FROZENSET] = load_frozenset + + def load_stop(self) -> None: + raise _Stop + + num2func[opcode.STOP] = load_stop + + def load_channel(self) -> None: + id = self._read_int4() + assert self.channelfactory is not None + newchannel = self.channelfactory.new(id) + self.stack.append(newchannel) + + num2func[opcode.CHANNEL] = load_channel + + +def dumps(obj: object) -> bytes: + """Serialize the given obj to a bytestring. + + The obj and all contained objects must be of a builtin + Python type (so nested dicts, sets, etc. are all OK but + not user-level instances). + """ + return _Serializer().save(obj, versioned=True) # type: ignore[return-value] + + +def dump(byteio, obj: object) -> None: + """write a serialized bytestring of the given obj to the given stream.""" + _Serializer(write=byteio.write).save(obj, versioned=True) + + +def loads(bytestring: bytes) -> Any: + """Deserialize the given bytestring to an object. + + If the bytestring was dumped with an incompatible protocol + version or if the bytestring is corrupted, the + ``execnet.DataFormatError`` will be raised. + """ + return load(BytesIO(bytestring)) + + +def load(io: ReadIO) -> Any: + """Derserialize an object form the specified stream. + + Behaviour is otherwise the same as with ``loads`` + """ + return Unserializer(io).load(versioned=True) + + +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: + io = BytesIO(bytestring) + return Unserializer(io, channelfactory).load() + + +def dumps_internal(obj: object) -> bytes: + return _Serializer().save(obj) # type: ignore[return-value] + + +def can_send(obj: object) -> bool: + """Whether ``obj`` can cross a channel as-is. + + True for execnet's simple builtin wire data -- ``None``, ``bool``, + ``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily + nested ``list``/``tuple``/``set``/``frozenset``/``dict`` of those -- + and for channel references. False for anything execnet has no wire + representation for, which ``channel.send`` would reject with + :class:`~execnet.DumpError`. + + Use it to branch *before* sending, instead of sending and handling the + error:: + + channel.send(value if execnet.can_send(value) else repr(value)) + """ + try: + _Serializer().save(obj) + except DumpError: + return False + return True + + +class _Serializer: + _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} + + def __init__(self, write: Callable[[bytes], None] | None = None) -> None: + if write is None: + self._streamlist: list[bytes] = [] + write = self._streamlist.append + self._write = write + + def save(self, obj: object, versioned: bool = False) -> bytes | None: + # calling here is not re-entrant but multiple instances + # may write to the same stream because of the common platform + # atomic-write guarantee (concurrent writes each happen atomically) + if versioned: + self._write(DUMPFORMAT_VERSION) + self._save(obj) + self._write(opcode.STOP) + try: + streamlist = self._streamlist + except AttributeError: + return None + return b"".join(streamlist) + + def _save(self, obj: object) -> None: + tp = type(obj) + try: + dispatch = self._dispatch[tp] + except KeyError: + methodname = "save_" + tp.__name__ + meth: Callable[[_Serializer, object], None] | None = getattr( + self.__class__, methodname, None + ) + if meth is None: + raise DumpError(f"can't serialize {tp}") from None + dispatch = self._dispatch[tp] = meth + dispatch(self, obj) + + def save_NoneType(self, non: None) -> None: + self._write(opcode.NONE) + + def save_bool(self, boolean: bool) -> None: + if boolean: + self._write(opcode.TRUE) + else: + self._write(opcode.FALSE) + + def save_bytes(self, bytes_: bytes) -> None: + self._write(opcode.BYTES) + self._write_byte_sequence(bytes_) + + def save_str(self, s: str) -> None: + self._write(opcode.STRING) + self._write_unicode_string(s) + + def _write_unicode_string(self, s: str) -> None: + try: + as_bytes = s.encode("utf-8") + except UnicodeEncodeError as e: + raise DumpError("strings must be utf-8 encodable") from e + self._write_byte_sequence(as_bytes) + + def _write_byte_sequence(self, bytes_: bytes) -> None: + self._write_int4(len(bytes_), "string is too long") + self._write(bytes_) + + def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: + self._write(short_op) + self._write_int4(i) + else: + self._write(long_op) + self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) + + def save_int(self, i: int) -> None: + self._save_integral(i, opcode.INT, opcode.LONGINT) + + def save_long(self, l: int) -> None: + self._save_integral(l, opcode.LONG, opcode.LONGLONG) + + def save_float(self, flt: float) -> None: + self._write(opcode.FLOAT) + self._write(struct.pack(FLOAT_FORMAT, flt)) + + def save_complex(self, cpx: complex) -> None: + self._write(opcode.COMPLEX) + self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) + + def _write_int4( + self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) + ) -> None: + if i > FOUR_BYTE_INT_MAX: + raise DumpError(error) + self._write(struct.pack("!i", i)) + + def save_list(self, L: list[object]) -> None: + self._write(opcode.NEWLIST) + self._write_int4(len(L), "list is too long") + for i, item in enumerate(L): + self._write_setitem(i, item) + + def _write_setitem(self, key: object, value: object) -> None: + self._save(key) + self._save(value) + self._write(opcode.SETITEM) + + def save_dict(self, d: dict[object, object]) -> None: + self._write(opcode.NEWDICT) + for key, value in d.items(): + self._write_setitem(key, value) + + def save_tuple(self, tup: tuple[object, ...]) -> None: + for item in tup: + self._save(item) + self._write(opcode.BUILDTUPLE) + self._write_int4(len(tup), "tuple is too long") + + def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: + for item in s: + self._save(item) + self._write(op) + self._write_int4(len(s), "set is too long") + + def save_set(self, s: set[object]) -> None: + self._write_set(s, opcode.SET) + + def save_frozenset(self, s: frozenset[object]) -> None: + self._write_set(s, opcode.FROZENSET) + + def save_Channel(self, channel: Channel) -> None: + self._write(opcode.CHANNEL) + self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py new file mode 100644 index 00000000..0839c2e3 --- /dev/null +++ b/src/execnet/_shim.py @@ -0,0 +1,47 @@ +"""Machinery for the deprecated pre-Trio module names. + +``execnet.gateway_base``, ``execnet.gateway``, ``execnet.multi``, +``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` were never +part of a documented public API -- they were importable only because +``import execnet`` pulled them in transitively. Their contents now live in +private modules grouped by concern; each old name survives as a thin module +whose ``__getattr__`` warns and forwards. + +The supported surfaces are :mod:`execnet` / :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.gevent`. +""" + +from __future__ import annotations + +import importlib +import warnings +from typing import Any + +#: shims are scheduled for removal in this release +REMOVED_IN = "execnet 3.0" + + +def forwarder(shim: str, moved: dict[str, str]) -> Any: + """Build the ``__getattr__`` for a deprecated module. + + ``moved`` maps each previously-reachable name to the private module that + now defines it (a relative name such as ``"._channel"``). + """ + + def __getattr__(name: str) -> Any: + try: + module = moved[name] + except KeyError: + raise AttributeError( + f"module 'execnet.{shim}' has no attribute {name!r}" + ) from None + warnings.warn( + f"execnet.{shim} is private and will be removed in {REMOVED_IN}; " + f"{name} now lives in execnet{module}. The supported surfaces are " + f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.gevent.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module(module, __package__), name) + + return __getattr__ diff --git a/src/execnet/_socketserver.py b/src/execnet/_socketserver.py new file mode 100644 index 00000000..ee6c4191 --- /dev/null +++ b/src/execnet/_socketserver.py @@ -0,0 +1,50 @@ +"""Trio socket server for execnet gateways. + +Listens on a TCP port and hands each accepted connection (by fd) to a fresh +worker subprocess that serves the gateway over it. No code is executed +inline. + +The supported entry point is ``execnet server`` (see :mod:`execnet._cli`) -- +run it on the target host, or install-free with +``uvx --from execnet execnet server``. The old ``execnet-socketserver`` +console command still works and forwards here with a DeprecationWarning. +""" + +from __future__ import annotations + + +async def serve(hostport: str, once: bool) -> None: + """Bind ``hostport`` and hand accepted connections to worker processes.""" + import trio + + from execnet import _trio_host + + host, _, port_str = hostport.rpartition(":") + listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) + addr = listeners[0].socket.getsockname() + # Report the bound address (port may be ephemeral) for callers to read. + print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) + + if once: + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + # The worker outlives this one-shot server. + await _trio_host.serve_socket_connection(stream, reap=False) + return + + async def handler(stream: trio.SocketStream) -> None: + await _trio_host.serve_socket_connection(stream, reap=True) + + await trio.serve_listeners(handler, listeners) + + +def main(argv: list[str] | None = None) -> None: + """Deprecated ``execnet-socketserver`` console entry point.""" + from ._cli import socketserver_main + + socketserver_main(argv) + + +if __name__ == "__main__": + main() diff --git a/src/execnet/_trace.py b/src/execnet/_trace.py new file mode 100644 index 00000000..edb650d8 --- /dev/null +++ b/src/execnet/_trace.py @@ -0,0 +1,47 @@ +"""Debug tracing, configured once from ``EXECNET_DEBUG``. + +:EXECNET_DEBUG=1: write per-process trace files to ``execnet-debug-PID`` +:EXECNET_DEBUG=2: trace to stderr (popen workers forward this to their + instantiator) + +Unset, ``trace`` is a no-op lambda so tracing costs a call and nothing else. +""" + +from __future__ import annotations + +import os +import sys + +DEBUG = os.environ.get("EXECNET_DEBUG") +pid = os.getpid() + +if DEBUG == "2": + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + sys.stderr.write(f"[{pid}] {line}\n") + sys.stderr.flush() + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +elif DEBUG: + import tempfile + + fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) + # sys.stderr.write("execnet-debug at %r" % (fn,)) + debugfile = open(fn, "w") + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + debugfile.write(line + "\n") + debugfile.flush() + except Exception as exc: + try: + sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +else: + notrace = trace = lambda *msg: None diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py new file mode 100644 index 00000000..44b7dbd3 --- /dev/null +++ b/src/execnet/_trio_gateway.py @@ -0,0 +1,1244 @@ +"""Trio-native gateway core: async dispatch loop and low-level raw channels. + +Async-first counterpart of the sync machinery in ``_channel`` / +``_gateway_base``: an +:class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch +task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline +on that task, so there is no receiver thread and no receive lock. + +Two-level channel model: + +* :class:`RawChannel` (this module) -- id-routed raw byte payload streams + over the gateway: no serialization and no callbacks. + ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top + decides what the bytes mean. +* ``AsyncChannel`` -- the serialized object API layered on a RawChannel. + +The code deliberately sticks to idioms an anyio backend can mirror later: +a neutral ``ByteStream`` protocol, the sans-IO ``FrameDecoder``, and +unbounded memory channels. +""" + +from __future__ import annotations + +import math +import os +import subprocess +import sys +import types +import uuid +from collections.abc import AsyncIterator +from collections.abc import Callable +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol + +import trio + +if TYPE_CHECKING: + from typing_extensions import Self + +from ._errors import GatewayReceivedTerminate +from ._errors import HostNotFound +from ._errors import RemoteError +from ._errors import TimeoutError +from ._exec_source import normalize_exec_source +from ._execmodel import resolve_profile +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace + +RECEIVE_CHUNK = 65536 + + +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. + + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ + + async def send_all(self, data: bytes) -> None: ... + + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + + async def send_eof(self) -> None: ... + + async def aclose(self) -> None: ... + + +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) + + +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) + + +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def popen_module_args( + spec: Any, *protocol: str, config_on_stdin: bool = False +) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). + """ + from . import _provision + + if getattr(spec, "python", None): + interpreter = _provision.shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += ["-m", "execnet", "worker", *protocol] + args += _provision.stdio_tokens(spec) + if config_on_stdin: + args += ["--config-fd", "0"] + else: + args += ["--config", _provision.worker_cli_arg(spec)] + return args + + +def popen_worker_argv(spec: Any, *protocol: str) -> list[str]: + """Argv for a popen worker: direct module launch, or uv-provisioned. + + A bare ``python=`` interpreter without execnet gets execnet + trio + provisioned via ``uv``; otherwise the worker module is launched directly. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + return _provision.uv_worker_argv(spec, *protocol) + return popen_module_args(spec, *protocol) + + +class RawChannel: + """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. + + Payload boundaries are preserved: every :meth:`send_bytes` arrives as + one :meth:`receive_bytes` result on the peer. No serialization and no + flow control beyond the gateway's outbound queue. + + Close semantics mirror the sync ``Channel`` state machine: + + * :meth:`aclose` closes both directions (``CHANNEL_CLOSE`` / + ``CHANNEL_CLOSE_ERROR`` to the peer). + * :meth:`send_eof` only ends our payload stream (``CHANNEL_LAST_MESSAGE``); + the peer drains, hits EOF, and may keep sending to us. + """ + + def __init__(self, gateway: AsyncGateway, id: int) -> None: + self.gateway = gateway + self.id = id + self._closed = False # no more sends (local aclose or remote close) + self._sent_eof = False + self._remote_closed = False + self._receive_closed = trio.Event() # no more payloads will arrive + self._remote_error: RemoteError | None = None + self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + # Diversion hooks for a bound facade (sync channel): when set, + # inbound payloads/closes route out of the loop instead of + # buffering for receive_bytes. + self._consumer_payload: Callable[[bytes], None] | None = None + self._consumer_close: Callable[[RemoteError | None, bool], None] | None = None + self._pending_close: tuple[RemoteError | None, bool] | None = None + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + async def send_bytes(self, data: bytes) -> None: + """Send one payload; the peer receives it as a single item. + + OSError is raised when the channel or gateway is closed, matching + the sync ``Channel.send`` contract. + """ + if self._closed or self._sent_eof: + raise OSError(f"cannot send to {self!r}") + await self.gateway._send(Message.CHANNEL_DATA, self.id, data) + + async def receive_bytes(self) -> bytes: + """Receive the next payload. + + Raises EOFError once the peer closed or sent EOF and all payloads + are drained; a peer close-with-error raises that ``RemoteError``. + """ + try: + return await self._payloads.receive() + except (trio.EndOfChannel, trio.ClosedResourceError): + raise self._pending_error() from None + + async def send_eof(self) -> None: + """Signal that no more payloads follow (peer keeps its send side).""" + if self._closed or self._sent_eof: + raise OSError(f"cannot send EOF to {self!r}") + self._sent_eof = True + await self.gateway._send(Message.CHANNEL_LAST_MESSAGE, self.id) + + async def aclose(self, error: str | None = None) -> None: + """Close both directions; ``error`` reaches the peer as a RemoteError.""" + if self._closed: + await trio.lowlevel.checkpoint() + return + self._closed = True + self._payload_send.close() + self._receive_closed.set() + self.gateway._forget_channel(self.id) + if not self._remote_closed: + # A peer-initiated close needs no reply; a dead gateway is + # already as closed as it gets. + with suppress(OSError): + if error is not None: + await self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error) + ) + else: + await self.gateway._send(Message.CHANNEL_CLOSE, self.id) + + def __aiter__(self) -> RawChannel: + return self + + async def __anext__(self) -> bytes: + try: + return await self.receive_bytes() + except EOFError: + raise StopAsyncIteration from None + + def _pending_error(self) -> BaseException: + return ( + self._remote_error + or self.gateway._error + or EOFError(f"raw channel {self.id} closed") + ) + + def set_consumer( + self, + on_payload: Callable[[bytes], None], + on_close: Callable[[RemoteError | None, bool], None], + ) -> None: + """Divert inbound payloads and the close to callbacks (loop thread). + + Already-buffered payloads flush to ``on_payload`` first, and a close + that arrived before binding is replayed to ``on_close`` -- so a + consumer bound late (facade channels bind via a portal post) sees + the exact inbound order. + """ + self._consumer_payload = on_payload + self._consumer_close = on_close + while True: + try: + data = self._payloads.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + break + on_payload(data) + if self._pending_close is not None: + error, sendonly = self._pending_close + self._pending_close = None + on_close(error, sendonly) + if not sendonly: + # the late binder has now claimed the buffered close + self.gateway._forget_channel(self.id) + + # dispatch-loop internals (inline on the gateway's serve task) + + def _feed(self, data: bytes) -> None: + if self._consumer_payload is not None: + self._consumer_payload(data) + return + try: + self._payload_send.send_nowait(data) + except (trio.BrokenResourceError, trio.ClosedResourceError): + pass # locally closed: drop, like the sync channel + + def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> None: + if error is not None: + self._remote_error = error + self._remote_closed = True + self._receive_closed.set() + if not sendonly: + self._closed = True + if self._consumer_close is not None: + # Without a consumer the closed channel stays registered: + # a passed-channel reference may still bind late and must + # find the buffered payloads and this close, not a fresh + # empty channel under the same id. + self.gateway._forget_channel(self.id) + self._payload_send.close() + if self._consumer_close is not None: + self._consumer_close(error, sendonly) + else: + self._pending_close = (error, sendonly) + + +class RawChannelStream: + """``ByteStream`` over a :class:`RawChannel` -- the frame-native via tunnel. + + The gateway writer performs one ``send_all`` per frame, so every raw + payload carries exactly one whole sub-protocol frame (the master relay + keeps that invariant in the other direction). ``receive_some`` buffers + payloads and honours ``max_bytes`` for the handshake read. + """ + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self._buf = bytearray() + self._eof = False + + async def send_all(self, data: bytes) -> None: + await self._raw.send_bytes(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: + try: + self._buf += await self._raw.receive_bytes() + except EOFError: + self._eof = True + except RemoteError as exc: + self._eof = True + raise EOFError(f"via tunnel closed: {exc}") from None + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] + return out + + async def send_eof(self) -> None: + with suppress(OSError): + await self._raw.send_eof() + + async def aclose(self) -> None: + await self._raw.aclose() + + +class AsyncChannel: + """Serialized object API over a :class:`RawChannel`. + + Every payload is one dumps/loads-serialized item; close/EOF semantics + and error propagation come from the raw layer. Channels are + async-iterable, and :meth:`receive` supports the familiar execnet + timeout (raising ``TimeoutError``). + + Channel objects themselves serialize: sending an AsyncChannel inside an + item transfers a reference the peer receives as its own AsyncChannel + for the same id (the wire CHANNEL opcode, as with sync channels). + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self.gateway = raw.gateway + self.id = raw.id + + def __repr__(self) -> str: + flag = "closed" if self.isclosed() else "open" + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._raw._closed + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side. + + The item must be a simple Python type; OSError is raised when the + channel or gateway is closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + await self._raw.send_bytes(dumps_internal(item)) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + Raises EOFError once the peer closed or sent EOF, a RemoteError for + a peer close-with-error, and TimeoutError if no item arrived within + ``timeout`` seconds. + """ + if timeout is None: + data = await self._raw.receive_bytes() + else: + try: + with trio.fail_after(timeout): + data = await self._raw.receive_bytes() + except trio.TooSlowError: + raise TimeoutError("no item after %r seconds" % timeout) from None + return loads_internal(data, self) + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._raw.send_eof() + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._raw.aclose(error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._raw._receive_closed.wait() + error = self._raw._remote_error or self.gateway._error + if error is not None: + raise error + + def __aiter__(self) -> AsyncChannel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + # Unserializer duck-type: loads_internal(data, self) reads + # _channelfactory off the object to resolve CHANNEL opcodes. + + @property + def _channelfactory(self) -> _AsyncChannelFactory: + return self.gateway._channelfactory + + +class _AsyncChannelFactory: + """Duck-typed factory for the Unserializer CHANNEL opcode (``.new(id)``).""" + + def __init__(self, gateway: AsyncGateway) -> None: + self.gateway = gateway + + def new(self, id: int) -> AsyncChannel: + return self.gateway.open_channel(id) + + +class AsyncGateway: + """Async-native gateway: the framed Message protocol over a ByteStream. + + Serving (``serve_gateway`` or ``nursery.start(gateway._serve)``) runs a + reader task that dispatches messages inline and a writer task draining + an unbounded outbound queue -- sends never block on the peer. + + ``_startcount`` follows the sync convention: locally allocated channel + ids step by two, coordinators from 1 (odd) and workers from 2 (even), + so the two peers never collide. + """ + + _error: BaseException | None = None + #: where the peer lives, when the transport knows (ssh host, socket addr) + remoteaddress: str | None = None + #: CHANNEL_EXEC handler ``(gateway, channelid, data) -> None`` -- set by + #: an exec strategy (the pure-async worker's TaskExec); without one, + #: exec requests are rejected. + _exec_handler: Callable[[AsyncGateway, int, bytes], None] | None = None + #: the exec strategy (for STATUS numexecuting), when serving as a worker + _task_exec: Any = None + + def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: + self._stream = stream + self.id = id + self._channels: dict[int, RawChannel] = {} + self._async_channels: dict[int, AsyncChannel] = {} + self._channelfactory = _AsyncChannelFactory(self) + self._count = _startcount + self._outbound_send, self._outbound = trio.open_memory_channel[ + tuple[bytes, Callable[[BaseException | None], None] | None] + ](math.inf) + self._closed = False + self._serve_started = False + self._writer_done = trio.Event() + self._done = trio.Event() + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + @property + def closed(self) -> bool: + return self._closed + + async def wait_closed(self) -> None: + """Wait until serving has fully shut down.""" + await self._done.wait() + + def _trace(self, *msg: object) -> None: + trace(self.id, *msg) + + def open_raw_channel(self, id: int | None = None) -> RawChannel: + """Return the raw channel for ``id``, allocating a fresh id if None. + + An explicit id attaches to a channel the peer references (e.g. an id + received in a request payload); the same object is returned if the + dispatch loop already routed data to it. + """ + if self._closed: + raise OSError(f"connection already closed: {self!r}") + if id is None: + id = self._count + self._count += 2 + return self._channel_for(id) + + def open_channel(self, id: int | None = None) -> AsyncChannel: + """Return the serialized channel for ``id``, allocating one if None.""" + raw = self.open_raw_channel(id) + try: + return self._async_channels[raw.id] + except KeyError: + channel = self._async_channels[raw.id] = AsyncChannel(raw) + return channel + + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> AsyncChannel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as the sync ``Gateway.remote_exec``: + a source string, a pure function called with ``channel`` and + ``**kwargs``, or a module. The remote end closes the channel when + execution finishes. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.open_channel() + await self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + if not self._closed: + with suppress(OSError): + await self._send(Message.GATEWAY_TERMINATE) + await self.aclose() + + async def aclose(self) -> None: + """Flush queued frames, close the stream, and wait for shutdown.""" + if not self._closed: + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() + if self._serve_started: + await self._done.wait() + else: + self._finish_channels() + + async def _serve( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: + """Run the reader and writer until EOF, termination, or ``aclose``.""" + self._serve_started = True + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + task_status.started() + await self._reader() + # Reader is done: let the writer flush queued frames + # (close replies), then stop it. + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + nursery.cancel_scope.cancel() + finally: + self._closed = True + self._outbound_send.close() + try: + await self._finalize() + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _finalize(self) -> None: + """Serve-shutdown hook: release the channel layer. + + The sync facade overrides this to close its sync channels and shut + down execution instead. + """ + self._finish_channels() + + async def _reader(self) -> None: + decoder = FrameDecoder() + try: + while True: + try: + data = await self._stream.receive_some(RECEIVE_CHUNK) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + if not self._closed: + self._error = exc + return + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("connection closed (no gateway termination)") + for message in decoder.feed(data): + self._trace("received", message) + self._dispatch(message) + except GatewayReceivedTerminate: + self._trace("received GATEWAY_TERMINATE") + except EOFError as exc: + self._trace("EOF without prior gateway termination message") + self._error = exc + + async def _writer(self) -> None: + error: BaseException | None = None + try: + async for frame, on_written in self._outbound: + try: + await self._stream.send_all(frame) + except BaseException as exc: + error = exc + if on_written is not None: + on_written(exc) + raise + if on_written is not None: + on_written(None) + except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: + self._trace("writer failed", exc) + if self._error is None: + self._error = exc + else: + # Queue closed and drained: signal write-EOF to the peer. + with suppress(Exception): + await self._stream.send_eof() + finally: + # No more writes will happen: fail queued frames instead of + # leaving their senders waiting on acknowledgements. + self._outbound_send.close() + self._fail_pending_writes(error) + self._writer_done.set() + + def _fail_pending_writes(self, error: BaseException | None) -> None: + exc = error if error is not None else OSError("cannot send (already closed?)") + while True: + try: + _frame, on_written = self._outbound.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + return + if on_written is not None: + on_written(exc) + + def _dispatch(self, message: Message) -> None: + """Route one message; runs inline on the serve task.""" + code = message.msgcode + channelid = message.channelid + if code == Message.CHANNEL_DATA: + self._channel_for(channelid)._feed(message.data) + elif code == Message.CHANNEL_CLOSE: + self._channel_for(channelid)._close_from_remote(None, sendonly=False) + elif code == Message.CHANNEL_CLOSE_ERROR: + error_message = loads_internal(message.data) + assert isinstance(error_message, str) + self._channel_for(channelid)._close_from_remote( + RemoteError(error_message), sendonly=False + ) + elif code == Message.CHANNEL_LAST_MESSAGE: + self._channel_for(channelid)._close_from_remote(None, sendonly=True) + elif code == Message.GATEWAY_TERMINATE: + raise GatewayReceivedTerminate(self) + elif code == Message.CHANNEL_EXEC and self._exec_handler is not None: + self._exec_handler(self, channelid, message.data) + elif code == Message.STATUS: + task_exec = self._task_exec + status = { + "numchannels": len(self._channels), + "numexecuting": task_exec.active_count() if task_exec else 0, + "profile": "trio", + # legacy key, same value -- pytest-xdist reads it + "execmodel": "trio", + } + self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.GATEWAY_INFO: + self._send_nowait( + Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) + ) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) + else: + # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core + self._trace("rejecting unsupported message", message) + self._send_nowait( + Message.CHANNEL_CLOSE_ERROR, + channelid, + dumps_internal(f"unsupported message on async gateway: {message!r}"), + ) + + def _channel_for(self, id: int) -> RawChannel: + try: + return self._channels[id] + except KeyError: + channel = self._channels[id] = RawChannel(self, id) + return channel + + def _forget_channel(self, id: int) -> None: + self._channels.pop(id, None) + self._async_channels.pop(id, None) + + async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + # The queue is unbounded, so this never waits on the peer -- but it + # is a real checkpoint and raises once the gateway is closed. + try: + await self._outbound_send.send( + (Message(msgcode, channelid, data).pack(), None) + ) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def enqueue_frame( + self, + frame: bytes, + on_written: Callable[[BaseException | None], None] | None = None, + ) -> None: + """Queue one wire frame (sync, loop thread only). + + ``on_written`` fires exactly once: after the frame reached the OS + write, or with the failure when it never will. Raises OSError when + the outbound side is already closed. + """ + try: + self._outbound_send.send_nowait((frame, on_written)) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" + with suppress(OSError): + self.enqueue_frame(Message(msgcode, channelid, data).pack()) + + def _finish_channels(self) -> None: + for channel in list(self._channels.values()): + channel._close_from_remote(None, sendonly=True) + self._channels.clear() + self._async_channels.clear() + + +@asynccontextmanager +async def serve_gateway( + stream: ByteStream, *, id: str, _startcount: int = 1 +) -> AsyncIterator[AsyncGateway]: + """Serve an :class:`AsyncGateway` over ``stream`` for the ``with`` body.""" + gateway = AsyncGateway(stream, id=id, _startcount=_startcount) + async with trio.open_nursery() as nursery: + await nursery.start(gateway._serve) + try: + yield gateway + finally: + await gateway.aclose() + + +#: how long to wait for an ssh worker to dial back before giving up +SSH_CONNECT_TIMEOUT = 60.0 + + +def _ssh_argv( + spec: Any, command: str, forward: tuple[str, str] | None = None +) -> list[str]: + """ssh/vagrant argv running ``command``, optionally with a ``-R`` forward. + + ``forward`` is ``(remote_socket, local_socket)``. ``StreamLocalBindUnlink`` + makes sshd replace a stale socket file rather than refuse to bind. + """ + from . import _provision + + options: list[str] = [] + if forward is not None: + remote_sock, local_sock = forward + options += ["-R", f"{remote_sock}:{local_sock}"] + options += ["-o", "StreamLocalBindUnlink=yes"] + if spec.ssh is not None: + return _provision.ssh_argv(spec.ssh, spec.ssh_config, command, options) + assert spec.vagrant_ssh is not None + return _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, command, options + ) + + +def ssh_transport_args(spec: Any) -> list[str]: + """ssh argv for a stdio-transport ssh worker.""" + from . import _provision + + assert spec.ssh is not None + return _ssh_argv(spec, _provision.ssh_remote_command(spec)) + + +def vagrant_transport_args(spec: Any) -> list[str]: + """vagrant-ssh argv for a stdio-transport vagrant_ssh worker.""" + from . import _provision + + assert spec.vagrant_ssh is not None + return _ssh_argv(spec, _provision.ssh_remote_command(spec)) + + +@asynccontextmanager +async def _dialback_listener() -> AsyncIterator[tuple[str, trio.SocketListener]]: + """A private unix socket the worker will be forwarded back to.""" + import shutil + import tempfile + + directory = tempfile.mkdtemp(prefix="execnet-dialback-") + os.chmod(directory, 0o700) + path = os.path.join(directory, "gw.sock") + sock = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM) + await sock.bind(path) + sock.listen(1) + listener = trio.SocketListener(sock) + try: + yield path, listener + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await listener.aclose() + shutil.rmtree(directory, ignore_errors=True) + + +async def _accept_or_diagnose( + listener: trio.SocketListener, + process: trio.Process, + remoteaddress: str, +) -> trio.SocketStream: + """Await the worker's dial-back, or explain why it never came. + + With the stdio transport a dead ssh shows up as EOF on the protocol + pipe. Here there is no such pipe, so the accept is raced against the + process exiting -- ssh's 255 still means "could not reach or + authenticate the host". + """ + accepted: list[trio.SocketStream] = [] + exited: list[int] = [] + + async def accept(nursery: trio.Nursery) -> None: + accepted.append(await listener.accept()) + nursery.cancel_scope.cancel() + + async def watch(nursery: trio.Nursery) -> None: + exited.append(await process.wait()) + nursery.cancel_scope.cancel() + + with trio.move_on_after(SSH_CONNECT_TIMEOUT): + async with trio.open_nursery() as nursery: + nursery.start_soon(accept, nursery) + nursery.start_soon(watch, nursery) + if accepted: + return accepted[0] + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + if exited and exited[0] == 255: + raise HostNotFound(remoteaddress) + if exited: + raise EOFError( + f"ssh worker exited with {exited[0]} before connecting back" + f" to {remoteaddress}" + ) + raise EOFError( + f"ssh worker did not connect back within {SSH_CONNECT_TIMEOUT}s" + f" ({remoteaddress})" + ) + + +async def connect_ssh_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn an ssh/vagrant worker on the transport ``spec`` resolves to. + + ``transport=socket`` forwards a private unix socket to the remote with + ``ssh -R`` and lets the worker dial back on it, which leaves ssh's + stdin free for the config (so it is not in the remote argv, where every + user on that host can read the ``env:`` values) and leaves the worker's + stdout and stderr free for the code it runs. + """ + from . import _provision + + remoteaddress = spec.ssh or spec.vagrant_ssh + assert remoteaddress is not None + wheel = _provision.ssh_wheel(spec) + if wheel is not None: + await deliver_remote_wheel(spec, wheel) + + if _provision.resolve_transport(spec) == "stdio": + args = ( + ssh_transport_args(spec) + if spec.ssh is not None + else vagrant_transport_args(spec) + ) + return await connect_command_worker(args, remoteaddress=remoteaddress) + + async with _dialback_listener() as (local_sock, listener): + remote_sock = f"/tmp/execnet-{uuid.uuid4().hex}.sock" + command = _provision.ssh_remote_command( + spec, + "--protocol-connect", + f"unix:{remote_sock}", + config_on_stdin=True, + ) + argv = _ssh_argv(spec, command, forward=(remote_sock, local_sock)) + # stdout/stderr stay inherited: remote output is the user's now. + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(_provision.worker_cli_arg(spec).encode()) + await process.stdin.aclose() + stream = await _accept_or_diagnose(listener, process, remoteaddress) + await read_handshake_ack(stream, "ssh") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + return stream, process + + +async def deliver_remote_wheel(spec: Any, wheel: Any) -> None: + """Ship ``wheel`` to the remote over its own connection, before launching. + + Out of band on purpose: the protocol stream never carries a payload, so + the launch command needs no ``head -c `` byte accounting and no + ``exec`` to keep an fd alive. Skipped remote-side when the file is + already cached there. + """ + from . import _provision + + argv = _ssh_argv(spec, _provision.wheel_delivery_command(wheel)) + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(wheel.read_bytes()) + await process.stdin.aclose() + code = await process.wait() + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + if code != 0: + raise HostNotFound( + f"could not deliver the execnet wheel to {spec.ssh or spec.vagrant_ssh}" + f" (exit {code})" + ) + + +async def connect_command_worker( + args: list[str], + *, + preamble: bytes = b"", + remoteaddress: str | None = None, +) -> tuple[ByteStream, trio.Process]: + """Spawn ``args`` and complete the worker ready handshake. + + ``preamble`` is streamed to the worker's stdin before the handshake + (a shipped wheel that the remote receives with ``head -c``). With a + ``remoteaddress``, a handshake EOF plus exit code 255 (ssh could not + reach or authenticate the host) becomes :class:`HostNotFound`. + """ + process = await open_popen_process(args) + try: + stream = staple_process_stream(process) + if preamble: + await stream.send_all(preamble) + await read_handshake_ack(stream, "bootstrap") + except BaseException as exc: + host_not_found = False + with trio.CancelScope(shield=True): + if isinstance(exc, EOFError) and remoteaddress is not None: + with trio.move_on_after(5): + host_not_found = await process.wait() == 255 + with trio.move_on_after(5): + process.kill() + await process.wait() + if host_not_found: + assert remoteaddress is not None + raise HostNotFound(remoteaddress) from None + raise + return stream, process + + +async def connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn a local worker for ``spec`` on its resolved transport. + + With ``transport=socket`` the protocol runs over an inherited + socketpair, so the child's stdin/stdout/stderr stay the user's: remote + ``print()`` reaches the terminal instead of being swallowed to keep the + wire clean. ``transport=stdio`` is the classic pipe pair. + """ + from . import _provision + + if _provision.resolve_transport(spec) == "stdio": + return await connect_command_worker(popen_worker_argv(spec)) + + import socket as _socket + + ours, theirs = _socket.socketpair() + try: + args = popen_worker_argv(spec, "--protocol-fd", str(theirs.fileno())) + process = await trio.lowlevel.open_process(args, pass_fds=(theirs.fileno(),)) + except BaseException: + ours.close() + theirs.close() + raise + # The child holds its own copy now; ours would keep the pair open. + theirs.close() + stream = trio.SocketStream(trio.socket.from_stdlib_socket(ours)) + try: + await read_handshake_ack(stream, "bootstrap") + except BaseException: + with trio.CancelScope(shield=True): + with trio.move_on_after(5): + process.kill() + await process.wait() + await stream.aclose() + raise + return stream, process + + +async def connect_socket_worker( + address: tuple[str, int], remoteaddress: str +) -> ByteStream: + """Connect to a running socketserver and complete the ready handshake.""" + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(remoteaddress) from exc + try: + await read_handshake_ack(stream, "socket") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + await stream.aclose() + raise + return stream + + +async def start_socketserver_via( + gateway: AsyncGateway, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = gateway.open_channel() + await gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = await channel.receive() + await channel.wait_closed() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) + + +class AsyncGroup: + """Trio-native group: an async context manager owning the gateway nursery. + + Gateways created with :meth:`makegateway` are served as child tasks of + the group's nursery. Leaving the ``async with`` block terminates every + gateway with the safe_terminate contract: GATEWAY_TERMINATE plus a + ``timeout`` grace, then kill -- bounded at roughly twice the timeout + even when a kill gets stuck (see issues #43 / #221). + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._nursery: trio.Nursery | None = None + self._gateways: list[AsyncGateway] = [] + self._processes: dict[AsyncGateway, trio.Process] = {} + + def __repr__(self) -> str: + ids = [gateway.id for gateway in self._gateways] + return f"" + + async def __aenter__(self) -> Self: + self._nursery_manager = trio.open_nursery() + self._nursery = await self._nursery_manager.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> bool | None: + # The serve tasks only end once their gateways shut down, so + # terminate before letting the nursery join its children. + # Shielded: cleanup stays bounded even under cancellation. + terminate_error: BaseException | None = None + try: + with trio.CancelScope(shield=True): + await self.terminate(self._termination_timeout) + except BaseException as error: + terminate_error = error + self._nursery = None + suppress_body_exc = await self._nursery_manager.__aexit__( + exc_type, exc_value, traceback + ) + if terminate_error is not None: + raise terminate_error + return suppress_body_exc + + async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: + """Create a gateway for ``spec`` served on the group's nursery. + + All transport types are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways relayed through a group + member. + """ + from ._xspec import XSpec + + if self._nursery is None: + raise RuntimeError(f"{self!r} is not entered") + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if spec.profile is None: + # An async coordinator does not imply an async worker: the + # worker's shape is its own choice, and the default stays the + # thread profile. Pass ``profile=trio`` for a worker that runs + # exec'd async sources as tasks. + spec.profile = "thread" + else: + resolve_profile(spec.profile) + if spec.id is None: + spec.id = "gw%d" % len(self._gateways) + process: trio.Process | None = None + remoteaddress: str | None = None + if spec.via: + stream: ByteStream = await self._open_via_stream(spec) + remote = spec.ssh or spec.vagrant_ssh + if remote: + remoteaddress = f"{remote}[via {spec.via}]" + elif spec.socket: + address, remoteaddress = await self._resolve_socket_address(spec) + stream = await connect_socket_worker(address, remoteaddress) + elif spec.ssh or spec.vagrant_ssh: + remoteaddress = spec.ssh or spec.vagrant_ssh + stream, process = await connect_ssh_worker(spec) + elif spec.popen or spec.python: + stream, process = await connect_popen_worker(spec) + else: + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + gateway = self._make_gateway(stream, spec) + gateway.remoteaddress = remoteaddress + await self._nursery.start(gateway._serve) + self._gateways.append(gateway) + if process is not None: + self._processes[gateway] = process + self._nursery.start_soon(self._reap_process, process) + return gateway + + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + """Construct the gateway object for a freshly connected stream. + + Overridden by the sync facade to build bridge gateways instead. + """ + return AsyncGateway(stream, id=spec.id, _startcount=1) + + async def _reap_process(self, process: trio.Process) -> None: + # Prompt reaping for workers that exit on their own (no zombies). + with suppress(Exception): + await process.wait() + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + """``((host, port), remoteaddress)`` for a ``socket=`` spec. + + ``installvia=`` asks that group member to start a one-shot + socketserver first. The sync facade overrides this to talk to its + sync master gateway. + """ + if getattr(spec, "installvia", None): + master = self._gateway_by_id(spec.installvia) + realhost, realport = await start_socketserver_via(master) + return (realhost, realport), "%s:%d" % (realhost, realport) + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + return (host_str, int(port_str)), spec.socket + + async def _open_via_stream(self, spec: Any) -> ByteStream: + """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a + raw channel (each payload one whole sub-protocol frame).""" + from . import _provision + + master = self._gateway_by_id(spec.via) + raw = master.open_raw_channel() + request = _provision.spawn_request(spec) + await master._send(Message.GATEWAY_START_SUB, raw.id, dumps_internal(request)) + stream = RawChannelStream(raw) + await read_handshake_ack(stream, "via") + return stream + + def _gateway_by_id(self, id: str) -> AsyncGateway: + for gateway in self._gateways: + if gateway.id == id: + return gateway + raise KeyError(f"no gateway {id!r} in {self!r}") + + async def terminate(self, timeout: float | None = None) -> None: + """Terminate all gateways; never hangs (kill after ``timeout``). + + Tunneled (``via``) gateways go first so their termination frames + still travel through a live master. + """ + gateways = list(self._gateways) + self._gateways.clear() + tunneled = [gw for gw in gateways if gw not in self._processes] + spawned = [gw for gw in gateways if gw in self._processes] + for batch in (tunneled, spawned): + if not batch: + continue + async with trio.open_nursery() as nursery: + for gateway in batch: + nursery.start_soon(self._terminate_one, gateway, timeout) + + async def _terminate_one( + self, gateway: AsyncGateway, timeout: float | None + ) -> None: + grace = math.inf if timeout is None else timeout + await gateway.terminate() + process = self._processes.pop(gateway, None) + if process is None: + return + with trio.move_on_after(grace): + await process.wait() + if process.returncode is None: + process.kill() + with trio.move_on_after(grace): + await process.wait() + + +@asynccontextmanager +async def open_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. + Convenience for a single-gateway :class:`AsyncGroup`. + """ + async with AsyncGroup() as group: + yield await group.makegateway(spec) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py new file mode 100644 index 00000000..dc5214b6 --- /dev/null +++ b/src/execnet/_trio_host.py @@ -0,0 +1,976 @@ +"""Trio host thread for execnet Message-protocol IO. + +Coordinator and worker both run framed read/write loops here. +Sync Channel/Gateway APIs talk to this host via thread-safe queues and +``trio.from_thread``. +""" + +from __future__ import annotations + +import functools +import itertools +import json +import math +import queue as _queue +import subprocess +import sys +import threading +import weakref +from collections.abc import Awaitable +from collections.abc import Callable +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +import trio + +from ._boundary import Flag +from ._channel import ENDMARKER +from ._channel import NO_ENDMARKER_WANTED +from ._channel import Endmarker +from ._errors import GatewayReceivedTerminate +from ._errors import RemoteError +from ._execmodel import ExecModel +from ._execmodel import get_execmodel +from ._host import DEFAULT_CALLBACK_THREADS +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._portal import LoopPortal +from ._portal import OneShot +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace +from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannelStream +from ._trio_gateway import open_popen_process +from ._trio_gateway import read_handshake_ack +from ._trio_gateway import ssh_transport_args + +#: bound on how long the endmarker callback may run during host shutdown +CONSUMER_ENDMARKER_GRACE = 10.0 + + +def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> None: + """Deserialize one payload and invoke the receiver callback (in a thread).""" + callback(loads_internal(data, channel)) + + +if TYPE_CHECKING: + from ._gateway import Gateway + from ._gateway_base import BaseGateway + from ._multi import Group + +T = TypeVar("T") + + +# Kept name: the ssh argv/preamble builder moved to the async core. +ssh_trio_args = ssh_transport_args + + +async def adopt_socket(socket_fd: int) -> trio.SocketStream: + """Worker side: wrap an inherited socket fd and send the handshake. + + Runs on the Trio host loop. The coordinator waits for ``b"1"`` before + starting the Message protocol; the worker config comes from the CLI. + """ + import socket as _socket + + sock = _socket.socket(fileno=socket_fd) + stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) + await stream.send_all(b"1") + return stream + + +class SyncIOHandle: + """Sync IO facade for Gateway.exit close_write and terminate wait/kill.""" + + remoteaddress: str + + def __init__( + self, + execmodel: ExecModel, + session: SyncBridgeGateway, + *, + process: trio.Process | None = None, + remoteaddress: str | None = None, + ) -> None: + self.execmodel = execmodel + self._session = session + self._process = process + if remoteaddress is not None: + self.remoteaddress = remoteaddress + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio IO handle") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio IO handle") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + self._session.request_close_write() + + def wait(self) -> int | None: + process = self._process + if process is None: + return None + + async def _wait() -> int | None: + # Always await wait() so the child is reaped (no zombies). + code: int | None = await process.wait() + return code + + try: + return self._session.host_call(_wait) + except Exception: + return process.returncode + + def kill(self) -> None: + process = self._process + if process is None: + return + + async def _kill() -> None: + with trio.move_on_after(5): + process.kill() + await process.wait() + + try: + self._session.host_call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + +class SyncBridgeGateway(AsyncGateway): + """Async engine serving a sync ``BaseGateway``. + + The reader/writer/framing machinery is inherited from + :class:`AsyncGateway`; dispatch is overridden to run the classic sync + ``Message`` handlers (channel queues, callbacks, exec scheduling) under + the gateway's receive lock instead of routing to raw channels. + + Foreign threads send through :meth:`enqueue_message`: every enqueue goes + through the portal so loop callbacks and threads land in one global FIFO, + and non-loop threads wait until the frame hit the OS write (120s -> + OSError) so an abrupt ``os._exit`` cannot drop already-"sent" data. + """ + + def __init__( + self, + stream: ByteStream, + *, + id: str, + sync_gateway: BaseGateway, + host: TrioHost, + ) -> None: + super().__init__(stream, id=id) + self.sync_gateway = sync_gateway + self.host = host + self._done_sync: OneShot[None] = OneShot(sync_gateway._new_wakener()) + self._send_closed = False + self._send_lock = threading.Lock() + # Attach before any serving can happen: the first inbound message + # may need to reply through gateway._send, which must already + # route to this session (not the sync IO stub). + sync_gateway._attach_trio_session(self) + + # -- engine hooks (run on the host loop) -- + + def _dispatch(self, message: Message) -> None: + """Route one message: sync-facade concerns here, the rest to the core. + + CHANNEL_DATA/CLOSE/CLOSE_ERROR/LAST_MESSAGE fall through to the + async core, which routes them to the RawChannel whose consumer is + the bound sync ``Channel``. + """ + gateway = self.sync_gateway + code = message.msgcode + try: + if code == Message.STATUS: + self._answer_status(message) + elif code == Message.GATEWAY_INFO: + gateway._send( + Message.CHANNEL_DATA, + message.channelid, + dumps_internal(gateway_info()), + ) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) + elif code == Message.CHANNEL_EXEC: + channel = gateway._channelfactory.new(message.channelid) + gateway._local_schedulexec(channel=channel, sourcetask=message.data) + elif code == Message.GATEWAY_START_SOCKET: + handle_start_socket(gateway, message.channelid, message.data) + elif code == Message.GATEWAY_START_SUB: + handle_start_sub(gateway, message.channelid, message.data) + else: + super()._dispatch(message) + except (GatewayReceivedTerminate, EOFError): + raise + except Exception as exc: + gateway._trace("dispatch failed:", gateway._geterrortext(exc)) + raise EOFError("error dispatching message") from exc + + def _answer_status(self, message: Message) -> None: + # we use the channelid to send back information + # but don't instantiate a channel object + gateway = self.sync_gateway + execpool = getattr(gateway, "_execpool", None) + d = { + "numchannels": len(gateway._channelfactory._channels), + "numexecuting": execpool.active_count() if execpool is not None else 0, + "profile": gateway.execmodel.backend, + # legacy key, same value -- pytest-xdist reads it + "execmodel": gateway.execmodel.backend, + } + gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) + + async def _finalize(self) -> None: + gateway = self.sync_gateway + with self._send_lock: + self._send_closed = True + if gateway._error is None: + gateway._error = self._error + gateway._trace("[trio-bridge] finishing channels") + gateway._channelfactory._finished_receiving() + # EOF the loop-side raw channels that have no sync consumer + # (via-tunnel relays and readers blocked in receive_bytes). + self._finish_channels() + # Unblock the worker's join() before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done_sync.set(None) + gateway._trace("[trio-bridge] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + + # -- sync session interface (any thread) -- + + def bind_sync_channel(self, channel: Any) -> None: + """Route inbound data for ``channel.id`` to the sync channel. + + Callable from any thread: binding happens on the loop via the + portal so it cannot interleave with dispatch, and the raw channel + replays anything (payloads, a close) that arrived first. The loop + side only holds a weakref, preserving the factory's weak-registry + semantics (GC of the last user reference sends the close message); + channels with callbacks are kept alive by the factory instead. + """ + ref = weakref.ref(channel) + channelid = channel.id + with suppress(trio.RunFinishedError): + self.host.portal.post(self._install_sync_consumer, ref, channelid) + + def _install_sync_consumer(self, ref: weakref.ref[Any], channelid: int) -> None: + """Route ``channelid``'s raw payloads/close to the sync channel (loop). + + Idempotent: re-binding to the same hooks just re-flushes the (empty) + raw buffer, so ``attach_consumer`` can call this itself instead of + depending on the separately-posted bind having run first. + """ + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) + + def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: + channel = ref() + if channel is not None: + channel._deliver_payload(data) + # dead ref: data for a deleted channel is dropped, like before + + def _sync_close( + self, + ref: weakref.ref[Any], + channelid: int, + error: RemoteError | None, + sendonly: bool, + ) -> None: + channel = ref() + if channel is None: + # channel already in "deleted" state + if error is not None: + error.warn() + self.sync_gateway._channelfactory._no_longer_opened(channelid) + return + channel._close_from_remote(error, sendonly=sendonly) + + def release_channel(self, channelid: int) -> None: + """Drop the loop-side raw channel for ``channelid`` (best-effort).""" + with suppress(trio.RunFinishedError): + self.host.portal.post(self._forget_channel, channelid) + + # -- receiver callbacks: a consumer task per channel -- + + def attach_consumer( + self, + channel: Any, + callback: Callable[[Any], Any], + endmarker: Endmarker, + ) -> None: + """Switch ``channel`` to callback mode: a loop task drains it. + + The switch runs on the loop (so it cannot interleave with delivery): + it moves any already-buffered items into the task's inbox, points the + channel's delivery/close at that inbox, and starts the consumer task. + It deliberately does *not* touch the raw channel's consumer (the sync + payload/close hooks bound by :meth:`bind_sync_channel` stay in place); + delivery keeps flowing through ``Channel._deliver_payload`` / + ``_close_from_remote``, which divert to the inbox once ``_has_consumer`` + is set. Rebinding the raw channel here would race the still-queued + ``bind()`` post (``run_sync`` and ``run_sync_soon`` are not mutually + ordered) and could be clobbered back to the mailbox. + + The task is handed a strong reference to ``channel`` and thus keeps it + alive for as long as it consumes -- the channel's lifecycle is bound to + the task (and to GC once the stream closes), not to a registry. + """ + + def switch() -> None: + mailbox = channel._mailbox + if mailbox is None: + raise OSError(f"{channel!r} has callback already registered") + inbox_send, inbox_recv = trio.open_memory_channel[bytes](math.inf) + + def feed(data: bytes) -> None: + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + inbox_send.send_nowait(data) + + def close_inbox() -> None: + with suppress(trio.ClosedResourceError): + inbox_send.close() + + # Drain items buffered before the switch into the task's inbox, + # preserving order. An ENDMARKER means the channel already closed. + saw_end = False + while True: + try: + item = mailbox.get_nowait() + except _queue.Empty: + break + if item is ENDMARKER: + saw_end = True + break + inbox_send.send_nowait(item) + channel._mailbox = None + done = Flag(channel.gateway._new_wakener()) + channel._consumer_done = done + channel._consumer_feed = feed + channel._consumer_close_inbox = close_inbox + + def stop() -> None: + # thread-safe: end the task's inbox from any thread (local close) + with suppress(trio.RunFinishedError, trio.ClosedResourceError): + self.host.portal.post(inbox_send.close) + + channel._consumer_stop = stop + channel._has_consumer = True + + # Guarantee the raw channel routes to us right now (idempotent with + # the bind posted at newchannel()): otherwise, if that bind has not + # run yet, inbound payloads would buffer unread in the raw channel + # and the consumer task would wait forever. + self._install_sync_consumer(weakref.ref(channel), channel.id) + + if saw_end: + close_inbox() + self.host.start_soon( + self._run_consumer, channel, inbox_recv, callback, endmarker, done + ) + + self.run_on_loop(switch) + + async def _run_consumer( + self, + channel: Any, + inbox: trio.MemoryReceiveChannel[bytes], + callback: Callable[[Any], Any], + endmarker: Endmarker, + done: Flag, + ) -> None: + """Drain ``inbox`` into ``callback`` (each call off the loop thread). + + Runs on the host loop; ``channel`` is held for the task's lifetime so + the channel stays alive while consuming. Items are delivered in order + and each callback runs in a threadpool thread. On completion (EOF, + local close, or a raising callback) the endmarker fires and ``done`` + is set -- which is what ``waitclose()`` waits on. + """ + limiter = self.host.callback_limiter + try: + async for data in inbox: + try: + await trio.to_thread.run_sync( + functools.partial(_run_callback, callback, data, channel), + limiter=limiter, + ) + except Exception as exc: + # trio.Cancelled is a BaseException and propagates past + # here (host shutdown); only a real callback/deserialize + # failure closes the channel with the error. + self._consumer_failed(channel, exc) + break + finally: + # Fire the endmarker and signal done even while the host is being + # torn down, but never let a stuck callback hang shutdown forever. + with trio.CancelScope(shield=True): + if endmarker is not NO_ENDMARKER_WANTED: + with ( + trio.move_on_after(CONSUMER_ENDMARKER_GRACE), + suppress(BaseException), + ): + await trio.to_thread.run_sync( + functools.partial(callback, endmarker), limiter=limiter + ) + done.set() + + def _consumer_failed(self, channel: Any, exc: BaseException) -> None: + """A callback (or its deserialization) raised: close with the error.""" + gateway = self.sync_gateway + gateway._trace("exception during callback: %s" % exc) + errortext = gateway._geterrortext(exc) + with suppress(OSError): + gateway._send( + Message.CHANNEL_CLOSE_ERROR, channel.id, dumps_internal(errortext) + ) + channel._close_from_remote(RemoteError(errortext), sendonly=False) + + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Blocking host call that parks correctly for the wait backend. + + ``wait=thread`` keeps the KI-deferred ``portal.run`` path; other + backends (gevent) wait on a OneShot with the gateway's wakener so + only the calling greenlet parks, not the whole hub. + """ + gateway = self.sync_gateway + if gateway._wait_backend == "thread": + return self.host.call(async_fn, *args) + pending = self.host.call_pending( + async_fn, *args, wakener=gateway._new_wakener() + ) + return pending.wait() + + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: + """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. + + Falls back to running inline once the loop is gone (no more + deliveries can interleave then anyway). + """ + portal = self.host.portal + if portal.is_loop_thread(): + return sync_fn() + try: + return portal.run_sync(sync_fn) + except trio.RunFinishedError: + return sync_fn() + + def enqueue_message(self, message: Message) -> None: + """Enqueue a frame; wait until written when safe to block. + + The Trio host thread (receiver callbacks) must not wait — that + would deadlock the writer task on the same event loop. + """ + frame = message.pack() + wait = not self.host.portal.is_loop_thread() + # The ack carries the write failure as a value (never raised into + # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable + # from a stream error. + ack: OneShot[BaseException | None] | None = ( + OneShot(self.sync_gateway._new_wakener()) if wait else None + ) + + def post() -> None: + try: + self.enqueue_frame(frame, ack.set if ack is not None else None) + except OSError as exc: + if ack is not None: + ack.set(exc) + + with self._send_lock: + if self._send_closed: + raise OSError("cannot send (already closed?)") + try: + # Through the portal even from the host thread so every + # send lands in one global FIFO order. + self.host.portal.post(post) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None + if ack is None: + return + try: + error = ack.wait(timeout=120.0) + except TimeoutError: + raise OSError("cannot send (write timed out)") from None + if error is not None: + raise OSError("cannot send (already closed?)") from error + + def post_message(self, message: Message) -> None: + """Best-effort non-waiting send (Channel.__del__ during GC).""" + frame = message.pack() + + def post() -> None: + with suppress(OSError): + self.enqueue_frame(frame) + + try: + self.host.portal.post(post) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None + + def request_close_write(self) -> None: + with self._send_lock: + if self._send_closed: + return + self._send_closed = True + with suppress(trio.RunFinishedError): + # The writer drains queued frames, then signals write-EOF. + self.host.portal.post(self._outbound_send.close) + + def wait_done(self, timeout: float | None = None) -> bool: + try: + self._done_sync.wait(timeout) + except TimeoutError: + return False + return True + + def is_alive(self) -> bool: + return not self._done_sync.is_set() + + +class TrioHost: + """Dedicated OS thread running ``trio.run`` for protocol IO.""" + + def __init__( + self, + name: str = "execnet-trio-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: + self._name = name + self._callback_threads = callback_threads + self._thread: threading.Thread | None = None + self._portal: LoopPortal | None = None + self._nursery: trio.Nursery | None = None + self._ready = threading.Event() + self._shutdown: trio.Event | None = None + self._started = False + self._callback_limiter: trio.CapacityLimiter | None = None + + def start(self) -> None: + if self._started: + return + self._thread = threading.Thread(target=self._run, name=self._name, daemon=True) + self._thread.start() + if not self._ready.wait(timeout=30): + raise RuntimeError("TrioHost failed to start") + self._started = True + + @property + def portal(self) -> LoopPortal: + if self._portal is None: + raise RuntimeError("TrioHost is not running") + return self._portal + + @property + def callback_limiter(self) -> trio.CapacityLimiter: + """Bound on concurrent threadpool threads running receiver callbacks.""" + if self._callback_limiter is None: + raise RuntimeError("TrioHost is not running") + return self._callback_limiter + + def is_host_thread(self) -> bool: + return self._portal is not None and self._portal.is_loop_thread() + + def _run(self) -> None: + trio.run(self._main) + + async def _main(self) -> None: + self._portal = LoopPortal() + self._shutdown = trio.Event() + self._callback_limiter = trio.CapacityLimiter(self._callback_threads) + try: + async with trio.open_nursery() as nursery: + self._nursery = nursery + self._ready.set() + await self._shutdown.wait() + nursery.cancel_scope.cancel() + finally: + self._nursery = None + + def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + return self.portal.run(async_fn, *args) + + def call_pending( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + wakener: Any = None, + ) -> OneShot[T]: + """Run ``async_fn`` as a host task, resolving a :class:`OneShot`. + + The non-blocking counterpart of :meth:`call` for consumers that + must not block their OS thread (a gevent hub: waiting on the + OneShot with a gevent wakener parks only the calling greenlet). + Unlike ``portal.run`` the wait is KeyboardInterrupt-interruptible. + """ + result: OneShot[T] = OneShot(wakener) + + async def runner() -> None: + try: + value = await async_fn(*args) + except trio.Cancelled: + if not result.is_set(): + result.set_error(RuntimeError("trio host was shut down")) + raise + except BaseException as exc: + result.set_error(exc) + else: + result.set(value) + + def spawn() -> None: + self.start_soon(runner) + + self.portal.post(spawn) + return result + + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + return self.portal.run_sync(sync_fn, *args) + + def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: + """Schedule a task on the root nursery (must be called on the host thread).""" + if not self.is_host_thread(): + raise RuntimeError("start_soon requires the Trio host thread") + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + self._nursery.start_soon(async_fn, *args) + + async def start_session( + self, gateway: BaseGateway, io: ByteStream + ) -> SyncBridgeGateway: + """Serve ``gateway`` over ``io`` as a task on the root nursery.""" + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + session = SyncBridgeGateway( + io, id=str(gateway.id), sync_gateway=gateway, host=self + ) + await self._nursery.start(session._serve) + return session + + def stop(self, timeout: float | None = 5.0) -> None: + if not self._started or self._portal is None or self._shutdown is None: + return + + def _set() -> None: + assert self._shutdown is not None + self._shutdown.set() + + try: + self._portal.run_sync(_set) + except Exception: + pass + if self._thread is not None: + self._thread.join(timeout=timeout) + self._started = False + + +class _TempIO: + """Placeholder IO used only while constructing a Trio-backed Gateway.""" + + def __init__(self, execmodel: ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio temp IO") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio temp IO") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +class FacadeAsyncGroup(AsyncGroup): + """AsyncGroup owning the async side of a sync ``Group``. + + Runs on the group's :class:`TrioHost`. Gateways come out as + :class:`SyncBridgeGateway` objects bound to freshly built sync + ``Gateway`` facades, and the via / installvia flows go through the sync + master gateway (its dispatch is sync, so async channels cannot be used + on it). + """ + + def __init__(self, group: Group, host: TrioHost) -> None: + super().__init__() + self.group = group + self.host = host + self.shutdown = trio.Event() + + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + import execnet + + sync_gw = execnet.Gateway(_TempIO(get_execmodel(spec.profile)), spec) + # the caller's concurrency library, inherited from the facade + sync_gw._wait_backend = self.group._wait_backend + return SyncBridgeGateway( + stream, id=spec.id, sync_gateway=sync_gw, host=self.host + ) + + async def _open_via_stream(self, spec: Any) -> ByteStream: + from . import _provision + + master = self.group[spec.via] + session = master._trio_session + assert isinstance(session, SyncBridgeGateway) + channelid = master._channelfactory.allocate_id() + # Create the raw channel before the request goes out so no relayed + # frame can arrive unrouted (we are on the loop: no dispatch races). + io = RawChannelStream(session._channel_for(channelid)) + request = _provision.spawn_request(spec) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) + await read_handshake_ack(io, "via") + return io + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + if getattr(spec, "installvia", None): + master = self.group[spec.installvia] + # Blocking sync channel receive on the master: run in a thread + # while this loop keeps dispatching the master's messages. + realhost, realport = await trio.to_thread.run_sync( + start_socketserver_via, master, abandon_on_cancel=True + ) + return (realhost, realport), "%s:%d" % (realhost, realport) + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + return (host_str, int(port_str)), spec.socket + + async def run(self, task_status: trio.TaskStatus[FacadeAsyncGroup]) -> None: + """Own the group nursery as a host task until :attr:`shutdown`.""" + async with self: + task_status.started(self) + await self.shutdown.wait() + + +def makegateway_trio(group: Group, spec: Any) -> Gateway: + """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" + host: TrioHost = group._ensure_trio_host() + async_group: FacadeAsyncGroup = group._ensure_async_group() + if group._wait_backend == "thread": + bridge = host.call(async_group.makegateway, spec) + else: + # e.g. a gevent app: wait on a OneShot so only the calling + # greenlet parks while the gateway comes up, not the whole hub. + from ._boundary import make_wakener + + bridge = host.call_pending( + async_group.makegateway, spec, wakener=make_wakener(group._wait_backend) + ).wait() + assert isinstance(bridge, SyncBridgeGateway) + gw: Gateway = bridge.sync_gateway # type: ignore[assignment] + gw._io = SyncIOHandle( + get_execmodel(spec.profile), + bridge, + process=async_group._processes.get(bridge), + remoteaddress=bridge.remoteaddress, + ) + return gw + + +_socket_worker_counter = itertools.count() + + +def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: + """Spawn a worker subprocess serving over the inherited socket ``fd``.""" + import execnet + + config = json.dumps( + { + "id": "socketworker%d" % next(_socket_worker_counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + return subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(fd), + "--config", + config, + ], + pass_fds=[fd], + ) + + +async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> None: + """Hand an accepted socket to a fresh worker subprocess (server side). + + ``reap`` waits for the worker (loop server); when false the worker outlives + this task (one-shot / installvia). + """ + proc = _spawn_socket_worker(stream.socket.fileno()) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if reap: + await trio.to_thread.run_sync(proc.wait) + + +async def _start_socket_and_reply( + gateway: BaseGateway, channelid: int, bind_host: str +) -> None: + """Bind an ephemeral port, reply with its address, then serve one connection. + + Runs as a task on the worker's Trio host (scheduled from the message + handler). The reply travels back on ``channelid`` like a STATUS reply. + """ + listeners = await trio.open_tcp_listeners(0, host=bind_host) + addr = listeners[0].socket.getsockname() + gateway._send(Message.CHANNEL_DATA, channelid, dumps_internal((addr[0], addr[1]))) + gateway._send(Message.CHANNEL_CLOSE, channelid) + + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + await serve_socket_connection(stream, reap=True) + + +def handle_start_socket(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SOCKET`` (on the host thread).""" + bind_host = loads_internal(data) + assert isinstance(bind_host, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + # The receiver runs on the host thread, so schedule the async work directly. + host.start_soon(_start_socket_and_reply, gateway, channelid, bind_host) + + +def start_socketserver_via( + via_gateway: Any, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``via_gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = via_gateway.newchannel() + via_gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = channel.receive() + channel.waitclose() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) + + +async def _run_delivery_step(argv: list[str], payload: bytes) -> None: + """Run an out-of-band delivery command, feeding ``payload`` to its stdin.""" + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(payload) + await process.stdin.aclose() + code = await process.wait() + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + if code != 0: + raise RuntimeError(f"delivery step failed with exit {code}: {argv[0]}") + + +async def _start_sub_and_relay( + gateway: BaseGateway, channelid: int, request: dict[str, Any] +) -> None: + """Spawn a requested sub-worker and relay its Message protocol frames. + + Runs on the master's Trio host (the ``via`` transport). The tunnel is + frame-native both ways: coordinator payloads arrive verbatim through the + session's raw channel and go to the sub's stdin unchanged (each payload + one whole frame), while the sub's stdout runs through a FrameDecoder so + every CHANNEL_DATA sent back carries exactly one frame -- except the + initial ready byte, which is forwarded on its own for the handshake. + A shipped wheel (dev-version ssh sub) is delivered first, over its own + connection, so the relayed stream carries protocol bytes only. + """ + from . import _provision + + def send_close_error(text: str) -> None: + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE_ERROR, channelid, dumps_internal(text)) + + try: + args, delivery = _provision.sub_spawn_argv(request) + if delivery is not None: + await _run_delivery_step(*delivery) + process = await open_popen_process(args) + except Exception as exc: + send_close_error(f"could not spawn via sub-gateway: {exc}") + return + session = gateway._trio_session + assert isinstance(session, SyncBridgeGateway) + raw = session._channel_for(channelid) + + async def coordinator_to_sub() -> None: + assert process.stdin is not None + with suppress(RemoteError): + async for data in raw: + await process.stdin.send_all(data) + with trio.move_on_after(5): + await process.stdin.aclose() + + async def sub_to_coordinator() -> None: + assert process.stdout is not None + ack = bytes(await process.stdout.receive_some(1)) + if ack: + gateway._send(Message.CHANNEL_DATA, channelid, ack) + decoder = FrameDecoder() + while True: + data = bytes(await process.stdout.receive_some(RECEIVE_CHUNK)) + if not data: + break + for message in decoder.feed(data): + gateway._send(Message.CHANNEL_DATA, channelid, message.pack()) + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE, channelid) + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(coordinator_to_sub) + nursery.start_soon(sub_to_coordinator) + except Exception as exc: + # Do not let a relay failure crash the host nursery; surface it on + # the channel so the coordinator does not hang on the handshake. + gateway._trace("via sub relay failed:", exc) + send_close_error(f"via sub-gateway relay failed: {exc}") + finally: + session._forget_channel(channelid) + with trio.move_on_after(5): + await process.wait() + + +def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SUB`` (on the host thread).""" + request = loads_internal(data) + assert isinstance(request, dict) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + host.start_soon(_start_sub_and_relay, gateway, channelid, request) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py new file mode 100644 index 00000000..50faccb4 --- /dev/null +++ b/src/execnet/_trio_worker.py @@ -0,0 +1,808 @@ +"""Worker-side Trio networking and exec scheduling for popen/import bootstrap.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +import threading +from collections.abc import Callable +from collections.abc import Sequence +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol + +import trio + +from ._boundary import Mailbox +from ._boundary import WaitBackend +from ._errors import geterrortext +from ._execmodel import effective_profile +from ._execmodel import get_execmodel +from ._gateway_base import WorkerGateway +from ._serialize import loads_internal +from ._trace import trace + +if TYPE_CHECKING: + from . import _trio_host + from ._channel import Channel + from ._execmodel import ExecModel + +ExecItem = tuple[Any, ...] + + +class PoolExec: + """Exec strategy: run each request on a worker thread (trio's pool). + + The building block of the ``thread`` profile; exec'd code may freely + start its own event loops (it never shares a thread with ours). + """ + + #: whether _run_worker must hand this strategy the process main thread + needs_primary_thread = False + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + """FIFO admission gate; always open for pool placement.""" + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + + def integrate_as_primary_thread(self) -> None: + raise RuntimeError("pool exec strategy does not use the main thread") + + def trigger_shutdown(self) -> None: + pass + + +class PrimaryThreadPump: + """Runs exec requests handed to it on the process main thread. + + The building block for every strategy that needs a real main thread: + :meth:`integrate_as_primary_thread` parks there draining a mailbox, and + :meth:`run` hands one request over and awaits its completion. + """ + + #: whether _run_worker must hand this strategy the process main thread + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + """FIFO admission gate; always open.""" + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def released(self) -> None: + """Hook: the main thread is free again (called on it, before done).""" + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running exec tasks until shutdown.""" + while True: + task = self._primary.get() + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + # Release before signalling: the next request should see the + # main thread free as early as we can make it. + self.released() + done.set() + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + +class HybridExec(PrimaryThreadPump): + """Exec strategy: primary on the main thread, overflow on pool threads. + + The classic ``thread`` profile shape: a request arriving while the + main thread is idle claims it (pytest and friends get a true main + thread); requests arriving while it is busy run on worker threads + instead of queueing. The claim is decided during FIFO admission so + the *first* request always gets the main thread. + + This is also what the retired ``main_thread_only`` profile now maps to: + it existed for the main-thread guarantee, which the claim provides, + and its extra behaviour -- refusing a second concurrent remote_exec + rather than overflowing -- was a deadlock guard, not a feature. + + One difference from that profile is worth knowing: it *serialized*, so + every sequential remote_exec was guaranteed the main thread. Here the + claim is released as the exec finishes, while the channel close that + tells the coordinator it may send the next one is emitted a moment + earlier -- so a coordinator that immediately re-execs can, rarely, be + admitted before the release lands and get a pool thread instead. The + *first* request always gets the main thread; a caller that needs the + guarantee for every request wants the ``trio`` or ``gevent`` profile, + where placement is not a race. + """ + + def __init__(self, gateway: WorkerGateway) -> None: + super().__init__(gateway) + self._pool = PoolExec(gateway) + self._claim_lock = threading.Lock() + self._primary_busy = False + self._claimed: set[int] = set() + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + with self._claim_lock: + if not self._primary_busy: + self._primary_busy = True + self._claimed.add(channel.id) + return True + + def released(self) -> None: + with self._claim_lock: + self._primary_busy = False + + async def run(self, channel: Channel, item: ExecItem) -> None: + with self._claim_lock: + claimed = channel.id in self._claimed + self._claimed.discard(channel.id) + if not claimed: + await self._pool.run(channel, item) + return + try: + await super().run(channel, item) + finally: + # released() normally cleared this on the main thread already; + # repeat it so a cancelled or failed run cannot strand the claim + self.released() + + +class GreenletExec: + """Exec strategy: greenlets on a gevent hub owning the main thread. + + ``profile=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's gevent wakeners make channel + operations park the greenlet, so concurrent remote_execs cooperate on + the one main thread. Requires gevent in the worker environment + (provisioning adds the ``gevent`` requirement automatically). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + from ._boundary import make_wakener + + self.gateway = gateway + # The integrate loop blocks in get() on the hub thread: a gevent + # wakener parks only its root greenlet, letting exec greenlets run. + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox(make_wakener("gevent")) + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Run the hub on the main thread, spawning a greenlet per exec.""" + import gevent + + def run_exec(channel: Channel, item: ExecItem, done: threading.Event) -> None: + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + while True: + task = self._primary.get() + if task is None: + break + gevent.spawn(run_exec, *task) + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + +# worker profile -> exec strategy for the sync-facade worker; the "trio" +# profile serves a plain AsyncGateway instead (TaskExec below). +# Future placement strategies slot in here (e.g. subinterpreters). +WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { + "thread": HybridExec, + "gevent": GreenletExec, +} + + +class TaskExec: + """Exec strategy for the pure-async profile (``profile=trio``). + + Sources run as tasks on the worker's own loop, in the one and only + thread of the process, and receive an ``AsyncChannel``. Sources must + be async: a plain function or a source string without top-level + ``await`` is rejected before it can starve the loop. Gateway + termination cancels running exec tasks (``trio.Cancelled`` inside the + source). + """ + + def __init__(self, gateway: Any, nursery: trio.Nursery) -> None: + self.gateway = gateway + self.nursery = nursery + self._running = 0 + + def active_count(self) -> int: + return self._running + + def handle_exec(self, gateway: Any, channelid: int, data: bytes) -> None: + """CHANNEL_EXEC hook; runs inline on the dispatch task.""" + item = loads_internal(data) + assert isinstance(item, tuple) + channel = gateway.open_channel(channelid) + self.nursery.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Any, item: ExecItem) -> None: + import ast + import inspect + + source, file_name, call_name, kwargs = item + self._running += 1 + try: + trace(f"async execution starts[{channel.id}]: {source[:50]!r}") + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + co = compile( + source + "\n", + file_name or "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) + if not toplevel_await and not call_name: + await channel.aclose( + "sync source under profile=trio: the source must use" + " top-level await (or pass an async function)" + ) + return + if toplevel_await: + await eval(co, loc) + else: + exec(co, loc) # define the function (no top-level awaits) + if call_name: + function = loc[call_name] + result = function(channel, **kwargs) + if not hasattr(result, "__await__"): + await channel.aclose( + f"sync function {call_name!r} under profile=trio:" + " remote_exec functions must be async" + ) + return + await result + except trio.Cancelled: + raise + except EOFError: + trace("ignoring EOFError from async exec") + except BaseException as exc: + trace(f"async exec got exception: {exc!r}") + await channel.aclose(geterrortext(exc)) + return + finally: + self._running -= 1 + trace("async execution finished") + await channel.aclose() + + +class TrioWorkerExec: + """FIFO admission pump feeding an exec placement strategy. + + Exec requests flow through a single pump task so admission happens + strictly in message-arrival order (trio task scheduling order is + deliberately unordered, so per-request tasks would race for e.g. the + main-thread claim). Where an admitted request runs is the + strategy's business (:data:`WORKER_EXEC_STRATEGIES`). + """ + + def __init__( + self, + host: _trio_host.TrioHost, + gateway: WorkerGateway, + strategy: Any, + ) -> None: + self.host = host + self.gateway = gateway + self.strategy = strategy + self._lock = threading.Lock() + self._running = 0 + self._shutting_down = False + self._idle = threading.Event() + self._idle.set() + self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] + self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] + self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) + self._pump_started = False + + @property + def needs_primary_thread(self) -> bool: + return bool(self.strategy.needs_primary_thread) + + def active_count(self) -> int: + with self._lock: + return self._running + + def _track_start(self) -> None: + with self._lock: + self._running += 1 + self._idle.clear() + + def _track_finish(self) -> None: + with self._lock: + self._running -= 1 + if self._running == 0: + self._idle.set() + + def schedule(self, channel: Channel, sourcetask: bytes) -> None: + """Called from the session dispatch on the Trio host thread. + + Must not block: admission checks and exec run in a nursery task. + """ + item = loads_internal(sourcetask) + assert isinstance(item, tuple) + with self._lock: + if self._shutting_down: + channel.close("execution disallowed") + return + # Already on the Trio host thread (Message handler). + if not self._pump_started: + self._pump_started = True + self.host.start_soon(self._pump) + self._pending_send.send_nowait((channel, item)) + + async def _pump(self) -> None: + """Admit queued exec requests in FIFO order, then run each as a task.""" + async for channel, item in self._pending_recv: + if await self.strategy.admit(channel, item): + self.host.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: + self._track_start() + try: + await self.strategy.run(channel, item) + finally: + self._track_finish() + + def integrate_as_primary_thread(self) -> None: + self.strategy.integrate_as_primary_thread() + + def trigger_shutdown(self) -> None: + with self._lock: + self._shutting_down = True + self.strategy.trigger_shutdown() + + def waitall(self, timeout: float | None = None) -> bool: + return self._idle.wait(timeout) + + +def _devnull() -> str: + try: + return os.devnull + except AttributeError: # pragma: no cover - defensive + return "NUL" if os.name == "nt" else "/dev/null" + + +def _dup_protocol_fds() -> tuple[int, int]: + """Move the protocol off stdin/stdout, leaving fd 0/1 free to redirect. + + Returns ``(read_fd, write_fd)`` for the Message protocol (the worker + reads what the coordinator writes to our stdin, and writes what the + coordinator reads from our stdout). What happens to fd 0/1 afterwards + is the caller's choice -- see :func:`apply_stdio`. + """ + if not hasattr(os, "dup"): # pragma: no cover - jython legacy + raise RuntimeError("the execnet worker requires os.dup") + return os.dup(0), os.dup(1) + + +def apply_stdio( + stdin: str = "inherit", stdout: str = "inherit", stderr: str = "inherit" +) -> None: + """Point the worker's standard fds where the launcher asked. + + ``inherit`` leaves an fd alone, which is the default once the protocol + has a transport of its own: a worker's output is then the user's, not + something execnet has to swallow to protect the wire. + + ``close`` (stdin only) reopens fd 0 on the null device *and* closes + ``sys.stdin``. Reads through Python raise, while the fd itself stays + reserved -- genuinely closing it would let the next ``os.open`` land on + fd 0, where anything writing to "stdin" would corrupt an unrelated file. + """ + if stdin in ("close", "devnull"): + fd = os.open(_devnull(), os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if stdin == "close": + with suppress(Exception): + sys.stdin.close() + sys.stdin = os.fdopen(0, "r", closefd=False) + sys.stdin.close() + else: + sys.stdin = os.fdopen(0, "r", closefd=False) + + if stdout == "stderr": + os.dup2(2, 1) + sys.stdout = os.fdopen(1, "w", buffering=1, closefd=False) + elif stdout == "devnull": + fd = os.open(_devnull(), os.O_WRONLY) + os.dup2(fd, 1) + os.close(fd) + sys.stdout = os.fdopen(1, "w", buffering=1, closefd=False) + + if stderr == "devnull": + fd = os.open(_devnull(), os.O_WRONLY) + os.dup2(fd, 2) + os.close(fd) + sys.stderr = os.fdopen(2, "w", buffering=1, closefd=False) + + +class _WorkerIOStub: + """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" + + def __init__(self, execmodel: ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not used on Trio worker") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not used on Trio worker") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def _build_worker_gateway( + host: _trio_host.TrioHost, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", +) -> tuple[WorkerGateway, TrioWorkerExec]: + """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" + trace(f"creating workergateway on trio id={id!r}") + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + gateway._wait_backend = wait + + try: + strategy_factory = WORKER_EXEC_STRATEGIES[effective_profile(model.backend)] + except KeyError: + raise ValueError( + f"profile {model.backend!r} has no worker exec strategy " + f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" + ) from None + strategy = strategy_factory(gateway) + trio_exec = TrioWorkerExec(host, gateway, strategy) + # Duck-type as the exec pool for STATUS / _terminate_execution. + gateway._execpool = trio_exec + gateway._trio_exec = trio_exec + return gateway, trio_exec + + +def _run_worker( + host: _trio_host.TrioHost, + io: Any, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", +) -> None: + """Attach ``io`` as the gateway session and serve until shutdown.""" + gateway, trio_exec = _build_worker_gateway(host, id, model, wait) + + async def _start() -> _trio_host.SyncBridgeGateway: + # The bridge attaches itself to the gateway before serving starts, + # so inbound messages can reply through gateway._send right away. + return await host.start_session(gateway, io) + + host.call(_start) + + try: + if trio_exec.needs_primary_thread: + trace("integrating as primary thread (trio worker)") + trio_exec.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) + + +async def _make_fd_io(read_fd: int, write_fd: int) -> Any: + from . import _trio_gateway + + return _trio_gateway.staple_fd_stream(read_fd, write_fd) + + +async def _serve_async_worker(stream: Any, id: str) -> None: + """Serve a plain AsyncGateway with task-based exec (profile=trio). + + The whole worker is this one trio run on the process main thread: the + dispatch loop and every exec'd source share it. Termination cancels + running exec tasks. + """ + from ._trio_gateway import AsyncGateway + + gateway = AsyncGateway(stream, id=id, _startcount=2) + async with trio.open_nursery() as nursery: + task_exec = TaskExec(gateway, nursery) + gateway._exec_handler = task_exec.handle_exec + gateway._task_exec = task_exec + await nursery.start(gateway._serve) + await gateway.wait_closed() + nursery.cancel_scope.cancel() + + +class Transport(Protocol): + """How a worker's protocol stream comes into being. + + Split in two because the stdio transport has to claim fd 0/1 *before* + anything else touches them (including reading ``--config-fd 0``), while + opening the stream itself needs a running trio loop. + """ + + #: default stdio disposition once this transport is serving + stdio_defaults: tuple[str, str, str] + + def prepare(self) -> None: + """Synchronous fd bookkeeping, before the config is read.""" + + async def open(self) -> Any: + """The protocol ByteStream, ready for the Message protocol.""" + + +async def _send_ready(stream: Any) -> Any: + """Write the single handshake byte the coordinator waits for.""" + await stream.send_all(b"1") + return stream + + +class StdioTransport: + """The protocol *is* this process's stdin/stdout (the classic shape). + + Nothing else can use fd 0/1 afterwards, so the default disposition + closes stdin and folds stdout onto stderr -- remote ``print()`` stays + visible on the coordinator instead of going to the null device, and it + cannot corrupt the wire because the wire is no longer fd 1. + """ + + stdio_defaults = ("close", "stderr", "inherit") + + def __init__(self) -> None: + self._fds: tuple[int, int] | None = None + + def prepare(self) -> None: + self._fds = _dup_protocol_fds() + + async def open(self) -> Any: + from ._trio_gateway import staple_fd_stream + + assert self._fds is not None, "prepare() first" + read_fd, write_fd = self._fds + return await _send_ready(staple_fd_stream(read_fd, write_fd)) + + +class FdTransport: + """The protocol runs over inherited fds: one socket, or a pipe pair.""" + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, fds: Sequence[int]) -> None: + self.fds = tuple(fds) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + from . import _trio_host + from ._trio_gateway import staple_fd_stream + + if len(self.fds) == 2: + read_fd, write_fd = self.fds + return await _send_ready(staple_fd_stream(read_fd, write_fd)) + (fd,) = self.fds + if not stat.S_ISSOCK(os.fstat(fd).st_mode): + raise ValueError( + f"--protocol-fd {fd} is not a socket; a single fd must be" + " bidirectional, use --protocol-fd READFD,WRITEFD for a pipe pair" + ) + # adopt_socket sends the handshake itself + return await _trio_host.adopt_socket(fd) + + +def parse_address(address: str) -> tuple[str, Any]: + """``unix:/path`` or ``host:port`` -> ``("unix", path)`` / ``("tcp", (h, p))``.""" + if address.startswith("unix:"): + return "unix", address[len("unix:") :] + host, sep, port = address.rpartition(":") + if not sep or not port.isdigit(): + raise ValueError(f"expected unix:/path or host:port, got {address!r}") + return "tcp", (host or "localhost", int(port)) + + +class ConnectTransport: + """The worker dials out to the coordinator and serves on that connection. + + This is what lets ssh carry the protocol without owning our stdio: the + coordinator listens on a local unix socket, ``ssh -R`` forwards it, and + we connect to the remote end of the forward. + """ + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, address: str) -> None: + self.kind, self.target = parse_address(address) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + if self.kind == "unix": + stream = await trio.open_unix_socket(self.target) + else: + stream = await trio.open_tcp_stream(*self.target) + return await _send_ready(stream) + + +class ListenTransport: + """The worker listens and serves the first coordinator that connects. + + The bound address is printed to stdout as JSON before serving, so a + launcher that asked for an ephemeral port can learn which one it got. + """ + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, address: str) -> None: + self.kind, self.target = parse_address(address) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + if self.kind == "unix": + sock = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM) + await sock.bind(self.target) + sock.listen(1) + listeners = [trio.SocketListener(sock)] + bound: Any = self.target + else: + host, port = self.target + listeners = await trio.open_tcp_listeners(port, host=host) + bound = listeners[0].socket.getsockname()[:2] + print(json.dumps({"listening": bound}), flush=True) + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + return await _send_ready(stream) + + +def serve_worker( + config: dict[str, Any], + transport: Transport, + *, + stdin: str | None = None, + stdout: str | None = None, + stderr: str | None = None, +) -> None: + """Serve one gateway for ``config`` over ``transport``, then exit. + + ``transport.prepare()`` must already have run (the CLI does it before + reading the config, so ``--config-fd 0`` still works under the stdio + transport). + """ + _check_version(config["coordinator_version"]) + _apply_worker_setup(config) + defaults = transport.stdio_defaults + apply_stdio( + stdin=stdin if stdin is not None else defaults[0], + stdout=stdout if stdout is not None else defaults[1], + stderr=stderr if stderr is not None else defaults[2], + ) + + id = config["id"] + # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed + # coordinator still connects. + profile = effective_profile(config.get("profile") or config["execmodel"]) + wait: WaitBackend = config.get("wait", "thread") + + if profile == "trio": + # pure-async profile: one thread, the loop owns the main thread + async def main() -> None: + await _serve_async_worker(await transport.open(), id) + + trio.run(main) + os._exit(0) + + from . import _trio_host + + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + io = host.call(transport.open) + _run_worker(host, io, id, get_execmodel(profile), wait) + + +def _rough_version(version: str) -> tuple[int, ...]: + """Leading numeric (major, minor) of a version string; ``()`` if unparsable.""" + parts: list[int] = [] + for chunk in version.split(".")[:2]: + number = "" + for char in chunk: + if char.isdigit(): + number += char + else: + break + if not number: + break + parts.append(int(number)) + return tuple(parts) + + +def _apply_worker_setup(config: dict[str, Any]) -> None: + """Apply chdir/nice/env from the worker config, before serving starts. + + This replaces the classic post-start ``remote_exec`` setup: valid for + every profile (a trio worker cannot run sync sources) and never + claims an exec slot. + """ + path = config.get("chdir") + if path: + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + nice = config.get("nice") + if nice and hasattr(os, "nice"): + os.nice(nice) + for name, value in config.get("env", {}).items(): + os.environ[name] = value + + +def _check_version(coordinator_version: str) -> None: + """Warn on a real (major/minor) execnet version mismatch across the wire. + + A minimal (patch-level) mismatch is tolerated. For same-interpreter popen + the versions are always identical; this guards the future remote paths. + """ + import execnet + + ours = _rough_version(execnet.__version__) + theirs = _rough_version(coordinator_version) + if ours and theirs and ours != theirs: + sys.stderr.write( + "WARNING: execnet version mismatch: coordinator %s worker %s\n" + % (coordinator_version, execnet.__version__) + ) + sys.stderr.flush() diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py new file mode 100644 index 00000000..1d2b3af2 --- /dev/null +++ b/src/execnet/_xspec.py @@ -0,0 +1,93 @@ +""" +(c) 2008-2013, holger krekel +""" + +from __future__ import annotations + + +class XSpec: + """Execution Specification: key1=value1//key2=value2 ... + + * Keys need to be unique within the specification scope + * Neither key nor value are allowed to contain "//" + * Keys are not allowed to contain "=" + * Keys are not allowed to start with underscore + * If no "=value" is given, assume a boolean True value + """ + + #: keys accepted under an older spelling, mapped to the canonical one. + #: ``execmodel`` described a *local* execution model that no longer + #: exists; what the key actually selects is the worker's profile. + _ALIASES = {"execmodel": "profile"} + + # XXX allow customization, for only allow specific key names + chdir: str | None = None + dont_write_bytecode: bool | None = None + profile: str | None = None + id: str | None = None + installvia: str | None = None + nice: str | None = None + popen: bool | None = None + python: str | None = None + socket: str | None = None + ssh: str | None = None + ssh_config: str | None = None + stderr: str | None = None + stdin: str | None = None + stdout: str | None = None + transport: str | None = None + vagrant_ssh: str | None = None + via: str | None = None + + def __init__(self, string: str) -> None: + self._spec = string + self.env = {} + for keyvalue in string.split("//"): + i = keyvalue.find("=") + value: str | bool + if i == -1: + key, value = keyvalue, True + else: + key, value = keyvalue[:i], keyvalue[i + 1 :] + if key[0] == "_": + raise AttributeError("%r not a valid XSpec key" % key) + # duplicates are checked on the canonical name, so + # ``execmodel=x//profile=y`` is rejected like any other clash + if self._ALIASES.get(key, key) in self.__dict__: + raise ValueError(f"duplicate key: {key!r} in {string!r}") + if key.startswith("env:"): + self.env[key[4:]] = value + else: + setattr(self, key, value) + + def __getattr__(self, name: str) -> None | bool | str: + if name[0] == "_": + raise AttributeError(name) + return None + + @property + def execmodel(self) -> str | None: + """Deprecated alias for :attr:`profile` (accepted indefinitely).""" + return self.profile + + @execmodel.setter + def execmodel(self, value: str | None) -> None: + self.profile = value + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self._spec + + def __hash__(self) -> int: + return hash(self._spec) + + def __eq__(self, other: object) -> bool: + return self._spec == getattr(other, "_spec", None) + + def __ne__(self, other: object) -> bool: + return self._spec != getattr(other, "_spec", None) + + def _samefilesystem(self) -> bool: + return self.popen is not None and self.chdir is None diff --git a/src/execnet/aio.py b/src/execnet/aio.py new file mode 100644 index 00000000..fb4f6c2a --- /dev/null +++ b/src/execnet/aio.py @@ -0,0 +1,415 @@ +"""The asyncio-native execnet API. + +Everything here is awaited inside your own asyncio event loop:: + + import asyncio + import execnet.aio + + async def main(): + async with execnet.aio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + asyncio.run(main()) + +Protocol IO keeps running on a Trio host thread (the same engine as the +blocking and trio-native APIs, all transports included); each awaited +operation runs as a task on that host and resolves an asyncio future via +``loop.call_soon_threadsafe``. No anyio port and no executor threads per +call. + +Cancellation crosses the bridge. Cancelling an awaited ``receive`` (say +by ``asyncio.timeout``) cancels the host-side operation too, so no item +is consumed and dropped. Operations that must not tear halfway -- +``send``, ``send_eof``, ``aclose``, ``terminate`` -- are shielded +instead: the ``CancelledError`` reaches you, but the operation still +completes on the host. + +The error types are shared with :mod:`execnet.sync` and +:mod:`execnet.trio`. Items you send must already be simple builtin data +(plus channels); the standalone serializer is intentionally not part of +the public API -- ``execnet.can_send`` checks a value before you send it; +see ``DumpError``. +""" + +from __future__ import annotations + +import asyncio +import functools +import types +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +import trio + +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._host import Host +from ._host import default_host +from ._trio_gateway import AsyncChannel as _TrioChannel +from ._trio_gateway import AsyncGateway as _TrioGateway +from ._trio_gateway import AsyncGroup as _TrioGroup +from ._xspec import XSpec + +if TYPE_CHECKING: + from typing_extensions import Self + +__all__ = [ + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", + "DataFormatError", + "DumpError", + "Host", + "HostNotFound", + "LoadError", + "RemoteError", + "TimeoutError", + "XSpec", + "default_host", + "open_gateway", +] + +T = TypeVar("T") + + +class _HostBridge: + """Await trio-native coroutines on a Trio host from asyncio.""" + + def __init__(self, trio_host: Any) -> None: + self._host = trio_host + + async def call( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + shield: bool = False, + ) -> T: + """Run ``async_fn`` on the host and await its result. + + Unless ``shield``, cancelling the await cancels the host-side + operation too, so a cancelled ``receive`` does not consume an item + that nobody will ever see. + """ + loop = asyncio.get_running_loop() + future: asyncio.Future[T] = loop.create_future() + # Built here rather than inside the task: the cancel post can + # otherwise overtake the task's first step, and cancelling a scope + # nobody has entered yet still cancels it once entered. + scope = trio.CancelScope() + + def resolve(result: Any, error: BaseException | None) -> None: + if future.cancelled(): + return + if error is not None: + future.set_exception(error) + else: + future.set_result(result) + + def post_result(result: Any, error: BaseException | None) -> None: + # The asyncio loop may already be gone at interpreter/test + # teardown; the result is undeliverable then. + with suppress(RuntimeError): + loop.call_soon_threadsafe(resolve, result, error) + + async def runner() -> None: + try: + with scope: + result = await async_fn(*args) + except trio.Cancelled: + # host shutdown: the nursery cancel must propagate + post_result(None, RuntimeError("execnet aio host was shut down")) + raise + except BaseException as exc: + post_result(None, exc) + return + if scope.cancelled_caught: + # cancelled by us -- the awaiter is already gone + return + post_result(result, None) + + def spawn() -> None: + self._host.start_soon(runner) + + try: + self._host.portal.post(spawn) + except trio.RunFinishedError: + raise RuntimeError("execnet aio group is not running") from None + + if shield: + # The host-side work runs to completion either way; shielding + # keeps a cancelled caller from abandoning a half-done send. + return await asyncio.shield(future) + try: + return await future + except asyncio.CancelledError: + with suppress(trio.RunFinishedError): + self._host.portal.post(scope.cancel) + raise + + +class _HostedGroup(_TrioGroup): + """Trio AsyncGroup living as a task on the host nursery.""" + + def __init__(self, termination_timeout: float) -> None: + super().__init__(termination_timeout) + self.shutdown = trio.Event() + self.finished = trio.Event() + + async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: + try: + async with self: + task_status.started(self) + await self.shutdown.wait() + finally: + self.finished.set() + + +class AsyncChannel: + """asyncio facade over a trio-native channel.""" + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, bridge: _HostBridge, channel: _TrioChannel) -> None: + self._bridge = bridge + self._channel = channel + + @property + def id(self) -> int: + return self._channel.id + + def __repr__(self) -> str: + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._channel.isclosed() + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side. + + Shielded: cancelling raises in the caller but the item is still + sent, rather than leaving a half-written frame on the wire. + """ + await self._bridge.call(self._channel.send, item, shield=True) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + EOFError once the peer closed or sent EOF, RemoteError for a peer + close-with-error, TimeoutError after ``timeout`` seconds. A + received channel reference arrives as an + :class:`~execnet.aio.AsyncChannel`. + + Cancellable: no item is consumed if the await is cancelled, so + ``asyncio.timeout`` is equivalent to passing ``timeout``. + """ + result = await self._bridge.call(self._channel.receive, timeout) + if isinstance(result, _TrioChannel): + return AsyncChannel(self._bridge, result) + return result + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._bridge.call(self._channel.send_eof, shield=True) + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._bridge.call(self._channel.aclose, error, shield=True) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._bridge.call(self._channel.wait_closed) + + def __aiter__(self) -> AsyncChannel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + +class AsyncGateway: + """asyncio facade over a trio-native gateway.""" + + def __init__(self, bridge: _HostBridge, gateway: _TrioGateway) -> None: + self._bridge = bridge + self._gateway = gateway + + @property + def id(self) -> str: + return self._gateway.id + + @property + def remoteaddress(self) -> str | None: + return self._gateway.remoteaddress + + def __repr__(self) -> str: + return f"" + + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> AsyncChannel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as ``Gateway.remote_exec``: a source + string, a pure function called with ``channel`` and ``**kwargs``, + or a module. + """ + channel = await self._bridge.call( + functools.partial(self._gateway.remote_exec, source, **kwargs) + ) + return AsyncChannel(self._bridge, channel) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + await self._bridge.call(self._gateway.terminate, shield=True) + + +class AsyncGroup: + """asyncio-native gateway group served on a Trio host thread. + + Usable as an async context manager, or driven explicitly with + :meth:`start` / :meth:`aclose` from application lifespan hooks. + Either way, shutting down terminates every gateway with the same + bounded contract as :class:`execnet.trio.AsyncGroup`. + """ + + def __init__( + self, + termination_timeout: float = 10.0, + *, + host: Host | None = None, + ) -> None: + self._termination_timeout = termination_timeout + self._host = default_host() if host is None else host + self._bridge: _HostBridge | None = None + self._group: _HostedGroup | None = None + + def __repr__(self) -> str: + state = "running" if self._group is not None else "idle" + return f"" + + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host + + async def start(self) -> None: + """Bring the host up and start the group task on it.""" + if self._group is not None: + raise RuntimeError(f"{self!r} is already started") + trio_host = await _start_host(self._host) + bridge = _HostBridge(trio_host) + + async def start_group() -> _HostedGroup: + # runs on the host loop + group = _HostedGroup(self._termination_timeout) + assert trio_host._nursery is not None + started: _HostedGroup = await trio_host._nursery.start(group.run) + return started + + self._bridge = bridge + self._group = await bridge.call(start_group, shield=True) + + async def aclose(self) -> None: + """Terminate every gateway and stop the group task (idempotent). + + The host thread is shared, so it keeps running for other groups. + """ + group, bridge = self._group, self._bridge + self._group = self._bridge = None + if group is None or bridge is None: + return + + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() + + with suppress(RuntimeError): + await bridge.call(stop_group, shield=True) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *exc_info: object) -> None: + await self.aclose() + + async def makegateway(self, spec: str | XSpec = "popen") -> AsyncGateway: + """Create a gateway for ``spec`` served on the group's host. + + All transports are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways. The worker profile + defaults to ``thread``; pass ``profile=trio`` for a worker that + runs exec'd async sources as tasks. + """ + group, bridge = self._group, self._bridge + if group is None or bridge is None: + raise RuntimeError(f"{self!r} is not started") + gateway = await bridge.call(group.makegateway, spec) + return AsyncGateway(bridge, gateway) + + +async def _start_host(host: Host) -> Any: + """Start ``host`` without blocking the asyncio loop or its executor. + + ``Host._ensure_started`` blocks until the trio loop is ready, so it + runs on a throwaway thread whose completion is posted back to the + loop -- never on the default executor, which belongs to the caller's + application. + """ + if host.running: + return host._ensure_started() + import threading + + loop = asyncio.get_running_loop() + future: asyncio.Future[Any] = loop.create_future() + + def start() -> None: + try: + trio_host = host._ensure_started() + except BaseException as exc: + loop.call_soon_threadsafe(_set_future_error, future, exc) + else: + loop.call_soon_threadsafe(_set_future_result, future, trio_host) + + threading.Thread(target=start, name="execnet-host-start", daemon=True).start() + return await future + + +def _set_future_result(future: asyncio.Future[Any], value: Any) -> None: + if not future.cancelled(): + future.set_result(value) + + +def _set_future_error(future: asyncio.Future[Any], error: BaseException) -> None: + if not future.cancelled(): + future.set_exception(error) + + +@asynccontextmanager +async def open_gateway(spec: str | XSpec = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one worker for ``spec`` and yield an asyncio gateway to it. + + Convenience for a single-gateway :class:`AsyncGroup`. + """ + async with AsyncGroup() as group: + yield await group.makegateway(spec) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 1d3eb59d..d15263ca 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -1,232 +1,19 @@ -"""Gateway code for initiating popen, socket and ssh connections. +"""Deprecated alias for :mod:`execnet._gateway`. -(c) 2004-2013, Holger Krekel and others +``Gateway`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import inspect -import linecache -import textwrap -import types -from collections.abc import Callable -from typing import TYPE_CHECKING -from typing import Any +from ._shim import forwarder -from . import gateway_base -from .gateway_base import IO -from .gateway_base import Channel -from .gateway_base import Message -from .multi import Group -from .xspec import XSpec +_MOVED = { + "Gateway": "._gateway", + "RInfo": "._gateway", + "RemoteStatus": "._gateway", + "normalize_exec_source": "._exec_source", + "_find_non_builtin_globals": "._exec_source", + "_source_of_function": "._exec_source", +} - -class Gateway(gateway_base.BaseGateway): - """Gateway to a local or remote Python Interpreter.""" - - _group: Group - - def __init__(self, io: IO, spec: XSpec) -> None: - """:private:""" - super().__init__(io=io, id=spec.id, _startcount=1) - self.spec = spec - self._initreceive() - - @property - def remoteaddress(self) -> str: - # Only defined for remote IO types. - return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] - - def __repr__(self) -> str: - """A string representing gateway type and status.""" - try: - r: str = (self.hasreceiver() and "receive-live") or "not-receiving" - i = str(len(self._channelfactory.channels())) - except AttributeError: - r = "uninitialized" - i = "no" - return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" - - def exit(self) -> None: - """Trigger gateway exit. - - Defer waiting for finishing of receiver-thread and subprocess activity - to when group.terminate() is called. - """ - self._trace("gateway.exit() called") - if self not in self._group: - self._trace("gateway already unregistered with group") - return - self._group._unregister(self) - try: - self._trace("--> sending GATEWAY_TERMINATE") - self._send(Message.GATEWAY_TERMINATE) - self._trace("--> io.close_write") - self._io.close_write() - except (ValueError, EOFError, OSError) as exc: - self._trace("io-error: could not send termination sequence") - self._trace(" exception: %r" % exc) - - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this gateway. - - The default is to try to convert py2 str as py3 str, but not to try and - convert py3 str to py2 str. - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = gateway_base.dumps_internal(self._strconfig) - self._send(Message.RECONFIGURE, data=data) - - def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote.""" - if update or not hasattr(self, "_cache_rinfo"): - ch = self.remote_exec(rinfo_source) - try: - self._cache_rinfo = RInfo(ch.receive()) - finally: - ch.waitclose() - return self._cache_rinfo - - def hasreceiver(self) -> bool: - """Whether gateway is able to receive data.""" - return self._receivepool.active_count() > 0 - - def remote_status(self) -> RemoteStatus: - """Obtain information about the remote execution status.""" - channel = self.newchannel() - self._send(Message.STATUS, channel.id) - statusdict = channel.receive() - # the other side didn't actually instantiate a channel - # so we just delete the internal id/channel mapping - self._channelfactory._local_close(channel.id) - return RemoteStatus(statusdict) - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs: object, - ) -> Channel: - """Return channel object and connect it to a remote - execution thread where the given ``source`` executes. - - * ``source`` is a string: execute source string remotely - with a ``channel`` put into the global namespace. - * ``source`` is a pure function: serialize source and - call function with ``**kwargs``, adding a - ``channel`` object to the keyword arguments. - * ``source`` is a pure module: execute source of module - with a ``channel`` in its global namespace. - - In all cases the binding ``__name__='__channelexec__'`` - will be available in the global namespace of the remotely - executing code. - """ - call_name = None - file_name = None - if isinstance(source, types.ModuleType): - file_name = inspect.getsourcefile(source) - linecache.updatecache(file_name) # type: ignore[arg-type] - source = inspect.getsource(source) - elif isinstance(source, types.FunctionType): - call_name = source.__name__ - file_name = inspect.getsourcefile(source) - source = _source_of_function(source) - else: - source = textwrap.dedent(str(source)) - - if not call_name and kwargs: - raise TypeError("can't pass kwargs to non-function remote_exec") - - channel = self.newchannel() - self._send( - Message.CHANNEL_EXEC, - channel.id, - gateway_base.dumps_internal((source, file_name, call_name, kwargs)), - ) - return channel - - def remote_init_threads(self, num: int | None = None) -> None: - """DEPRECATED. Is currently a NO-OPERATION already.""" - print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") - - -class RInfo: - def __init__(self, kwargs) -> None: - self.__dict__.update(kwargs) - - def __repr__(self) -> str: - info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) - return "" % info - - if TYPE_CHECKING: - - def __getattr__(self, name: str) -> Any: ... - - -RemoteStatus = RInfo - - -def rinfo_source(channel) -> None: - import os - import sys - - channel.send( - dict( - executable=sys.executable, - version_info=sys.version_info[:5], - platform=sys.platform, - cwd=os.getcwd(), - pid=os.getpid(), - ) - ) - - -def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: - import ast - import builtins - - vars = dict.fromkeys(codeobj.co_varnames) - return [ - node.id - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.Name) - and node.id not in vars - and node.id not in builtins.__dict__ - ] - - -def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: - if function.__name__ == "": - raise ValueError("can't evaluate lambda functions'") - # XXX: we dont check before remote instantiation - # if arguments are used properly - try: - sig = inspect.getfullargspec(function) - except AttributeError: - args = inspect.getargspec(function)[0] - else: - args = sig.args - if not args or args[0] != "channel": - raise ValueError("expected first function argument to be `channel`") - - closure = function.__closure__ - codeobj = function.__code__ - - if closure is not None: - raise ValueError("functions with closures can't be passed") - - try: - source = inspect.getsource(function) - except OSError as e: - raise ValueError("can't find source file for %s" % function) from e - - source = textwrap.dedent(source) # just for inner functions - - used_globals = _find_non_builtin_globals(source, codeobj) - if used_globals: - raise ValueError("the use of non-builtin globals isn't supported", used_globals) - - leading_ws = "\n" * (codeobj.co_firstlineno - 1) - return leading_ws + source +__getattr__ = forwarder("gateway", _MOVED) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 73dc7175..40eea46c 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,1799 +1,77 @@ -"""Base execnet gateway code send to the other side for bootstrapping. +"""Deprecated: the pre-Trio core module, now split by concern. -:copyright: 2004-2015 -:authors: - - Holger Krekel - - Armin Rigo - - Benjamin Peterson - - Ronny Pfannschmidt - - many others +``execnet.gateway_base`` was never a supported public API. Its contents live +in private modules -- ``_trace``, ``_errors``, ``_execmodel``, ``_message``, +``_serialize``, ``_channel`` and ``_gateway_base`` -- and this shim forwards +to them with a :class:`DeprecationWarning`. + +Use :mod:`execnet` / :mod:`execnet.sync`, :mod:`execnet.trio`, +:mod:`execnet.aio` or :mod:`execnet.gevent` instead. The standalone +serializer stays internal; :func:`execnet.can_send` answers "can this value +cross a channel?" without it. """ from __future__ import annotations -import abc -import os -import struct -import sys -import traceback -import weakref -from _thread import interrupt_main -from collections.abc import Callable -from collections.abc import Iterator -from contextlib import suppress -from io import BytesIO -from typing import Any -from typing import Literal -from typing import Protocol -from typing import cast -from typing import overload - - -class WriteIO(Protocol): - def write(self, data: bytes, /) -> None: ... - - -class ReadIO(Protocol): - def read(self, numbytes: int, /) -> bytes: ... - - -class IO(Protocol): - execmodel: ExecModel - - def read(self, numbytes: int, /) -> bytes: ... - - def write(self, data: bytes, /) -> None: ... - - def close_read(self) -> None: ... - - def close_write(self) -> None: ... - - def wait(self) -> int | None: ... - - def kill(self) -> None: ... - - -class Event(Protocol): - """Protocol for types which look like threading.Event.""" - - def is_set(self) -> bool: ... - - def set(self) -> None: ... - - def clear(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... - - -class ExecModel(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def backend(self) -> str: - raise NotImplementedError() - - def __repr__(self) -> str: - return "" % self.backend - - @property - @abc.abstractmethod - def queue(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def subprocess(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def socket(self): - raise NotImplementedError() - - @abc.abstractmethod - def start(self, func, args=()) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def get_ident(self) -> int: - raise NotImplementedError() - - @abc.abstractmethod - def sleep(self, delay: float) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def fdopen(self, fd, mode, bufsize=1, closefd=True): - raise NotImplementedError() - - @abc.abstractmethod - def Lock(self): - raise NotImplementedError() - - @abc.abstractmethod - def RLock(self): - raise NotImplementedError() - - @abc.abstractmethod - def Event(self) -> Event: - raise NotImplementedError() - - -class ThreadExecModel(ExecModel): - backend = "thread" - - @property - def queue(self): - import queue - - return queue - - @property - def subprocess(self): - import subprocess - - return subprocess - - @property - def socket(self): - import socket - - return socket - - def get_ident(self) -> int: - import _thread - - return _thread.get_ident() - - def sleep(self, delay: float) -> None: - import time - - time.sleep(delay) - - def start(self, func, args=()) -> None: - import _thread - - _thread.start_new_thread(func, args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import os - - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) - - def Lock(self): - import threading - - return threading.RLock() - - def RLock(self): - import threading - - return threading.RLock() - - def Event(self): - import threading - - return threading.Event() - - -class MainThreadOnlyExecModel(ThreadExecModel): - backend = "main_thread_only" - - -class EventletExecModel(ExecModel): - backend = "eventlet" - - @property - def queue(self): - import eventlet - - return eventlet.queue - - @property - def subprocess(self): - import eventlet.green.subprocess - - return eventlet.green.subprocess - - @property - def socket(self): - import eventlet.green.socket - - return eventlet.green.socket - - def get_ident(self) -> int: - import eventlet.green.thread - - return eventlet.green.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import eventlet - - eventlet.sleep(delay) - - def start(self, func, args=()) -> None: - import eventlet - - eventlet.spawn_n(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import eventlet.green.os - - return eventlet.green.os.fdopen(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def RLock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def Event(self): - import eventlet.green.threading - - return eventlet.green.threading.Event() - - -class GeventExecModel(ExecModel): - backend = "gevent" - - @property - def queue(self): - import gevent.queue - - return gevent.queue - - @property - def subprocess(self): - import gevent.subprocess - - return gevent.subprocess - - @property - def socket(self): - import gevent - - return gevent.socket - - def get_ident(self) -> int: - import gevent.thread - - return gevent.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import gevent - - gevent.sleep(delay) - - def start(self, func, args=()) -> None: - import gevent - - gevent.spawn(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import gevent.fileobject - - # Prefer FileObject (FileObjectPosix on Unix). FileObjectThread keeps a - # native threadpool alive and can prevent interpreter shutdown, which - # hangs tests/scripts that open stdio via init_popen_io and then exit. - return gevent.fileobject.FileObject(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import gevent.lock - - return gevent.lock.RLock() - - def RLock(self): - import gevent.lock - - return gevent.lock.RLock() - - def Event(self): - import gevent.event - - return gevent.event.Event() - - -def get_execmodel(backend: str | ExecModel) -> ExecModel: - if isinstance(backend, ExecModel): - return backend - if backend == "thread": - return ThreadExecModel() - elif backend == "main_thread_only": - return MainThreadOnlyExecModel() - elif backend == "eventlet": - return EventletExecModel() - elif backend == "gevent": - return GeventExecModel() - else: - raise ValueError(f"unknown execmodel {backend!r}") - - -class Reply: - """Provide access to the result of a function execution that got dispatched - through WorkerPool.spawn().""" - - def __init__(self, task, threadmodel: ExecModel) -> None: - self.task = task - self._result_ready = threadmodel.Event() - self.running = True - - def get(self, timeout: float | None = None): - """get the result object from an asynchronous function execution. - if the function execution raised an exception, - then calling get() will reraise that exception - including its traceback. - """ - self.waitfinish(timeout) - try: - return self._result - except AttributeError: - raise self._exc from None - - def waitfinish(self, timeout: float | None = None) -> None: - if not self._result_ready.wait(timeout): - raise OSError(f"timeout waiting for {self.task!r}") - - def run(self) -> None: - func, args, kwargs = self.task - try: - try: - self._result = func(*args, **kwargs) - except BaseException as exc: - self._exc = exc - finally: - self._result_ready.set() - self.running = False - - -class WorkerPool: - """A WorkerPool allows to spawn function executions - to threads, returning a reply object on which you - can ask for the result (and get exceptions reraised). - - This implementation allows the main thread to integrate - itself into performing function execution through - calling integrate_as_primary_thread() which will return - when the pool received a trigger_shutdown(). - - By default allows unlimited number of spawns. - """ - - _primary_thread_task: Reply | None - - def __init__(self, execmodel: ExecModel, hasprimary: bool = False) -> None: - self.execmodel = execmodel - self._running_lock = self.execmodel.Lock() - self._running: set[Reply] = set() - self._shuttingdown = False - self._waitall_events: list[Event] = [] - if hasprimary: - if self.execmodel.backend not in ("thread", "main_thread_only"): - raise ValueError("hasprimary=True requires thread model") - self._primary_thread_task_ready: Event | None = self.execmodel.Event() - else: - self._primary_thread_task_ready = None - - def integrate_as_primary_thread(self) -> None: - """Integrate the thread with which we are called as a primary - thread for executing functions triggered with spawn().""" - assert self.execmodel.backend in ("thread", "main_thread_only"), self.execmodel - primary_thread_task_ready = self._primary_thread_task_ready - assert primary_thread_task_ready is not None - # interacts with code at REF1 - while 1: - primary_thread_task_ready.wait() - reply = self._primary_thread_task - if reply is None: # trigger_shutdown() woke us up - break - self._perform_spawn(reply) - # we are concurrent with trigger_shutdown and spawn - with self._running_lock: - if self._shuttingdown: - break - # Only clear if _try_send_to_primary_thread has not - # yet set the next self._primary_thread_task reply - # after waiting for this one to complete. - if reply is self._primary_thread_task: - primary_thread_task_ready.clear() - - def trigger_shutdown(self) -> None: - with self._running_lock: - self._shuttingdown = True - if self._primary_thread_task_ready is not None: - self._primary_thread_task = None - self._primary_thread_task_ready.set() - - def active_count(self) -> int: - return len(self._running) - - def _perform_spawn(self, reply: Reply) -> None: - reply.run() - with self._running_lock: - self._running.remove(reply) - if not self._running: - while self._waitall_events: - waitall_event = self._waitall_events.pop() - waitall_event.set() - - def _try_send_to_primary_thread(self, reply: Reply) -> bool: - # REF1 in 'thread' model we give priority to running in main thread - # note that we should be called with _running_lock hold - primary_thread_task_ready = self._primary_thread_task_ready - if primary_thread_task_ready is not None: - if not primary_thread_task_ready.is_set(): - self._primary_thread_task = reply - # wake up primary thread - primary_thread_task_ready.set() - return True - elif ( - self.execmodel.backend == "main_thread_only" - and self._primary_thread_task is not None - ): - self._primary_thread_task.waitfinish() - self._primary_thread_task = reply - # wake up primary thread (it's okay if this is already set - # because we waited for the previous task to finish above - # and integrate_as_primary_thread will not clear it when - # it enters self._running_lock if it detects that a new - # task is available) - primary_thread_task_ready.set() - return True - return False - - def spawn(self, func, *args, **kwargs) -> Reply: - """Asynchronously dispatch func(*args, **kwargs) and return a Reply.""" - reply = Reply((func, args, kwargs), self.execmodel) - with self._running_lock: - if self._shuttingdown: - raise ValueError("pool is shutting down") - self._running.add(reply) - if not self._try_send_to_primary_thread(reply): - self.execmodel.start(self._perform_spawn, (reply,)) - return reply - - def terminate(self, timeout: float | None = None) -> bool: - """Trigger shutdown and wait for completion of all executions.""" - self.trigger_shutdown() - return self.waitall(timeout=timeout) - - def waitall(self, timeout: float | None = None) -> bool: - """Wait until all active spawns have finished executing.""" - with self._running_lock: - if not self._running: - return True - # if a Reply still runs, we let run_and_release - # signal us -- note that we are still holding the - # _running_lock to avoid race conditions - my_waitall_event = self.execmodel.Event() - self._waitall_events.append(my_waitall_event) - return my_waitall_event.wait(timeout=timeout) - - -sysex = (KeyboardInterrupt, SystemExit) - - -DEBUG = os.environ.get("EXECNET_DEBUG") -pid = os.getpid() -if DEBUG == "2": - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - sys.stderr.write(f"[{pid}] {line}\n") - sys.stderr.flush() - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -elif DEBUG: - import os - import tempfile - - fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) - # sys.stderr.write("execnet-debug at %r" % (fn,)) - debugfile = open(fn, "w") - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - debugfile.write(line + "\n") - debugfile.flush() - except Exception as exc: - try: - sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -else: - notrace = trace = lambda *msg: None - - -class Popen2IO: - error = (IOError, OSError, EOFError) - - def __init__(self, outfile, infile, execmodel: ExecModel) -> None: - # we need raw byte streams - self.outfile, self.infile = outfile, infile - if sys.platform == "win32": - import msvcrt - - try: - msvcrt.setmode(infile.fileno(), os.O_BINARY) - msvcrt.setmode(outfile.fileno(), os.O_BINARY) - except (AttributeError, OSError): - pass - self._read = getattr(infile, "buffer", infile).read - self._write = getattr(outfile, "buffer", outfile).write - self.execmodel = execmodel - - def read(self, numbytes: int) -> bytes: - """Read exactly 'numbytes' bytes from the pipe.""" - # a file in non-blocking mode may return less bytes, so we loop - buf = b"" - while numbytes > len(buf): - data = self._read(numbytes - len(buf)) - if not data: - raise EOFError("expected %d bytes, got %d" % (numbytes, len(buf))) - buf += data - return buf - - def write(self, data: bytes) -> None: - """Write out all data bytes.""" - assert isinstance(data, bytes) - self._write(data) - self.outfile.flush() - - def close_read(self) -> None: - self.infile.close() - - def close_write(self) -> None: - self.outfile.close() - - -class Message: - """Encapsulates Messages and their wire protocol.""" - - # message code -> name, handler - _types: dict[int, tuple[str, Callable[[Message, BaseGateway], None]]] = {} - - def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - self.msgcode = msgcode - self.channelid = channelid - self.data = data - - @staticmethod - def from_io(io: ReadIO) -> Message: - try: - header = io.read(9) # type 1, channel 4, payload 4 - if not header: - raise EOFError("empty read") - except EOFError as e: - raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = struct.unpack("!bii", header) - return Message(msgtype, channel, io.read(payload)) - - def to_io(self, io: WriteIO) -> None: - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - io.write(header + self.data) - - def received(self, gateway: BaseGateway) -> None: - handler = self._types[self.msgcode][1] - handler(self, gateway) - - def __repr__(self) -> str: - name = self._types[self.msgcode][0] - return f"" - - def _status(message: Message, gateway: BaseGateway) -> None: - # we use the channelid to send back information - # but don't instantiate a channel object - d = { - "numchannels": len(gateway._channelfactory._channels), - # TODO(typing): Attribute `_execpool` is only on WorkerGateway. - "numexecuting": gateway._execpool.active_count(), # type: ignore[attr-defined] - "execmodel": gateway.execmodel.backend, - } - gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) - gateway._send(Message.CHANNEL_CLOSE, message.channelid) - - STATUS = 0 - _types[STATUS] = ("STATUS", _status) - - def _reconfigure(message: Message, gateway: BaseGateway) -> None: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - strconfig: tuple[bool, bool] = data - if message.channelid == 0: - gateway._strconfig = strconfig - else: - gateway._channelfactory.new(message.channelid)._strconfig = strconfig - - RECONFIGURE = 1 - _types[RECONFIGURE] = ("RECONFIGURE", _reconfigure) - - def _gateway_terminate(message: Message, gateway: BaseGateway) -> None: - raise GatewayReceivedTerminate(gateway) - - GATEWAY_TERMINATE = 2 - _types[GATEWAY_TERMINATE] = ("GATEWAY_TERMINATE", _gateway_terminate) - - def _channel_exec(message: Message, gateway: BaseGateway) -> None: - channel = gateway._channelfactory.new(message.channelid) - gateway._local_schedulexec(channel=channel, sourcetask=message.data) - - CHANNEL_EXEC = 3 - _types[CHANNEL_EXEC] = ("CHANNEL_EXEC", _channel_exec) - - def _channel_data(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_receive(message.channelid, message.data) - - CHANNEL_DATA = 4 - _types[CHANNEL_DATA] = ("CHANNEL_DATA", _channel_data) - - def _channel_close(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid) - - CHANNEL_CLOSE = 5 - _types[CHANNEL_CLOSE] = ("CHANNEL_CLOSE", _channel_close) - - def _channel_close_error(message: Message, gateway: BaseGateway) -> None: - error_message = loads_internal(message.data) - assert isinstance(error_message, str) - remote_error = RemoteError(error_message) - gateway._channelfactory._local_close(message.channelid, remote_error) - - CHANNEL_CLOSE_ERROR = 6 - _types[CHANNEL_CLOSE_ERROR] = ("CHANNEL_CLOSE_ERROR", _channel_close_error) - - def _channel_last_message(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid, sendonly=True) - - CHANNEL_LAST_MESSAGE = 7 - _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) - - -class GatewayReceivedTerminate(Exception): - """Receiverthread got termination message.""" - - -def geterrortext( - exc: BaseException, - format_exception=traceback.format_exception, - sysex: tuple[type[BaseException], ...] = sysex, -) -> str: - try: - # In py310, can change this to: - # l = format_exception(exc) - l = format_exception(type(exc), exc, exc.__traceback__) - errortext = "".join(l) - except sysex: - raise - except BaseException: - errortext = f"{type(exc).__name__}: {exc}" - return errortext - - -class RemoteError(Exception): - """Exception containing a stringified error from the other side.""" - - def __init__(self, formatted: str) -> None: - super().__init__() - self.formatted = formatted - - def __str__(self) -> str: - return self.formatted - - def __repr__(self) -> str: - return f"{self.__class__.__name__}: {self.formatted}" - - def warn(self) -> None: - if self.formatted != INTERRUPT_TEXT: - # XXX do this better - sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") - - -class TimeoutError(IOError): - """Exception indicating that a timeout was reached.""" - - -NO_ENDMARKER_WANTED = object() - - -class Channel: - """Communication channel between two Python Interpreter execution points.""" - - RemoteError = RemoteError - TimeoutError = TimeoutError - _INTERNALWAKEUP = 1000 - _executing = False - - def __init__(self, gateway: BaseGateway, id: int) -> None: - """:private:""" - assert isinstance(id, int) - assert not isinstance(gateway, type) - self.gateway = gateway - # XXX: defaults copied from Unserializer - self._strconfig = getattr(gateway, "_strconfig", (True, False)) - self.id = id - self._items = self.gateway.execmodel.queue.Queue() - self._closed = False - self._receiveclosed = self.gateway.execmodel.Event() - self._remoteerrors: list[RemoteError] = [] - - def _trace(self, *msg: object) -> None: - self.gateway._trace(self.id, *msg) - - def setcallback( - self, - callback: Callable[[Any], Any], - endmarker: object = NO_ENDMARKER_WANTED, - ) -> None: - """Set a callback function for receiving items. - - All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver thread - for each received data item and calls to ``receive()`` will - raise an error. - If an endmarker is specified the callback will eventually - be called with the endmarker when the channel closes. - """ - _callbacks = self.gateway._channelfactory._callbacks - with self.gateway._receivelock: - if self._items is None: - raise OSError(f"{self!r} has callback already registered") - items = self._items - self._items = None - while 1: - try: - olditem = items.get(block=False) - except self.gateway.execmodel.queue.Empty: - if not (self._closed or self._receiveclosed.is_set()): - _callbacks[self.id] = (callback, endmarker, self._strconfig) - break - else: - if olditem is ENDMARKER: - items.put(olditem) # for other receivers - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - else: - callback(olditem) - - def __repr__(self) -> str: - flag = (self.isclosed() and "closed") or "open" - return "" % (self.id, flag) - - def __del__(self) -> None: - if self.gateway is None: # can be None in tests - return # type: ignore[unreachable] - - self._trace("channel.__del__") - # no multithreading issues here, because we have the last ref to 'self' - if self._closed: - # state transition "closed" --> "deleted" - for error in self._remoteerrors: - error.warn() - elif self._receiveclosed.is_set(): - # state transition "sendonly" --> "deleted" - # the remote channel is already in "deleted" state, nothing to do - pass - else: - # state transition "opened" --> "deleted" - # check if we are in the middle of interpreter shutdown - # in which case the process will go away and we probably - # don't need to try to send a closing or last message - # (and often it won't work anymore to send things out) - if Message is not None: - if self._items is None: # has_callback - msgcode = Message.CHANNEL_LAST_MESSAGE - else: - msgcode = Message.CHANNEL_CLOSE - with suppress(OSError, ValueError): # ignore problems with sending - self.gateway._send(msgcode, self.id) - - def _getremoteerror(self): - try: - return self._remoteerrors.pop(0) - except IndexError: - try: - return self.gateway._error - except AttributeError: - pass - return None - - # - # public API for channel objects - # - def isclosed(self) -> bool: - """Return True if the channel is closed. - - A closed channel may still hold items. - """ - return self._closed - - @overload - def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: - pass - - @overload - def makefile( - self, - mode: Literal["w"] = ..., - proxyclose: bool = ..., - ) -> ChannelFileWrite: - pass - - def makefile( - self, - mode: Literal["r", "w"] = "w", - proxyclose: bool = False, - ) -> ChannelFileWrite | ChannelFileRead: - """Return a file-like object. - - mode can be 'w' or 'r' for writeable/readable files. - If proxyclose is true, file.close() will also close the channel. - """ - if mode == "w": - return ChannelFileWrite(channel=self, proxyclose=proxyclose) - elif mode == "r": - return ChannelFileRead(channel=self, proxyclose=proxyclose) - raise ValueError(f"mode {mode!r} not available") - - def close(self, error=None) -> None: - """Close down this channel with an optional error message. - - Note that closing of a channel tied to remote_exec happens - automatically at the end of execution and cannot - be done explicitly. - """ - if self._executing: - raise OSError("cannot explicitly close channel within remote_exec") - if self._closed: - self.gateway._trace(self, "ignoring redundant call to close()") - if not self._closed: - # state transition "opened/sendonly" --> "closed" - # threads warning: the channel might be closed under our feet, - # but it's never damaging to send too many CHANNEL_CLOSE messages - # however, if the other side triggered a close already, we - # do not send back a closed message. - if not self._receiveclosed.is_set(): - put = self.gateway._send - if error is not None: - put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) - else: - put(Message.CHANNEL_CLOSE, self.id) - self._trace("sent channel close message") - if isinstance(error, RemoteError): - self._remoteerrors.append(error) - self._closed = True # --> "closed" - self._receiveclosed.set() - queue = self._items - if queue is not None: - queue.put(ENDMARKER) - self.gateway._channelfactory._no_longer_opened(self.id) - - def waitclose(self, timeout: float | None = None) -> None: - """Wait until this channel is closed (or the remote side - otherwise signalled that no more data was being sent). - - The channel may still hold receiveable items, but not receive - any more after waitclose() has returned. - - Exceptions from executing code on the other side are reraised as local - channel.RemoteErrors. - - EOFError is raised if the reading-connection was prematurely closed, - which often indicates a dying process. - - self.TimeoutError is raised after the specified number of seconds - (default is None, i.e. wait indefinitely). - """ - # wait for non-"opened" state - self._receiveclosed.wait(timeout=timeout) - if not self._receiveclosed.is_set(): - raise self.TimeoutError("Timeout after %r seconds" % timeout) - error = self._getremoteerror() - if error: - raise error - - def send(self, item: object) -> None: - """Sends the given item to the other side of the channel, - possibly blocking if the sender queue is full. - - The item must be a simple Python type and will be - copied to the other side by value. - - OSError is raised if the write pipe was prematurely closed. - """ - if self.isclosed(): - raise OSError(f"cannot send to {self!r}") - self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) - - def receive(self, timeout: float | None = None) -> Any: - """Receive a data item that was sent from the other side. - - timeout: None [default] blocked waiting. A positive number - indicates the number of seconds after which a channel.TimeoutError - exception will be raised if no item was received. - - Note that exceptions from the remotely executing code will be - reraised as channel.RemoteError exceptions containing - a textual representation of the remote traceback. - """ - itemqueue = self._items - if itemqueue is None: - raise OSError("cannot receive(), channel has receiver callback") - try: - x = itemqueue.get(timeout=timeout) - except self.gateway.execmodel.queue.Empty: - raise self.TimeoutError("no item after %r seconds" % timeout) from None - if x is ENDMARKER: - itemqueue.put(x) # for other receivers - raise self._getremoteerror() or EOFError() - else: - return x - - def __iter__(self) -> Iterator[Any]: - return self - - def next(self) -> Any: - try: - return self.receive() - except EOFError: - raise StopIteration from None - - __next__ = next - - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this channel. - - The default is to try to convert py2 str as py3 str, - but not to try and convert py3 str to py2 str - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = dumps_internal(self._strconfig) - self.gateway._send(Message.RECONFIGURE, self.id, data=data) - - -ENDMARKER = object() -INTERRUPT_TEXT = "keyboard-interrupted" -MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( - "concurrent remote_exec would cause deadlock for main_thread_only execmodel" -) - - -class ChannelFactory: - def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: - self._channels: weakref.WeakValueDictionary[int, Channel] = ( - weakref.WeakValueDictionary() - ) - # Channel ID => (callback, end marker, strconfig) - self._callbacks: dict[ - int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] - ] = {} - self._writelock = gateway.execmodel.Lock() - self.gateway = gateway - self.count = startcount - self.finished = False - self._list = list # needed during interp-shutdown - - def new(self, id: int | None = None) -> Channel: - """Create a new Channel with 'id' (or create new id if None).""" - with self._writelock: - if self.finished: - raise OSError(f"connection already closed: {self.gateway}") - if id is None: - id = self.count - self.count += 2 - try: - channel = self._channels[id] - except KeyError: - channel = self._channels[id] = Channel(self.gateway, id) - return channel - - def channels(self) -> list[Channel]: - return self._list(self._channels.values()) - - # - # internal methods, called from the receiver thread - # - def _no_longer_opened(self, id: int) -> None: - self._channels.pop(id, None) - item = self._callbacks.pop(id, None) - if item is not None: - callback, endmarker, _strconfig = item - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - - def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - channel = self._channels.get(id) - if channel is None: - # channel already in "deleted" state - if remoteerror: - remoteerror.warn() - self._no_longer_opened(id) - else: - # state transition to "closed" state - if remoteerror: - channel._remoteerrors.append(remoteerror) - queue = channel._items - if queue is not None: - queue.put(ENDMARKER) - self._no_longer_opened(id) - if not sendonly: # otherwise #--> "sendonly" - channel._closed = True # --> "closed" - channel._receiveclosed.set() - - def _local_receive(self, id: int, data) -> None: - # executes in receiver thread - channel = self._channels.get(id) - try: - callback, _endmarker, strconfig = self._callbacks[id] - except KeyError: - queue = channel._items if channel is not None else None - if queue is None: - pass # drop data - else: - item = loads_internal(data, channel) - queue.put(item) - else: - try: - data = loads_internal(data, channel, strconfig) - callback(data) # even if channel may be already closed - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, id, dumps_internal(errortext) - ) - self._local_close(id, errortext) - - def _finished_receiving(self) -> None: - with self._writelock: - self.finished = True - for id in self._list(self._channels): - self._local_close(id, sendonly=True) - for id in self._list(self._callbacks): - self._no_longer_opened(id) - - -class ChannelFile: - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - self.channel = channel - self._proxyclose = proxyclose - - def isatty(self) -> bool: - return False - - def close(self) -> None: - if self._proxyclose: - self.channel.close() - - def __repr__(self) -> str: - state = (self.channel.isclosed() and "closed") or "open" - return "" % (self.channel.id, state) - - -class ChannelFileWrite(ChannelFile): - def write(self, out: bytes) -> None: - self.channel.send(out) - - def flush(self) -> None: - pass - - -class ChannelFileRead(ChannelFile): - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - super().__init__(channel, proxyclose) - self._buffer: str | None = None - - def read(self, n: int) -> str: - try: - if self._buffer is None: - self._buffer = cast(str, self.channel.receive()) - while len(self._buffer) < n: - self._buffer += cast(str, self.channel.receive()) - except EOFError: - self.close() - if self._buffer is None: - ret = "" - else: - ret = self._buffer[:n] - self._buffer = self._buffer[n:] - return ret - - def readline(self) -> str: - if self._buffer is not None: - i = self._buffer.find("\n") - if i != -1: - return self.read(i + 1) - line = self.read(len(self._buffer) + 1) - else: - line = self.read(1) - while line and line[-1] != "\n": - c = self.read(1) - if not c: - break - line += c - return line - - -class BaseGateway: - _sysex = sysex - id = "" - - def __init__(self, io: IO, id, _startcount: int = 2) -> None: - self.execmodel = io.execmodel - self._io = io - self.id = id - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) - self._channelfactory = ChannelFactory(self, _startcount) - self._receivelock = self.execmodel.RLock() - # globals may be NONE at process-termination - self.__trace = trace - self._geterrortext = geterrortext - self._receivepool = WorkerPool(self.execmodel) - - def _trace(self, *msg: object) -> None: - self.__trace(self.id, *msg) - - def _initreceive(self) -> None: - self._receivepool.spawn(self._thread_receiver) - - def _thread_receiver(self) -> None: - def log(*msg: object) -> None: - self._trace("[receiver-thread]", *msg) - - log("RECEIVERTHREAD: starting to run") - io = self._io - try: - while 1: - msg = Message.from_io(io) - log("received", msg) - with self._receivelock: - msg.received(self) - del msg - except (KeyboardInterrupt, GatewayReceivedTerminate): - pass - except EOFError as exc: - log("EOF without prior gateway termination message") - self._error = exc - except Exception as exc: - log(self._geterrortext(exc)) - log("finishing receiving thread") - # wake up and terminate any execution waiting to receive - self._channelfactory._finished_receiving() - log("terminating execution") - self._terminate_execution() - log("closing read") - self._io.close_read() - log("closing write") - self._io.close_write() - log("terminating our receive pseudo pool") - self._receivepool.trigger_shutdown() - - def _terminate_execution(self) -> None: - pass - - def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - message = Message(msgcode, channelid, data) - try: - message.to_io(self._io) - self._trace("sent", message) - except (OSError, ValueError) as e: - self._trace("failed to send", message, e) - # ValueError might be because the IO is already closed - raise OSError("cannot send (already closed?)") from e - - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - channel.close("execution disallowed") - - # _____________________________________________________________________ - # - # High Level Interface - # _____________________________________________________________________ - # - def newchannel(self) -> Channel: - """Return a new independent channel.""" - return self._channelfactory.new() - - def join(self, timeout: float | None = None) -> None: - """Wait for receiverthread to terminate.""" - self._trace("waiting for receiver thread to finish") - self._receivepool.waitall(timeout) - - -class WorkerGateway(BaseGateway): - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - if self._execpool.execmodel.backend == "main_thread_only": - assert self._executetask_complete is not None - # It's necessary to wait for a short time in order to ensure - # that we do not report a false-positive deadlock error, since - # channel close does not elicit a response that would provide - # a guarantee to remote_exec callers that the previous task - # has released the main thread. If the timeout expires then it - # should be practically impossible to report a false-positive. - if not self._executetask_complete.wait(timeout=1): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return - # It's only safe to clear here because the above wait proves - # that there is not a previous task about to set it again. - self._executetask_complete.clear() - - sourcetask_ = loads_internal(sourcetask) - self._execpool.spawn(self.executetask, (channel, sourcetask_)) - - def _terminate_execution(self) -> None: - # called from receiverthread - self._trace("shutting down execution pool") - self._execpool.trigger_shutdown() - if not self._execpool.waitall(5.0): - self._trace("execution ongoing after 5 secs, trying interrupt_main") - # We try hard to terminate execution based on the assumption - # that there is only one gateway object running per-process. - if sys.platform != "win32": - self._trace("sending ourselves a SIGINT") - os.kill(os.getpid(), 2) # send ourselves a SIGINT - elif interrupt_main is not None: - self._trace("calling interrupt_main()") - interrupt_main() - if not self._execpool.waitall(10.0): - self._trace( - "execution did not finish in another 10 secs, calling os._exit()" - ) - os._exit(1) - - def serve(self) -> None: - def trace(msg: str) -> None: - self._trace("[serve] " + msg) - - hasprimary = self.execmodel.backend in ("thread", "main_thread_only") - self._execpool = WorkerPool(self.execmodel, hasprimary=hasprimary) - self._executetask_complete = None - if self.execmodel.backend == "main_thread_only": - self._executetask_complete = self.execmodel.Event() - # Initialize state to indicate that there is no previous task - # executing so that we don't need a separate flag to track this. - self._executetask_complete.set() - trace("spawning receiver thread") - self._initreceive() - try: - if hasprimary: - # this will return when we are in shutdown - trace("integrating as primary thread") - self._execpool.integrate_as_primary_thread() - trace("joining receiver thread") - self.join() - except KeyboardInterrupt: - # in the worker we can't really do anything sensible - trace("swallowing keyboardinterrupt, serve finished") - - def executetask( - self, - item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], - ) -> None: - try: - channel, (source, file_name, call_name, kwargs) = item - loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} - self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") - channel._executing = True - try: - co = compile(source + "\n", file_name or "", "exec") - exec(co, loc) - if call_name: - self._trace("calling %s(**%60r)" % (call_name, kwargs)) - function = loc[call_name] - function(channel, **kwargs) - finally: - channel._executing = False - self._trace("execution finished") - except KeyboardInterrupt: - channel.close(INTERRUPT_TEXT) - raise - except EOFError: - self._trace("ignoring EOFError because receiving finished") - - except BaseException as exc: - if not channel.gateway._channelfactory.finished: - self._trace(f"got exception: {exc!r}") - errortext = self._geterrortext(exc) - channel.close(errortext) - return - channel.close() - if self._executetask_complete is not None: - # Indicate that this task has finished executing, meaning - # that there is no possibility of it triggering a deadlock - # for the next spawn call. - self._executetask_complete.set() - - -# -# Cross-Python pickling code, tested from test_serializer.py -# - - -class DataFormatError(Exception): - pass - - -class DumpError(DataFormatError): - """Error while serializing an object.""" - - -class LoadError(DataFormatError): - """Error while unserializing an object.""" - - -def bchr(n: int) -> bytes: - return bytes([n]) - - -DUMPFORMAT_VERSION = bchr(2) - -FOUR_BYTE_INT_MAX = 2147483647 - -FLOAT_FORMAT = "!d" -FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) -COMPLEX_FORMAT = "!dd" -COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) - - -class _Stop(Exception): - pass - - -class opcode: - """Container for name -> num mappings.""" - - BUILDTUPLE = b"@" - BYTES = b"A" - CHANNEL = b"B" - FALSE = b"C" - FLOAT = b"D" - FROZENSET = b"E" - INT = b"F" - LONG = b"G" - LONGINT = b"H" - LONGLONG = b"I" - NEWDICT = b"J" - NEWLIST = b"K" - NONE = b"L" - PY2STRING = b"M" - PY3STRING = b"N" - SET = b"O" - SETITEM = b"P" - STOP = b"Q" - TRUE = b"R" - UNICODE = b"S" - COMPLEX = b"T" - - -class Unserializer: - num2func: dict[bytes, Callable[[Unserializer], None]] = {} - py2str_as_py3str = True # True - py3str_as_py2str = False # false means py2 will get unicode - - def __init__( - self, - stream: ReadIO, - channel_or_gateway: Channel | BaseGateway | None = None, - strconfig: tuple[bool, bool] | None = None, - ) -> None: - if isinstance(channel_or_gateway, Channel): - gw: BaseGateway | None = channel_or_gateway.gateway - else: - gw = channel_or_gateway - if channel_or_gateway is not None: - strconfig = channel_or_gateway._strconfig - if strconfig: - self.py2str_as_py3str, self.py3str_as_py2str = strconfig - self.stream = stream - if gw is None: - self.channelfactory = None - else: - self.channelfactory = gw._channelfactory - - def load(self, versioned: bool = False) -> Any: - if versioned: - ver = self.stream.read(1) - if ver != DUMPFORMAT_VERSION: - raise LoadError("wrong dumpformat version %r" % ver) - self.stack: list[object] = [] - try: - while True: - opcode = self.stream.read(1) - if not opcode: - raise EOFError - try: - loader = self.num2func[opcode] - except KeyError: - raise LoadError( - f"unknown opcode {opcode!r} - wire protocol corruption?" - ) from None - loader(self) - except _Stop: - if len(self.stack) != 1: - raise LoadError("internal unserialization error") from None - return self.stack.pop(0) - else: - raise LoadError("didn't get STOP") - - def load_none(self) -> None: - self.stack.append(None) - - num2func[opcode.NONE] = load_none - - def load_true(self) -> None: - self.stack.append(True) - - num2func[opcode.TRUE] = load_true - - def load_false(self) -> None: - self.stack.append(False) - - num2func[opcode.FALSE] = load_false - - def load_int(self) -> None: - i = self._read_int4() - self.stack.append(i) - - num2func[opcode.INT] = load_int - - def load_longint(self) -> None: - s = self._read_byte_string() - self.stack.append(int(s)) - - num2func[opcode.LONGINT] = load_longint - - load_long = load_int - num2func[opcode.LONG] = load_long - load_longlong = load_longint - num2func[opcode.LONGLONG] = load_longlong - - def load_float(self) -> None: - binary = self.stream.read(FLOAT_FORMAT_SIZE) - self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) - - num2func[opcode.FLOAT] = load_float - - def load_complex(self) -> None: - binary = self.stream.read(COMPLEX_FORMAT_SIZE) - self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) - - num2func[opcode.COMPLEX] = load_complex - - def _read_int4(self) -> int: - value: int = struct.unpack("!i", self.stream.read(4))[0] - return value - - def _read_byte_string(self) -> bytes: - length = self._read_int4() - as_bytes = self.stream.read(length) - return as_bytes - - def load_py3string(self) -> None: - as_bytes = self._read_byte_string() - if self.py3str_as_py2str: - # XXX Should we try to decode into latin-1? - self.stack.append(as_bytes) - else: - self.stack.append(as_bytes.decode("utf-8")) - - num2func[opcode.PY3STRING] = load_py3string - - def load_py2string(self) -> None: - as_bytes = self._read_byte_string() - if self.py2str_as_py3str: - s: bytes | str = as_bytes.decode("latin-1") - else: - s = as_bytes - self.stack.append(s) - - num2func[opcode.PY2STRING] = load_py2string - - def load_bytes(self) -> None: - s = self._read_byte_string() - self.stack.append(s) - - num2func[opcode.BYTES] = load_bytes - - def load_unicode(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.UNICODE] = load_unicode - - def load_newlist(self) -> None: - length = self._read_int4() - self.stack.append([None] * length) - - num2func[opcode.NEWLIST] = load_newlist - - def load_setitem(self) -> None: - if len(self.stack) < 3: - raise LoadError("not enough items for setitem") - value = self.stack.pop() - key = self.stack.pop() - self.stack[-1][key] = value # type: ignore[index] - - num2func[opcode.SETITEM] = load_setitem - - def load_newdict(self) -> None: - self.stack.append({}) - - num2func[opcode.NEWDICT] = load_newdict - - def _load_collection(self, type_: type) -> None: - length = self._read_int4() - if length: - res = type_(self.stack[-length:]) - del self.stack[-length:] - self.stack.append(res) - else: - self.stack.append(type_()) - - def load_buildtuple(self) -> None: - self._load_collection(tuple) - - num2func[opcode.BUILDTUPLE] = load_buildtuple - - def load_set(self) -> None: - self._load_collection(set) - - num2func[opcode.SET] = load_set - - def load_frozenset(self) -> None: - self._load_collection(frozenset) - - num2func[opcode.FROZENSET] = load_frozenset - - def load_stop(self) -> None: - raise _Stop - - num2func[opcode.STOP] = load_stop - - def load_channel(self) -> None: - id = self._read_int4() - assert self.channelfactory is not None - newchannel = self.channelfactory.new(id) - self.stack.append(newchannel) - - num2func[opcode.CHANNEL] = load_channel - - -def dumps(obj: object) -> bytes: - """Serialize the given obj to a bytestring. - - The obj and all contained objects must be of a builtin - Python type (so nested dicts, sets, etc. are all OK but - not user-level instances). - """ - return _Serializer().save(obj, versioned=True) # type: ignore[return-value] - - -def dump(byteio, obj: object) -> None: - """write a serialized bytestring of the given obj to the given stream.""" - _Serializer(write=byteio.write).save(obj, versioned=True) - - -def loads( - bytestring: bytes, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: - """Deserialize the given bytestring to an object. - - py2str_as_py3str: If true then string (str) objects previously - dumped on Python2 will be loaded as Python3 - strings which really are text objects. - py3str_as_py2str: If true then string (str) objects previously - dumped on Python3 will be loaded as Python2 - strings instead of unicode objects. - - If the bytestring was dumped with an incompatible protocol - version or if the bytestring is corrupted, the - ``execnet.DataFormatError`` will be raised. - """ - io = BytesIO(bytestring) - return load( - io, py2str_as_py3str=py2str_as_py3str, py3str_as_py2str=py3str_as_py2str - ) - - -def load( - io: ReadIO, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: - """Derserialize an object form the specified stream. - - Behaviour and parameters are otherwise the same as with ``loads`` - """ - strconfig = (py2str_as_py3str, py3str_as_py2str) - return Unserializer(io, strconfig=strconfig).load(versioned=True) - - -def loads_internal( - bytestring: bytes, - channelfactory=None, - strconfig: tuple[bool, bool] | None = None, -) -> Any: - io = BytesIO(bytestring) - return Unserializer(io, channelfactory, strconfig).load() - - -def dumps_internal(obj: object) -> bytes: - return _Serializer().save(obj) # type: ignore[return-value] - - -class _Serializer: - _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} - - def __init__(self, write: Callable[[bytes], None] | None = None) -> None: - if write is None: - self._streamlist: list[bytes] = [] - write = self._streamlist.append - self._write = write - - def save(self, obj: object, versioned: bool = False) -> bytes | None: - # calling here is not re-entrant but multiple instances - # may write to the same stream because of the common platform - # atomic-write guarantee (concurrent writes each happen atomically) - if versioned: - self._write(DUMPFORMAT_VERSION) - self._save(obj) - self._write(opcode.STOP) - try: - streamlist = self._streamlist - except AttributeError: - return None - return b"".join(streamlist) - - def _save(self, obj: object) -> None: - tp = type(obj) - try: - dispatch = self._dispatch[tp] - except KeyError: - methodname = "save_" + tp.__name__ - meth: Callable[[_Serializer, object], None] | None = getattr( - self.__class__, methodname, None - ) - if meth is None: - raise DumpError(f"can't serialize {tp}") from None - dispatch = self._dispatch[tp] = meth - dispatch(self, obj) - - def save_NoneType(self, non: None) -> None: - self._write(opcode.NONE) - - def save_bool(self, boolean: bool) -> None: - if boolean: - self._write(opcode.TRUE) - else: - self._write(opcode.FALSE) - - def save_bytes(self, bytes_: bytes) -> None: - self._write(opcode.BYTES) - self._write_byte_sequence(bytes_) - - def save_str(self, s: str) -> None: - self._write(opcode.PY3STRING) - self._write_unicode_string(s) - - def _write_unicode_string(self, s: str) -> None: - try: - as_bytes = s.encode("utf-8") - except UnicodeEncodeError as e: - raise DumpError("strings must be utf-8 encodable") from e - self._write_byte_sequence(as_bytes) - - def _write_byte_sequence(self, bytes_: bytes) -> None: - self._write_int4(len(bytes_), "string is too long") - self._write(bytes_) - - def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - if i <= FOUR_BYTE_INT_MAX: - self._write(short_op) - self._write_int4(i) - else: - self._write(long_op) - self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) - - def save_int(self, i: int) -> None: - self._save_integral(i, opcode.INT, opcode.LONGINT) - - def save_long(self, l: int) -> None: - self._save_integral(l, opcode.LONG, opcode.LONGLONG) - - def save_float(self, flt: float) -> None: - self._write(opcode.FLOAT) - self._write(struct.pack(FLOAT_FORMAT, flt)) - - def save_complex(self, cpx: complex) -> None: - self._write(opcode.COMPLEX) - self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) - - def _write_int4( - self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) - ) -> None: - if i > FOUR_BYTE_INT_MAX: - raise DumpError(error) - self._write(struct.pack("!i", i)) - - def save_list(self, L: list[object]) -> None: - self._write(opcode.NEWLIST) - self._write_int4(len(L), "list is too long") - for i, item in enumerate(L): - self._write_setitem(i, item) - - def _write_setitem(self, key: object, value: object) -> None: - self._save(key) - self._save(value) - self._write(opcode.SETITEM) - - def save_dict(self, d: dict[object, object]) -> None: - self._write(opcode.NEWDICT) - for key, value in d.items(): - self._write_setitem(key, value) - - def save_tuple(self, tup: tuple[object, ...]) -> None: - for item in tup: - self._save(item) - self._write(opcode.BUILDTUPLE) - self._write_int4(len(tup), "tuple is too long") - - def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: - for item in s: - self._save(item) - self._write(op) - self._write_int4(len(s), "set is too long") - - def save_set(self, s: set[object]) -> None: - self._write_set(s, opcode.SET) - - def save_frozenset(self, s: frozenset[object]) -> None: - self._write_set(s, opcode.FROZENSET) - - def save_Channel(self, channel: Channel) -> None: - self._write(opcode.CHANNEL) - self._write_int4(channel.id) - - -def init_popen_io(execmodel: ExecModel) -> Popen2IO: - if not hasattr(os, "dup"): # jython - io = Popen2IO(sys.stdout, sys.stdin, execmodel) - import tempfile - - sys.stdin = tempfile.TemporaryFile("r") - sys.stdout = tempfile.TemporaryFile("w") - else: - try: - devnull = os.devnull - except AttributeError: - devnull = "NUL" if os.name == "nt" else "/dev/null" - # stdin - stdin = execmodel.fdopen(os.dup(0), "r", 1) - fd = os.open(devnull, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - - # stdout - stdout = execmodel.fdopen(os.dup(1), "w", 1) - fd = os.open(devnull, os.O_WRONLY) - os.dup2(fd, 1) - - # stderr for win32 - if os.name == "nt": - sys.stderr = execmodel.fdopen(os.dup(2), "w", 1) - os.dup2(fd, 2) - os.close(fd) - io = Popen2IO(stdout, stdin, execmodel) - # Use closefd=False since 0 and 1 are shared with - # sys.__stdin__ and sys.__stdout__. - sys.stdin = execmodel.fdopen(0, "r", 1, closefd=False) - sys.stdout = execmodel.fdopen(1, "w", 1, closefd=False) - return io - - -def serve(io: IO, id) -> None: - trace(f"creating workergateway on {io!r}") - WorkerGateway(io=io, id=id, _startcount=2).serve() +from ._shim import forwarder + +_MOVED = { + # tracing + "DEBUG": "._trace", + "pid": "._trace", + "trace": "._trace", + "notrace": "._trace", + # errors and error texts + "sysex": "._errors", + "INTERRUPT_TEXT": "._errors", + "GatewayReceivedTerminate": "._errors", + "HostNotFound": "._errors", + "geterrortext": "._errors", + "RemoteError": "._errors", + "TimeoutError": "._errors", + "DataFormatError": "._errors", + "DumpError": "._errors", + "LoadError": "._errors", + # execution model presets + "ExecModel": "._execmodel", + "get_execmodel": "._execmodel", + # wire protocol + "WriteIO": "._message", + "ReadIO": "._message", + "IO": "._message", + "Message": "._message", + "gateway_info": "._message", + "FrameDecoder": "._message", + # serializer + "bchr": "._serialize", + "DUMPFORMAT_VERSION": "._serialize", + "FOUR_BYTE_INT_MAX": "._serialize", + "FOUR_BYTE_INT_MIN": "._serialize", + "FLOAT_FORMAT": "._serialize", + "FLOAT_FORMAT_SIZE": "._serialize", + "COMPLEX_FORMAT": "._serialize", + "COMPLEX_FORMAT_SIZE": "._serialize", + "opcode": "._serialize", + "Unserializer": "._serialize", + "_Serializer": "._serialize", + "_Stop": "._serialize", + "dumps": "._serialize", + "dump": "._serialize", + "loads": "._serialize", + "load": "._serialize", + "dumps_internal": "._serialize", + "loads_internal": "._serialize", + # channels + "Channel": "._channel", + "ChannelFactory": "._channel", + "ChannelFile": "._channel", + "ChannelFileWrite": "._channel", + "ChannelFileRead": "._channel", + "ENDMARKER": "._channel", + "NO_ENDMARKER_WANTED": "._channel", + # gateways + "BaseGateway": "._gateway_base", + "WorkerGateway": "._gateway_base", +} + +__getattr__ = forwarder("gateway_base", _MOVED) diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py deleted file mode 100644 index e9d7efe1..00000000 --- a/src/execnet/gateway_bootstrap.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Code to initialize the remote side of a gateway once the IO is created.""" - -from __future__ import annotations - -import inspect -import os - -import execnet - -from . import gateway_base -from .gateway_base import IO -from .xspec import XSpec - -importdir = os.path.dirname(os.path.dirname(execnet.__file__)) - - -class HostNotFound(Exception): - pass - - -def bootstrap_import(io: IO, spec: XSpec) -> None: - # Only insert the importdir into the path if we must. This prevents - # bugs where backports expect to be shadowed by the standard library on - # newer versions of python but would instead shadow the standard library. - sendexec( - io, - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet.gateway_base import serve, init_popen_io, get_execmodel", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "execmodel = get_execmodel(%r)" % spec.execmodel, - "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1", repr(s) - - -def bootstrap_exec(io: IO, spec: XSpec) -> None: - try: - sendexec( - io, - inspect.getsource(gateway_base), - "execmodel = get_execmodel(%r)" % spec.execmodel, - "io = init_popen_io(execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1" - except EOFError: - ret = io.wait() - if ret == 255 and hasattr(io, "remoteaddress"): - raise HostNotFound(io.remoteaddress) from None - - -def bootstrap_socket(io: IO, id) -> None: - # XXX: switch to spec - from execnet.gateway_socket import SocketIO - - sendexec( - io, - inspect.getsource(gateway_base), - "import socket", - inspect.getsource(SocketIO), - "try: execmodel", - "except NameError:", - " execmodel = get_execmodel('thread')", - "io = SocketIO(clientsock, execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % id, - ) - s = io.read(1) - assert s == b"1" - - -def sendexec(io: IO, *sources: str) -> None: - source = "\n".join(sources) - io.write((repr(source) + "\n").encode("utf-8")) - - -def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: - if spec.popen: - if spec.via or spec.python: - bootstrap_exec(io, spec) - else: - bootstrap_import(io, spec) - elif spec.ssh or spec.vagrant_ssh: - bootstrap_exec(io, spec) - elif spec.socket: - bootstrap_socket(io, spec) - else: - raise ValueError("unknown gateway type, can't bootstrap") - gw = execnet.Gateway(io, spec) - return gw diff --git a/src/execnet/gateway_io.py b/src/execnet/gateway_io.py deleted file mode 100644 index 21285ab4..00000000 --- a/src/execnet/gateway_io.py +++ /dev/null @@ -1,255 +0,0 @@ -"""execnet IO initialization code. - -Creates IO instances used for gateway IO. -""" - -from __future__ import annotations - -import shlex -import sys -from typing import TYPE_CHECKING -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - from execnet.xspec import XSpec - -try: - from execnet.gateway_base import Message - from execnet.gateway_base import Popen2IO -except ImportError: - from __main__ import Message # type: ignore[no-redef] - from __main__ import Popen2IO # type: ignore[no-redef] - -from functools import partial - - -class Popen2IOMaster(Popen2IO): - # Set externally, for some specs only. - remoteaddress: str - - def __init__(self, args, execmodel: ExecModel) -> None: - PIPE = execmodel.subprocess.PIPE - self.popen = p = execmodel.subprocess.Popen(args, stdout=PIPE, stdin=PIPE) - super().__init__(p.stdin, p.stdout, execmodel=execmodel) - - def wait(self) -> int | None: - try: - return self.popen.wait() # type: ignore[no-any-return] - except OSError: - return None - - def kill(self) -> None: - try: - self.popen.kill() - except OSError as e: - sys.stderr.write("ERROR killing: %s\n" % e) - sys.stderr.flush() - - -popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" - - -def shell_split_path(path: str) -> list[str]: - """ - Use shell lexer to split the given path into a list of components, - taking care to handle Windows' '\' correctly. - """ - if sys.platform.startswith("win"): - # replace \\ by / otherwise shlex will strip them out - path = path.replace("\\", "/") - return shlex.split(path) - - -def popen_args(spec: XSpec) -> list[str]: - args = shell_split_path(spec.python) if spec.python else [sys.executable] - args.append("-u") - if spec.dont_write_bytecode: - args.append("-B") - args.extend(["-c", popen_bootstrapline]) - return args - - -def ssh_args(spec: XSpec) -> list[str]: - # NOTE: If changing this, you need to sync those changes to vagrant_args - # as well, or, take some time to further refactor the commonalities of - # ssh_args and vagrant_args. - remotepython = spec.python or "python" - args = ["ssh", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - - assert spec.ssh is not None - args.extend(spec.ssh.split()) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.append(remotecmd) - return args - - -def vagrant_ssh_args(spec: XSpec) -> list[str]: - # This is the vagrant-wrapped version of SSH. Unfortunately the - # command lines are incompatible to just channel through ssh_args - # due to ordering/templating issues. - # NOTE: This should be kept in sync with the ssh_args behaviour. - # spec.vagrant is identical to spec.ssh in that they both carry - # the remote host "address". - assert spec.vagrant_ssh is not None - remotepython = spec.python or "python" - args = ["vagrant", "ssh", spec.vagrant_ssh, "--", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.extend([remotecmd]) - return args - - -def create_io(spec: XSpec, execmodel: ExecModel) -> Popen2IOMaster: - if spec.popen: - args = popen_args(spec) - return Popen2IOMaster(args, execmodel) - if spec.ssh: - args = ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.ssh - return io - if spec.vagrant_ssh: - args = vagrant_ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.vagrant_ssh - return io - assert False - - -# -# Proxy Gateway handling code -# -# master: proxy initiator -# forwarder: forwards between master and sub -# sub: sub process that is proxied to the initiator - -RIO_KILL = 1 -RIO_WAIT = 2 -RIO_REMOTEADDRESS = 3 -RIO_CLOSE_WRITE = 4 - - -class ProxyIO: - """A Proxy IO object allows to instantiate a Gateway - through another "via" gateway. - - A master:ProxyIO object provides an IO object effectively connected to the - sub via the forwarder. To achieve this, master:ProxyIO interacts with - forwarder:serve_proxy_io() which itself instantiates and interacts with the - sub. - """ - - def __init__(self, proxy_channel: Channel, execmodel: ExecModel) -> None: - # after exchanging the control channel we use proxy_channel - # for messaging IO - self.controlchan = proxy_channel.gateway.newchannel() - proxy_channel.send(self.controlchan) - self.iochan = proxy_channel - self.iochan_file = self.iochan.makefile("r") - self.execmodel = execmodel - - def read(self, nbytes: int) -> bytes: - # TODO(typing): The IO protocol requires bytes here but ChannelFileRead - # returns str. - return self.iochan_file.read(nbytes) # type: ignore[return-value] - - def write(self, data: bytes) -> None: - self.iochan.send(data) - - def _controll(self, event: int) -> object: - self.controlchan.send(event) - return self.controlchan.receive() - - def close_write(self) -> None: - self._controll(RIO_CLOSE_WRITE) - - def close_read(self) -> None: - raise NotImplementedError() - - def kill(self) -> None: - self._controll(RIO_KILL) - - def wait(self) -> int | None: - response = self._controll(RIO_WAIT) - assert response is None or isinstance(response, int) - return response - - @property - def remoteaddress(self) -> str: - response = self._controll(RIO_REMOTEADDRESS) - assert isinstance(response, str) - return response - - def __repr__(self) -> str: - return f"" - - -class PseudoSpec: - def __init__(self, vars) -> None: - self.__dict__.update(vars) - - def __getattr__(self, name: str) -> None: - return None - - -def serve_proxy_io(proxy_channelX: Channel) -> None: - execmodel = proxy_channelX.gateway.execmodel - log = partial( - proxy_channelX.gateway._trace, "serve_proxy_io:%s" % proxy_channelX.id - ) - spec = cast("XSpec", PseudoSpec(proxy_channelX.receive())) - # create sub IO object which we will proxy back to our proxy initiator - sub_io = create_io(spec, execmodel) - control_chan = cast("Channel", proxy_channelX.receive()) - log("got control chan", control_chan) - - # read data from master, forward it to the sub - # XXX writing might block, thus blocking the receiver thread - def forward_to_sub(data: bytes) -> None: - log("forward data to sub, size %s" % len(data)) - sub_io.write(data) - - proxy_channelX.setcallback(forward_to_sub) - - def control(data: int) -> None: - if data == RIO_WAIT: - control_chan.send(sub_io.wait()) - elif data == RIO_KILL: - sub_io.kill() - control_chan.send(None) - elif data == RIO_REMOTEADDRESS: - control_chan.send(sub_io.remoteaddress) - elif data == RIO_CLOSE_WRITE: - sub_io.close_write() - control_chan.send(None) - - control_chan.setcallback(control) - - # write data to the master coming from the sub - forward_to_master_file = proxy_channelX.makefile("w") - - # read bootstrap byte from sub, send it on to master - log("reading bootstrap byte from sub", spec.id) - initial = sub_io.read(1) - assert initial == b"1", initial - log("forwarding bootstrap byte from sub", spec.id) - forward_to_master_file.write(initial) - - # enter message forwarding loop - while True: - try: - message = Message.from_io(sub_io) - except EOFError: - log("EOF from sub, terminating proxying loop", spec.id) - break - message.to_io(forward_to_master_file) - # proxy_channelX will be closed from remote_exec's finalization code - - -if __name__ == "__channelexec__": - serve_proxy_io(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/gateway_socket.py b/src/execnet/gateway_socket.py deleted file mode 100644 index be42f1ab..00000000 --- a/src/execnet/gateway_socket.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import sys -from typing import cast - -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_bootstrap import HostNotFound -from execnet.multi import Group -from execnet.xspec import XSpec - - -class SocketIO: - remoteaddress: str - - def __init__(self, sock, execmodel: ExecModel) -> None: - self.sock = sock - self.execmodel = execmodel - socket = execmodel.socket - try: - # IPTOS_LOWDELAY - sock.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) - sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - except (AttributeError, OSError): - sys.stderr.write("WARNING: cannot set socketoption") - - def read(self, numbytes: int) -> bytes: - "Read exactly 'bytes' bytes from the socket." - buf = b"" - while len(buf) < numbytes: - t = self.sock.recv(numbytes - len(buf)) - if not t: - raise EOFError - buf += t - return buf - - def write(self, data: bytes) -> None: - self.sock.sendall(data) - - def close_read(self) -> None: - try: - self.sock.shutdown(0) - except self.execmodel.socket.error: - pass - - def close_write(self) -> None: - try: - self.sock.shutdown(1) - except self.execmodel.socket.error: - pass - - def wait(self) -> None: - pass - - def kill(self) -> None: - pass - - -def start_via( - gateway: Gateway, hostport: tuple[str, int] | None = None -) -> tuple[str, int]: - """Instantiate a socketserver on the given gateway. - - Returns a host, port tuple. - """ - if hostport is None: - host, port = ("localhost", 0) - else: - host, port = hostport - - from execnet.script import socketserver - - # execute the above socketserverbootstrap on the other side - channel = gateway.remote_exec(socketserver) - channel.send((host, port)) - realhost, realport = cast("tuple[str, int]", channel.receive()) - # self._trace("new_remote received" - # "port=%r, hostname = %r" %(realport, hostname)) - if not realhost or realhost == "0.0.0.0": - realhost = "localhost" - return realhost, realport - - -def create_io(spec: XSpec, group: Group, execmodel: ExecModel) -> SocketIO: - assert spec.socket is not None - assert not spec.python, "socket: specifying python executables not yet supported" - gateway_id = spec.installvia - if gateway_id: - host, port = start_via(group[gateway_id]) - else: - host, port_str = spec.socket.split(":") - port = int(port_str) - - socket = execmodel.socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - io = SocketIO(sock, execmodel) - io.remoteaddress = "%s:%d" % (host, port) - try: - sock.connect((host, port)) - except execmodel.socket.gaierror as e: - raise HostNotFound() from e - return io diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py new file mode 100644 index 00000000..df5aadfb --- /dev/null +++ b/src/execnet/gevent.py @@ -0,0 +1,75 @@ +"""The blocking execnet API for gevent applications. + +Identical to :mod:`execnet.sync` except that every blocking wait parks the +calling *greenlet* rather than its OS thread:: + + import execnet.gevent + + group = execnet.gevent.Group() + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + print(channel.receive()) # parks this greenlet, not the hub + +Protocol IO runs on the shared Trio host thread as it does for every +blocking surface; the difference is only which primitive a waiter parks +on, so a slow ``receive`` no longer stalls the whole hub. Requires +gevent (``execnet[gevent]``). + +This is about the *caller*: the worker's own shape is the ``profile=`` +spec key, and ``profile=gevent`` is an independent choice. Importing this +module does not monkey-patch anything -- do that yourself, as early as +usual. +""" + +from __future__ import annotations + +import gevent # noqa: F401 -- fail at import time when gevent is missing + +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._multi import Group as _SyncGroup +from ._multi import MultiChannel +from ._xspec import XSpec +from .sync import Channel +from .sync import Gateway +from .sync import Host +from .sync import RSync + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "Host", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "makegateway", +] + + +class Group(_SyncGroup): + """A gateway group whose blocking waits park greenlets. + + Every gateway it creates inherits the gevent wait backend, so + ``channel.receive()``, ``waitclose()``, sends waiting on their write + acknowledgement, ``join()`` and ``Group.terminate()`` all yield to the + hub instead of blocking the thread running it. + """ + + _wait_backend = "gevent" + + +#: convenience group for scripts; real applications should own a Group +default_group = Group() +makegateway = default_group.makegateway diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4dbf8b89..06683457 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -1,374 +1,22 @@ -""" -Managing Gateway Groups and interactions with multiple channels. +"""Deprecated alias for :mod:`execnet._multi`. -(c) 2008-2014, Holger Krekel and others +``Group``, ``MultiChannel``, ``default_group``, ``makegateway`` and +``set_execmodel`` are exported from :mod:`execnet` and :mod:`execnet.sync`; +use those. """ from __future__ import annotations -import atexit -import types -from collections.abc import Callable -from collections.abc import Iterable -from collections.abc import Iterator -from collections.abc import Sequence -from functools import partial -from threading import Lock -from typing import TYPE_CHECKING -from typing import Any -from typing import Literal -from typing import TypeAlias -from typing import overload - -from . import gateway_bootstrap -from . import gateway_io -from .gateway_base import Channel -from .gateway_base import ExecModel -from .gateway_base import WorkerPool -from .gateway_base import get_execmodel -from .gateway_base import trace -from .xspec import XSpec - -if TYPE_CHECKING: - from .gateway import Gateway - - -NO_ENDMARKER_WANTED = object() - - -class Group: - """Gateway Group.""" - - defaultspec = "popen" - - def __init__( - self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" - ) -> None: - """Initialize a group and make gateways as specified. - - execmodel can be one of the supported execution models. - """ - self._gateways: list[Gateway] = [] - self._autoidcounter = 0 - self._autoidlock = Lock() - self._gateways_to_join: list[Gateway] = [] - # we use the same execmodel for all of the Gateway objects - # we spawn on our side. Probably we should not allow different - # execmodels between different groups but not clear. - # Note that "other side" execmodels may differ and is typically - # specified by the spec passed to makegateway. - self.set_execmodel(execmodel) - for xspec in xspecs: - self.makegateway(xspec) - atexit.register(self._cleanup_atexit) - - @property - def execmodel(self) -> ExecModel: - return self._execmodel - - @property - def remote_execmodel(self) -> ExecModel: - return self._remote_execmodel - - def set_execmodel( - self, execmodel: str, remote_execmodel: str | None = None - ) -> None: - """Set the execution model for local and remote site. - - execmodel can be one of the supported execution models. - It determines the execution model for any newly created gateway. - If remote_execmodel is not specified it takes on the value of execmodel. - - NOTE: Execution models can only be set before any gateway is created. - """ - if self._gateways: - raise ValueError( - "can not set execution models if gateways have been created already" - ) - if remote_execmodel is None: - remote_execmodel = execmodel - self._execmodel = get_execmodel(execmodel) - self._remote_execmodel = get_execmodel(remote_execmodel) - - def __repr__(self) -> str: - idgateways = [gw.id for gw in self] - return "" % idgateways - - def __getitem__(self, key: int | str | Gateway) -> Gateway: - if isinstance(key, int): - return self._gateways[key] - for gw in self._gateways: - if gw == key or gw.id == key: - return gw - raise KeyError(key) - - def __contains__(self, key: str) -> bool: - try: - self[key] - return True - except KeyError: - return False - - def __len__(self) -> int: - return len(self._gateways) - - def __iter__(self) -> Iterator[Gateway]: - return iter(list(self._gateways)) - - def makegateway(self, spec: XSpec | str | None = None) -> Gateway: - """Create and configure a gateway to a Python interpreter. - - The ``spec`` string encodes the target gateway type - and configuration information. The general format is:: - - key1=value1//key2=value2//... - - If you leave out the ``=value`` part a True value is assumed. - Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. - Valid configuration:: - - id= specifies the gateway id - python= specifies which python interpreter to execute - execmodel=model 'thread', 'main_thread_only', 'eventlet', 'gevent' execution model - chdir= specifies to which directory to change - nice= specifies process priority of new process - env:NAME=value specifies a remote environment variable setting. - - If no spec is given, self.defaultspec is used. - """ - if not spec: - spec = self.defaultspec - if not isinstance(spec, XSpec): - spec = XSpec(spec) - self.allocate_id(spec) - if spec.execmodel is None: - spec.execmodel = self.remote_execmodel.backend - if spec.via: - assert not spec.socket - master = self[spec.via] - proxy_channel = master.remote_exec(gateway_io) - proxy_channel.send(vars(spec)) - proxy_io_master = gateway_io.ProxyIO(proxy_channel, self.execmodel) - gw = gateway_bootstrap.bootstrap(proxy_io_master, spec) - elif spec.popen or spec.ssh or spec.vagrant_ssh: - io = gateway_io.create_io(spec, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(io, spec) - elif spec.socket: - from . import gateway_socket - - sio = gateway_socket.create_io(spec, self, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(sio, spec) - else: - raise ValueError(f"no gateway type found for {spec._spec!r}") - gw.spec = spec - self._register(gw) - if spec.chdir or spec.nice or spec.env: - channel = gw.remote_exec( - """ - import os - path, nice, env = channel.receive() - if path: - if not os.path.exists(path): - os.mkdir(path) - os.chdir(path) - if nice and hasattr(os, 'nice'): - os.nice(nice) - if env: - for name, value in env.items(): - os.environ[name] = value - """ - ) - nice = (spec.nice and int(spec.nice)) or 0 - channel.send((spec.chdir, nice, spec.env)) - channel.waitclose() - return gw - - def allocate_id(self, spec: XSpec) -> None: - """(re-entrant) allocate id for the given xspec object.""" - if spec.id is None: - with self._autoidlock: - id = "gw" + str(self._autoidcounter) - self._autoidcounter += 1 - if id in self: - raise ValueError(f"already have gateway with id {id!r}") - spec.id = id - - def _register(self, gateway: Gateway) -> None: - assert not hasattr(gateway, "_group") - assert gateway.id - assert gateway.id not in self - self._gateways.append(gateway) - gateway._group = self - - def _unregister(self, gateway: Gateway) -> None: - self._gateways.remove(gateway) - self._gateways_to_join.append(gateway) - - def _cleanup_atexit(self) -> None: - trace(f"=== atexit cleanup {self!r} ===") - self.terminate(timeout=1.0) - - def terminate(self, timeout: float | None = None) -> None: - """Trigger exit of member gateways and wait for termination - of member gateways and associated subprocesses. - - After waiting timeout seconds try to to kill local sub processes of - popen- and ssh-gateways. - - Timeout defaults to None meaning open-ended waiting and no kill - attempts. - """ - while self: - vias: set[str] = set() - for gw in self: - if gw.spec.via: - vias.add(gw.spec.via) - for gw in self: - if gw.id not in vias: - gw.exit() - - def join_wait(gw: Gateway) -> None: - gw.join() - gw._io.wait() - - def kill(gw: Gateway) -> None: - trace("Gateways did not come down after timeout: %r" % gw) - gw._io.kill() - - safe_terminate( - self.execmodel, - timeout, - [ - (partial(join_wait, gw), partial(kill, gw)) - for gw in self._gateways_to_join - ], - ) - self._gateways_to_join[:] = [] - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs, - ) -> MultiChannel: - """remote_exec source on all member gateways and return - a MultiChannel connecting to all sub processes.""" - channels = [] - for gw in self: - channels.append(gw.remote_exec(source, **kwargs)) - return MultiChannel(channels) - - -class MultiChannel: - def __init__(self, channels: Sequence[Channel]) -> None: - self._channels = channels - - def __len__(self) -> int: - return len(self._channels) - - def __iter__(self) -> Iterator[Channel]: - return iter(self._channels) - - def __getitem__(self, key: int) -> Channel: - return self._channels[key] - - def __contains__(self, chan: Channel) -> bool: - return chan in self._channels - - def send_each(self, item: object) -> None: - for ch in self._channels: - ch.send(item) - - @overload - def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: - pass - - @overload - def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: - pass - - def receive_each( - self, withchannel: bool = False - ) -> list[tuple[Channel, Any]] | list[Any]: - assert not hasattr(self, "_queue") - l: list[object] = [] - for ch in self._channels: - obj = ch.receive() - if withchannel: - l.append((ch, obj)) - else: - l.append(obj) - return l - - def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): - try: - return self._queue # type: ignore[has-type] - except AttributeError: - self._queue = None - for ch in self._channels: - if self._queue is None: - self._queue = ch.gateway.execmodel.queue.Queue() - - def putreceived(obj, channel: Channel = ch) -> None: - self._queue.put((channel, obj)) # type: ignore[union-attr] - - if endmarker is NO_ENDMARKER_WANTED: - ch.setcallback(putreceived) - else: - ch.setcallback(putreceived, endmarker=endmarker) - return self._queue - - def waitclose(self) -> None: - first = None - for ch in self._channels: - try: - ch.waitclose() - except ch.RemoteError as exc: - if first is None: - first = exc - if first: - raise first - - -TermKillFunc: TypeAlias = Callable[[], object] -TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] - - -def safe_terminate( - execmodel: ExecModel, - timeout: float | None, - list_of_paired_functions: Sequence[TermKillPair], -) -> None: - """Run terminate/kill pairs in parallel with a hard wait bound. - - Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - Waiting for the worker pool is also bounded so a stuck kill cannot hang - the caller forever (see issues #43 / #221). - """ - workerpool = WorkerPool(execmodel) - - def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - termreply = workerpool.spawn(termfunc) - try: - termreply.get(timeout=timeout) - except OSError: - killfunc() - - replylist = [ - workerpool.spawn(termkill, termfunc, killfunc) - for termfunc, killfunc in list_of_paired_functions - ] - # Allow term timeout plus a kill attempt; never block indefinitely. - wait_timeout = None if timeout is None else timeout * 2 - for reply in replylist: - try: - reply.waitfinish(timeout=wait_timeout) - except OSError: - # termkill still running (typically stuck in killfunc). - continue - reply.get() # propagate worker exceptions, if any - workerpool.waitall(timeout=wait_timeout) +from ._shim import forwarder +_MOVED = { + "Group": "._multi", + "MultiChannel": "._multi", + "default_group": "._multi", + "makegateway": "._multi", + "set_execmodel": "._multi", + "safe_terminate": "._multi", + "NO_ENDMARKER_WANTED": "._multi", +} -default_group = Group() -makegateway = default_group.makegateway -set_execmodel = default_group.set_execmodel +__getattr__ = forwarder("multi", _MOVED) diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index f92bc37a..786ed42b 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -1,249 +1,12 @@ -""" -1:N rsync implementation on top of execnet. +"""Deprecated alias for :mod:`execnet._rsync`. -(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +``RSync`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import os -import stat -from collections.abc import Callable -from hashlib import md5 -from queue import Queue -from typing import Literal - -import execnet.rsync_remote -from execnet.gateway import Gateway -from execnet.gateway_base import BaseGateway -from execnet.gateway_base import Channel - - -class RSync: - """This class allows to send a directory structure (recursively) - to one or multiple remote filesystems. - - There is limited support for symlinks, which means that symlinks - pointing to the sourcetree will be send "as is" while external - symlinks will be just copied (regardless of existence of such - a path on remote side). - """ - - def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: - self._sourcedir = str(sourcedir) - self._verbose = verbose - assert callback is None or callable(callback) - self._callback = callback - self._channels: dict[Channel, Callable[[], None] | None] = {} - self._receivequeue: Queue[ - tuple[ - Channel, - ( - None - | tuple[Literal["send"], tuple[list[str], bytes]] - | tuple[Literal["list_done"], None] - | tuple[Literal["ack"], str] - | tuple[Literal["links"], None] - | tuple[Literal["done"], None] - ), - ] - ] = Queue() - self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] - - def filter(self, path: str) -> bool: - return True - - def _end_of_channel(self, channel: Channel) -> None: - if channel in self._channels: - # too early! we must have got an error - channel.waitclose() - # or else we raise one - raise OSError(f"connection unexpectedly closed: {channel.gateway} ") - - def _process_link(self, channel: Channel) -> None: - for link in self._links: - channel.send(link) - # completion marker, this host is done - channel.send(42) - - def _done(self, channel: Channel) -> None: - """Call all callbacks.""" - finishedcallback = self._channels.pop(channel) - if finishedcallback: - finishedcallback() - channel.waitclose() - - def _list_done(self, channel: Channel) -> None: - # sum up all to send - if self._callback: - s = sum([self._paths[i] for i in self._to_send[channel]]) - self._callback("list", s, channel) - - def _send_item( - self, - channel: Channel, - modified_rel_path_components: list[str], - checksum: bytes, - ) -> None: - """Send one item.""" - modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) - try: - with open(modified_path, "rb") as fp: - data = fp.read() - except OSError: - data = None - - # provide info to progress callback function - modified_rel_path = "/".join(modified_rel_path_components) - if data is not None: - self._paths[modified_rel_path] = len(data) - else: - self._paths[modified_rel_path] = 0 - if channel not in self._to_send: - self._to_send[channel] = [] - self._to_send[channel].append(modified_rel_path) - # print "sending", modified_rel_path, data and len(data) or 0, checksum - - if data is not None: - if checksum is not None and checksum == md5(data).digest(): - data = None # not really modified - else: - self._report_send_file(channel.gateway, modified_rel_path) - channel.send(data) - - def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: - if self._verbose: - print(f"{gateway} <= {modified_rel_path}") - - def send(self, raises: bool = True) -> None: - """Sends a sourcedir to all added targets. - - raises indicates whether to raise an error or return in case of lack of - targets. - """ - if not self._channels: - if raises: - raise OSError( - "no targets available, maybe you are trying call send() twice?" - ) - return - # normalize a trailing '/' away - self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) - # send directory structure and file timestamps/sizes - self._send_directory_structure(self._sourcedir) - - # paths and to_send are only used for doing - # progress-related callbacks - self._paths: dict[str, int] = {} - self._to_send: dict[Channel, list[str]] = {} - - # send modified file to clients - while self._channels: - channel, req = self._receivequeue.get() - if req is None: - self._end_of_channel(channel) - else: - if req[0] == "links": - self._process_link(channel) - elif req[0] == "done": - self._done(channel) - elif req[0] == "ack": - if self._callback: - self._callback("ack", self._paths[req[1]], channel) - elif req[0] == "list_done": - self._list_done(channel) - elif req[0] == "send": - self._send_item(channel, req[1][0], req[1][1]) - else: - assert "Unknown command %s" % req[0] # type: ignore[unreachable] - - def add_target( - self, - gateway: Gateway, - destdir: str | os.PathLike[str], - finishedcallback: Callable[[], None] | None = None, - **options, - ) -> None: - """Add a remote target specified via a gateway and a remote destination - directory.""" - for name in options: - assert name in ("delete",) - - def itemcallback(req) -> None: - self._receivequeue.put((channel, req)) - - channel = gateway.remote_exec(execnet.rsync_remote) - channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) - channel.setcallback(itemcallback, endmarker=None) - channel.send((str(destdir), options)) - self._channels[channel] = finishedcallback - - def _broadcast(self, msg: object) -> None: - for channel in self._channels: - channel.send(msg) - - def _send_link( - self, - linktype: Literal["linkbase", "link"], - basename: str, - linkpoint: str, - ) -> None: - self._links.append((linktype, basename, linkpoint)) - - def _send_directory(self, path: str) -> None: - # dir: send a list of entries - names = [] - subpaths = [] - for name in os.listdir(path): - p = os.path.join(path, name) - if self.filter(p): - names.append(name) - subpaths.append(p) - mode = os.lstat(path).st_mode - self._broadcast([mode, *names]) - for p in subpaths: - self._send_directory_structure(p) +from ._shim import forwarder - def _send_link_structure(self, path: str) -> None: - sourcedir = self._sourcedir - basename = path[len(self._sourcedir) + 1 :] - linkpoint = os.readlink(path) - # On Windows, readlink returns an extended path (//?/) for - # absolute links, but relpath doesn't like mixing extended - # and non-extended paths. So fix it up ourselves. - if ( - os.path.__name__ == "ntpath" - and linkpoint.startswith("\\\\?\\") - and not self._sourcedir.startswith("\\\\?\\") - ): - sourcedir = "\\\\?\\" + self._sourcedir - try: - relpath = os.path.relpath(linkpoint, sourcedir) - except ValueError: - relpath = None - if ( - relpath is not None - and relpath not in (os.curdir, os.pardir) - and not relpath.startswith(os.pardir + os.sep) - ): - self._send_link("linkbase", basename, relpath) - else: - # relative or absolute link, just send it - self._send_link("link", basename, linkpoint) - self._broadcast(None) +_MOVED = {"RSync": "._rsync"} - def _send_directory_structure(self, path: str) -> None: - try: - st = os.lstat(path) - except OSError: - self._broadcast((None, 0, 0)) - return - if stat.S_ISREG(st.st_mode): - # regular file: send a mode/timestamp/size pair - self._broadcast((st.st_mode, st.st_mtime, st.st_size)) - elif stat.S_ISDIR(st.st_mode): - self._send_directory(path) - elif stat.S_ISLNK(st.st_mode): - self._send_link_structure(path) - else: - raise ValueError(f"cannot sync {path!r}") +__getattr__ = forwarder("rsync", _MOVED) diff --git a/src/execnet/rsync_remote.py b/src/execnet/rsync_remote.py index a8467b76..4770918a 100644 --- a/src/execnet/rsync_remote.py +++ b/src/execnet/rsync_remote.py @@ -1,126 +1,13 @@ -""" -(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +"""Deprecated alias for :mod:`execnet._rsync_remote`. + +The worker half of the rsync protocol; :class:`execnet.RSync` ships it to the +remote side itself, so there is no reason to reference this module. """ from __future__ import annotations -from contextlib import suppress -from typing import TYPE_CHECKING -from typing import Literal -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - - -def serve_rsync(channel: Channel) -> None: - import os - import shutil - import stat - from hashlib import md5 - - destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) - modifiedfiles = [] - - def remove(path: str) -> None: - assert path.startswith(destdir) - try: - os.unlink(path) - except OSError: - # assume it's a dir - shutil.rmtree(path, True) - - def receive_directory_structure(path: str, relcomponents: list[str]) -> None: - try: - st = os.lstat(path) - except OSError: - st = None - msg = channel.receive() - if isinstance(msg, list): - if st and not stat.S_ISDIR(st.st_mode): - os.unlink(path) - st = None - if not st: - os.makedirs(path) - mode = msg.pop(0) - if mode: - # Ensure directories are writable, otherwise a - # permission denied error (EACCES) would be raised - # when attempting to receive read-only directory - # structures. - os.chmod(path, mode | 0o700) - entrynames = {} - for entryname in msg: - destpath = os.path.join(path, entryname) - receive_directory_structure(destpath, [*relcomponents, entryname]) - entrynames[entryname] = True - if options.get("delete"): - for othername in os.listdir(path): - if othername not in entrynames: - otherpath = os.path.join(path, othername) - remove(otherpath) - elif msg is not None: - assert isinstance(msg, tuple) - checksum = None - if st: - if stat.S_ISREG(st.st_mode): - msg_mode, msg_mtime, msg_size = msg - if msg_size != st.st_size: - pass - elif msg_mtime != st.st_mtime: - with open(path, "rb") as fp: - checksum = md5(fp.read()).digest() - elif msg_mode and msg_mode != st.st_mode: - os.chmod(path, msg_mode | 0o700) - return - else: - return # already fine - else: - remove(path) - channel.send(("send", (relcomponents, checksum))) - modifiedfiles.append((path, msg)) - - receive_directory_structure(destdir, []) - - STRICT_CHECK = False # seems most useful this way for py.test - channel.send(("list_done", None)) - - for path, (mode, time, size) in modifiedfiles: - data = cast(bytes, channel.receive()) - channel.send(("ack", path[len(destdir) + 1 :])) - if data is not None: - if STRICT_CHECK and len(data) != size: - raise OSError(f"file modified during rsync: {path!r}") - with open(path, "wb") as fp: - fp.write(data) - try: - if mode: - os.chmod(path, mode) - os.utime(path, (time, time)) - except OSError: - pass - del data - channel.send(("links", None)) - - msg = channel.receive() - while msg != 42: - # we get symlink - _type, relpath, linkpoint = cast( - "tuple[Literal['linkbase', 'link'], str, str]", msg - ) - path = os.path.join(destdir, relpath) - with suppress(OSError): - remove(path) - - if _type == "linkbase": - src = os.path.join(destdir, linkpoint) - else: - assert _type == "link", _type - src = linkpoint - os.symlink(src, path) - msg = channel.receive() - channel.send(("done", None)) +from ._shim import forwarder +_MOVED = {"serve_rsync": "._rsync_remote"} -if __name__ == "__channelexec__": - serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 +__getattr__ = forwarder("rsync_remote", _MOVED) diff --git a/src/execnet/script/__init__.py b/src/execnet/script/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/src/execnet/script/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/src/execnet/script/loop_socketserver.py b/src/execnet/script/loop_socketserver.py deleted file mode 100644 index a4688a80..00000000 --- a/src/execnet/script/loop_socketserver.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess -import sys - -if __name__ == "__main__": - directory = os.path.dirname(os.path.abspath(sys.argv[0])) - script = os.path.join(directory, "socketserver.py") - while 1: - cmdlist = ["python", script] - cmdlist.extend(sys.argv[1:]) - text = "starting subcommand: " + " ".join(cmdlist) - print(text) - process = subprocess.Popen(cmdlist) - process.wait() diff --git a/src/execnet/script/quitserver.py b/src/execnet/script/quitserver.py deleted file mode 100644 index 4c94c383..00000000 --- a/src/execnet/script/quitserver.py +++ /dev/null @@ -1,17 +0,0 @@ -""" - -send a "quit" signal to a remote server - -""" - -from __future__ import annotations - -import socket -import sys - -host, port = sys.argv[1].split(":") -hostport = (host, int(port)) - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(hostport) -sock.sendall(b'"raise KeyboardInterrupt"\n') diff --git a/src/execnet/script/shell.py b/src/execnet/script/shell.py deleted file mode 100644 index de6f8ded..00000000 --- a/src/execnet/script/shell.py +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env python -""" -a remote python shell - -for injection into startserver.py -""" - -import os -import select -import socket -import sys -from threading import Thread -from traceback import print_exc -from typing import NoReturn - - -def clientside() -> NoReturn: - print("client side starting") - host, portstr = sys.argv[1].split(":") - port = int(portstr) - with open(os.path.abspath(sys.argv[0])) as fd: - myself = fd.read() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.sendall((repr(myself) + "\n").encode()) - print("send boot string") - inputlist = [sock, sys.stdin] - try: - while 1: - r, _w, _e = select.select(inputlist, [], []) - if sys.stdin in r: - line = input() - sock.sendall((line + "\n").encode()) - if sock in r: - line = sock.recv(4096).decode() - sys.stdout.write(line) - sys.stdout.flush() - except BaseException: - import traceback - - traceback.print_exc() - - sys.exit(1) - - -class promptagent(Thread): - def __init__(self, clientsock) -> None: - print("server side starting") - super().__init__() # type: ignore[call-overload] - self.clientsock = clientsock - - def run(self) -> None: - print("Entering thread prompt loop") - clientfile = self.clientsock.makefile("w") - - filein = self.clientsock.makefile("r") - loc = self.clientsock.getsockname() - - while 1: - try: - clientfile.write("{} {} >>> ".format(*loc)) - clientfile.flush() - line = filein.readline() - if not line: - raise EOFError("nothing") - if line.strip(): - oldout, olderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = clientfile, clientfile - try: - try: - exec(compile(line + "\n", "", "single")) - except BaseException: - print_exc() - finally: - sys.stdout = oldout - sys.stderr = olderr - clientfile.flush() - except EOFError: - sys.stderr.write("connection close, prompt thread returns") - break - - self.clientsock.close() - - -sock = globals().get("clientsock") -if sock is not None: - prompter = promptagent(sock) - prompter.start() - print("promptagent - thread started") -else: - clientside() diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py deleted file mode 100644 index fa98743c..00000000 --- a/src/execnet/script/socketserver.py +++ /dev/null @@ -1,133 +0,0 @@ -#! /usr/bin/env python -""" -start socket based minimal readline exec server - -it can exeuted in 2 modes of operation - -1. as normal script, that listens for new connections - -2. via existing_gateway.remote_exec (as imported module) - -""" - -# this part of the program only executes on the server side -# -from __future__ import annotations - -import os -import sys -from typing import TYPE_CHECKING - -try: - import fcntl -except ImportError: - fcntl = None # type: ignore[assignment] - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - -progname = "socket_readline_exec_server-1.2" - - -debug = 0 - -if debug: # and not os.isatty(sys.stdin.fileno()) - f = open("/tmp/execnet-socket-pyout.log", "w") - old = sys.stdout, sys.stderr - sys.stdout = sys.stderr = f - - -def print_(*args) -> None: - print(" ".join(str(arg) for arg in args)) - - -exec( - """def exec_(source, locs): - exec(source, locs)""" -) - - -def exec_from_one_connection(serversock) -> None: - print_(progname, "Entering Accept loop", serversock.getsockname()) - clientsock, address = serversock.accept() - print_(progname, "got new connection from {} {}".format(*address)) - clientfile = clientsock.makefile("rb") - print_("reading line") - # rstrip so that we can use \r\n for telnet testing - source = clientfile.readline().rstrip() - clientfile.close() - g = {"clientsock": clientsock, "address": address, "execmodel": execmodel} - source = eval(source) - if source: - co = compile(source + "\n", "", "exec") - print_(progname, "compiled source, executing") - try: - exec_(co, g) # type: ignore[name-defined] # noqa: F821 - finally: - print_(progname, "finished executing code") - # background thread might hold a reference to this (!?) - # clientsock.close() - - -def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): - socket = execmodel.socket - if isinstance(hostport, str): - host, port = hostport.split(":") - hostport = (host, int(port)) - serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # set close-on-exec - if hasattr(fcntl, "FD_CLOEXEC"): - old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD) - fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) - # allow the address to be reused in a reasonable amount of time - if os.name == "posix" and sys.platform != "cygwin": - serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - serversock.bind(hostport) - serversock.listen(5) - return serversock - - -def startserver(serversock, loop: bool = False) -> None: - execute_path = os.getcwd() - try: - while 1: - try: - exec_from_one_connection(serversock) - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: - if debug: - import traceback - - traceback.print_exc() - else: - print_("got exception", exc) - os.chdir(execute_path) - if not loop: - break - finally: - print_("leaving socketserver execloop") - serversock.shutdown(2) - - -if __name__ == "__main__": - import sys - - hostport = sys.argv[1] if len(sys.argv) > 1 else ":8888" - from execnet.gateway_base import get_execmodel - - execmodel = get_execmodel("thread") - serversock = bind_and_listen(hostport, execmodel) - startserver(serversock, loop=True) - -elif __name__ == "__channelexec__": - chan: Channel = globals()["channel"] - execmodel = chan.gateway.execmodel - bindname = chan.receive() - assert isinstance(bindname, (str, tuple)) - sock = bind_and_listen(bindname, execmodel) - port = sock.getsockname() - chan.send(port) - startserver(sock) diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py deleted file mode 100644 index 18e375c4..00000000 --- a/src/execnet/script/socketserverservice.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -A windows service wrapper for the py.execnet socketserver. - -To use, run: - python socketserverservice.py register - net start ExecNetSocketServer -""" - -import sys -import threading - -import servicemanager -import win32event -import win32evtlogutil -import win32service -import win32serviceutil - -from execnet.gateway_base import get_execmodel - -from . import socketserver - -appname = "ExecNetSocketServer" - - -class SocketServerService(win32serviceutil.ServiceFramework): - _svc_name_ = appname - _svc_display_name_ = "%s" % appname - _svc_deps_ = ["EventLog"] - - def __init__(self, args) -> None: - # The exe-file has messages for the Event Log Viewer. - # Register the exe-file as event source. - # - # Probably it would be better if this is done at installation time, - # so that it also could be removed if the service is uninstalled. - # Unfortunately it cannot be done in the 'if __name__ == "__main__"' - # block below, because the 'frozen' exe-file does not run this code. - # - win32evtlogutil.AddSourceToRegistry( - self._svc_display_name_, servicemanager.__file__, "Application" - ) - super().__init__(args) - self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) - self.WAIT_TIME = 1000 # in milliseconds - - def SvcStop(self) -> None: - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self.hWaitStop) - - def SvcDoRun(self) -> None: - # Redirect stdout and stderr to prevent "IOError: [Errno 9] - # Bad file descriptor". Windows services don't have functional - # output streams. - sys.stdout = sys.stderr = open("nul", "w") - - # Write a 'started' event to the event log... - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STARTED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("Begin: %s" % self._svc_display_name_) - - hostport = ":8888" - print("Starting py.execnet SocketServer on %s" % hostport) - exec_model = get_execmodel("thread") - serversock = socketserver.bind_and_listen(hostport, exec_model) - thread = threading.Thread( - target=socketserver.startserver, args=(serversock,), kwargs={"loop": True} - ) - thread.setDaemon(True) - thread.start() - - # wait to be stopped or self.WAIT_TIME to pass - while True: - result = win32event.WaitForSingleObject(self.hWaitStop, self.WAIT_TIME) - if result == win32event.WAIT_OBJECT_0: - break - - # write a 'stopped' event to the event log. - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STOPPED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("End: %s" % appname) - - -if __name__ == "__main__": - # Note that this code will not be run in the 'frozen' exe-file!!! - win32serviceutil.HandleCommandLine(SocketServerService) diff --git a/src/execnet/sync.py b/src/execnet/sync.py new file mode 100644 index 00000000..e32951d8 --- /dev/null +++ b/src/execnet/sync.py @@ -0,0 +1,45 @@ +"""The blocking execnet API. + +A facade over the trio-native core in :mod:`execnet.trio`: gateways run +their protocol IO on a dedicated Trio host thread while this surface +blocks the calling thread. The top-level ``execnet.*`` names are aliases +into this module. +""" + +from ._channel import Channel +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._gateway import Gateway +from ._host import Host +from ._multi import Group +from ._multi import MultiChannel +from ._multi import default_group +from ._multi import makegateway +from ._multi import set_execmodel +from ._multi import set_profile +from ._rsync import RSync +from ._xspec import XSpec + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "Host", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "makegateway", + "set_execmodel", + "set_profile", +] diff --git a/src/execnet/trio.py b/src/execnet/trio.py new file mode 100644 index 00000000..0fc3b5df --- /dev/null +++ b/src/execnet/trio.py @@ -0,0 +1,47 @@ +"""The trio-native execnet API. + +Everything here is awaited directly inside your own ``trio.run`` — no +host thread, no blocking calls:: + + import trio + import execnet.trio + + async def main(): + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + trio.run(main) + +The error types are shared with the blocking API in :mod:`execnet.sync`. +Items you send must already be simple builtin data (plus channels); the +standalone serializer is intentionally not part of the public API -- +``execnet.can_send`` checks a value before you send it; see ``DumpError``. +""" + +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_gateway import open_gateway +from ._xspec import XSpec + +__all__ = [ + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", + "DataFormatError", + "DumpError", + "HostNotFound", + "LoadError", + "RemoteError", + "TimeoutError", + "XSpec", + "open_gateway", +] diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index 0559ed8c..9f5a2634 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -1,73 +1,13 @@ -""" -(c) 2008-2013, holger krekel +"""Deprecated alias for :mod:`execnet._xspec`. + +``XSpec`` is exported from :mod:`execnet`, :mod:`execnet.sync`, +:mod:`execnet.trio` and :mod:`execnet.aio`; use those. """ from __future__ import annotations +from ._shim import forwarder -class XSpec: - """Execution Specification: key1=value1//key2=value2 ... - - * Keys need to be unique within the specification scope - * Neither key nor value are allowed to contain "//" - * Keys are not allowed to contain "=" - * Keys are not allowed to start with underscore - * If no "=value" is given, assume a boolean True value - """ - - # XXX allow customization, for only allow specific key names - chdir: str | None = None - dont_write_bytecode: bool | None = None - execmodel: str | None = None - id: str | None = None - installvia: str | None = None - nice: str | None = None - popen: bool | None = None - python: str | None = None - socket: str | None = None - ssh: str | None = None - ssh_config: str | None = None - vagrant_ssh: str | None = None - via: str | None = None - - def __init__(self, string: str) -> None: - self._spec = string - self.env = {} - for keyvalue in string.split("//"): - i = keyvalue.find("=") - value: str | bool - if i == -1: - key, value = keyvalue, True - else: - key, value = keyvalue[:i], keyvalue[i + 1 :] - if key[0] == "_": - raise AttributeError("%r not a valid XSpec key" % key) - if key in self.__dict__: - raise ValueError(f"duplicate key: {key!r} in {string!r}") - if key.startswith("env:"): - self.env[key[4:]] = value - else: - setattr(self, key, value) - - def __getattr__(self, name: str) -> None | bool | str: - if name[0] == "_": - raise AttributeError(name) - return None - - def __repr__(self) -> str: - return f"" - - def __str__(self) -> str: - return self._spec - - def __hash__(self) -> int: - return hash(self._spec) - - def __eq__(self, other: object) -> bool: - return self._spec == getattr(other, "_spec", None) - - def __ne__(self, other: object) -> bool: - return self._spec != getattr(other, "_spec", None) +_MOVED = {"XSpec": "._xspec"} - def _samefilesystem(self) -> bool: - return self.popen is not None and self.chdir is None +__getattr__ = forwarder("xspec", _MOVED) diff --git a/testing/conftest.py b/testing/conftest.py index c75f96c7..29231d27 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -5,15 +5,15 @@ from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import pytest import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool -from execnet.gateway_base import get_execmodel +from execnet import Gateway +from execnet._execmodel import ExecModel +from execnet._execmodel import get_execmodel collect_ignore = ["build", "doc/_build"] @@ -69,6 +69,44 @@ def pytest_addoption(parser: pytest.Parser) -> None: "page on invalid addresses" ), ) + group.addoption( + "--stress", + action="store", + dest="stress", + default=None, + metavar="N", + help=( + "how hard the Hypothesis stress tests try: number of examples " + "per test (e.g. --stress=500). Without it a quick profile runs." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + # Register Hypothesis profiles scaled by --stress. The stress tests reuse + # a function-scoped gateway across examples on purpose (spawning one per + # example would dominate the runtime), and each round-trip can be slow, so + # the health checks for those are suppressed. + try: + from hypothesis import HealthCheck + from hypothesis import settings + except ImportError: + return + suppress = [HealthCheck.function_scoped_fixture, HealthCheck.too_slow] + settings.register_profile( + "execnet-quick", max_examples=15, deadline=None, suppress_health_check=suppress + ) + stress = config.getoption("stress") + if stress is not None: + settings.register_profile( + "execnet-stress", + max_examples=int(stress), + deadline=None, + suppress_health_check=suppress, + ) + settings.load_profile("execnet-stress") + else: + settings.load_profile("execnet-quick") @pytest.fixture @@ -128,10 +166,6 @@ def anypython(request: pytest.FixtureRequest) -> str: executable = getexecutable(name) if executable is None: pytest.skip(f"no {name} found") - if "execmodel" in request.fixturenames and name != "sys.executable": - backend = request.getfixturevalue("execmodel").backend - if backend not in ("thread", "main_thread_only"): - pytest.xfail(f"cannot run {backend!r} execmodel with bare {name}") return executable @@ -145,54 +179,53 @@ def group() -> Iterator[execnet.Group]: @pytest.fixture def gw( request: pytest.FixtureRequest, - execmodel: ExecModel, + profile: str, group: execnet.Group, ) -> Gateway: try: return group[request.param] except KeyError: if request.param == "popen": - gw = group.makegateway("popen//id=popen//execmodel=%s" % execmodel.backend) + gw = group.makegateway("popen//id=popen//profile=%s" % profile) elif request.param == "socket": - # if execmodel.backend != "thread": - # pytest.xfail( - # "cannot set remote non-thread execmodel for sockets") pname = "sproxy1" if pname not in group: proxygw = group.makegateway("popen//id=%s" % pname) # assert group['proxygw'].remote_status().receiving gw = group.makegateway( - f"socket//id=socket//installvia={pname}//execmodel={execmodel.backend}" + f"socket//id=socket//installvia={pname}//profile={profile}" ) # TODO(typing): Clarify this assignment. gw.proxygw = proxygw # type: ignore[attr-defined] assert pname in group elif request.param == "ssh": sshhost = request.getfixturevalue("specssh").ssh - # we don't use execmodel.backend here - # but you can set it when specifying the ssh spec + # the profile is not forced here; set it in the ssh spec instead gw = group.makegateway(f"ssh={sshhost}//id=ssh") elif request.param == "proxy": group.makegateway("popen//id=proxy-transport") gw = group.makegateway( - "popen//via=proxy-transport//id=proxy//execmodel=%s" % execmodel.backend + "popen//via=proxy-transport//id=proxy//profile=%s" % profile ) else: - assert 0, f"unknown execmodel: {request.param}" + assert 0, f"unknown gateway type: {request.param}" return gw -@pytest.fixture( - params=["thread", "main_thread_only", "eventlet", "gevent"], scope="session" -) -def execmodel(request: pytest.FixtureRequest) -> ExecModel: - if request.param not in ("thread", "main_thread_only"): - pytest.importorskip(request.param) - if request.param in ("eventlet", "gevent") and sys.platform == "win32": - pytest.xfail(request.param + " does not work on win32") - return get_execmodel(request.param) +@pytest.fixture(params=["thread"], scope="session") +def profile(request: pytest.FixtureRequest) -> str: + """The worker profile gateways in this test run are created with.""" + param: str = request.param + return param + + +@pytest.fixture(scope="session") +def execmodel(profile: str) -> ExecModel: + """The deprecated ExecModel shim for ``profile`` (pytest-xdist compat).""" + return get_execmodel(profile) @pytest.fixture -def pool(execmodel: ExecModel) -> WorkerPool: - return WorkerPool(execmodel=execmodel) +def executor() -> Iterator[ThreadPoolExecutor]: + with ThreadPoolExecutor() as tpe: + yield tpe diff --git a/testing/sshkeys/README.md b/testing/sshkeys/README.md new file mode 100644 index 00000000..d0133e2c --- /dev/null +++ b/testing/sshkeys/README.md @@ -0,0 +1,17 @@ +# Intentionally insecure test SSH keys + +These ed25519 keypairs are **committed on purpose** and are **not secret**. They +exist only so the local ssh-connect tests (`testing/test_ssh_local.py`) can run +an in-process asyncssh server that the system `ssh` client authenticates against, +without generating keys at runtime. + +- `insecure_host_ed25519[.pub]` — the test SSH **server** host key. +- `insecure_client_ed25519[.pub]` — the test **client** identity; its public key + is the server's sole authorized key. + +**Never** use these anywhere real. They grant nothing beyond a throwaway server +bound to `127.0.0.1` on an ephemeral port during a test run. + +Note: git does not preserve `0600` permissions, and OpenSSH refuses a +world-readable private key, so the tests copy the client key to a temp dir and +`chmod 0600` it before use. diff --git a/testing/sshkeys/insecure_client_ed25519 b/testing/sshkeys/insecure_client_ed25519 new file mode 100644 index 00000000..6f5650c8 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzwAAAKBzGIF1cxiB +dQAAAAtzc2gtZWQyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzw +AAAEBCIAXAZCBC6mZFORLaloyPr6HZsRkWpVxVd/vKSZpIhXoxkPC5tjkt9PGK4HGGg6ZF +lp5sdxO5EL9qEeAMMNfPAAAAHGV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1jbGllbnQB +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_client_ed25519.pub b/testing/sshkeys/insecure_client_ed25519.pub new file mode 100644 index 00000000..b4d824b9 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHoxkPC5tjkt9PGK4HGGg6ZFlp5sdxO5EL9qEeAMMNfP execnet-insecure-test-client diff --git a/testing/sshkeys/insecure_host_ed25519 b/testing/sshkeys/insecure_host_ed25519 new file mode 100644 index 00000000..0a31038e --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9RwAAAKDBS525wUud +uQAAAAtzc2gtZWQyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9Rw +AAAECXfzL6Ua9c9U+UkQn+Q7A1aKlBboBhABFALH+/ojmbMGqqgc+IMHZ5QmUZUIszZ5G1 +96W7XfzCcYsNEd3UAb1HAAAAGmV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1ob3N0AQID +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_host_ed25519.pub b/testing/sshkeys/insecure_host_ed25519.pub new file mode 100644 index 00000000..93d3cf93 --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGqqgc+IMHZ5QmUZUIszZ5G196W7XfzCcYsNEd3UAb1H execnet-insecure-test-host diff --git a/testing/test_aio.py b/testing/test_aio.py new file mode 100644 index 00000000..688643f3 --- /dev/null +++ b/testing/test_aio.py @@ -0,0 +1,181 @@ +"""The asyncio-native API: execnet.aio bridged over the Trio host.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any +from typing import TypeVar + +import pytest + +import execnet.aio + +T = TypeVar("T") + +TESTTIMEOUT = 30.0 + + +def run(main: Awaitable[T]) -> T: + return asyncio.run(asyncio.wait_for(main, TESTTIMEOUT)) + + +def test_popen_roundtrip() -> None: + async def main() -> None: + async with execnet.aio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + run(main()) + + +def test_open_gateway_iteration() -> None: + async def main() -> list[int]: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(4): channel.send(i * 2)" + ) + return [item async for item in channel] + + assert run(main()) == [0, 2, 4, 6] + + +def test_receive_timeout() -> None: + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec("channel.receive()") + with pytest.raises(channel.TimeoutError): + await channel.receive(timeout=0.05) + await channel.send(None) + + run(main()) + + +def test_remote_error() -> None: + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError(17)") + with pytest.raises(execnet.aio.RemoteError, match="ValueError"): + await channel.receive() + + run(main()) + + +def test_channel_passing_wraps_aio() -> None: + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + """ + c = channel.gateway.newchannel() + channel.send(c) + c.send(42) + """ + ) + passed = await channel.receive() + assert isinstance(passed, execnet.aio.AsyncChannel) + assert await passed.receive() == 42 + + run(main()) + + +def test_multiple_gateways_and_send_each() -> None: + async def main() -> list[Any]: + async with execnet.aio.AsyncGroup() as group: + gateways = [await group.makegateway("popen") for _ in range(2)] + channels = [ + await gw.remote_exec("channel.send(channel.receive() * 2)") + for gw in gateways + ] + for i, channel in enumerate(channels): + await channel.send(i + 1) + return [await channel.receive() for channel in channels] + + assert run(main()) == [2, 4] + + +def test_group_not_entered() -> None: + async def main() -> None: + group = execnet.aio.AsyncGroup() + with pytest.raises(RuntimeError, match="not started"): + await group.makegateway("popen") + + run(main()) + + +def test_terminate_gateway_explicitly() -> None: + async def main() -> None: + async with execnet.aio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(1)") + assert await channel.receive() == 1 + await gateway.terminate() + + run(main()) + + +def test_cancelled_receive_does_not_consume_an_item() -> None: + # The bridge cancels the host-side receive, so the item stays queued + # instead of being consumed and dropped -- an asyncio timeout around a + # receive must behave like passing timeout=. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + """ + channel.receive() + for i in range(3): + channel.send(i) + """ + ) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(channel.receive(), 0.05) + # release the worker; nothing was consumed by the cancelled wait + await channel.send("go") + assert [await channel.receive() for _ in range(3)] == [0, 1, 2] + + run(main()) + + +def test_cancelled_send_still_arrives() -> None: + # send is shielded: the caller sees CancelledError but the item is on + # the wire, rather than a frame half-written to the peer. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec("channel.send(channel.receive() * 2)") + task = asyncio.ensure_future(channel.send(21)) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await channel.receive() == 42 + + run(main()) + + +def test_group_start_and_aclose_explicitly() -> None: + # asyncio apps drive this from lifespan hooks rather than "async with" + async def main() -> None: + group = execnet.aio.AsyncGroup() + await group.start() + try: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(7)") + assert await channel.receive() == 7 + finally: + await group.aclose() + await group.aclose() # idempotent + with pytest.raises(RuntimeError, match="not started"): + await group.makegateway("popen") + + run(main()) + + +def test_groups_share_the_default_host() -> None: + async def main() -> None: + async with execnet.aio.AsyncGroup() as a, execnet.aio.AsyncGroup() as b: + assert a.host is b.host + assert a.host is execnet.aio.default_host() + + run(main()) diff --git a/testing/test_basics.py b/testing/test_basics.py index 1756ec34..4cebf7ef 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -15,13 +15,15 @@ import pytest import execnet -from execnet import gateway -from execnet import gateway_base -from execnet import gateway_io -from execnet.gateway_base import ChannelFactory -from execnet.gateway_base import ExecModel -from execnet.gateway_base import Message -from execnet.gateway_base import Popen2IO +from execnet import _boundary +from execnet import _errors +from execnet import _exec_source +from execnet import _gateway_base +from execnet import _message +from execnet import _serialize +from execnet._channel import ChannelFactory +from execnet._execmodel import ExecModel +from execnet._message import Message skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -29,37 +31,49 @@ ) +# The standalone serializer is an internal detail (execnet._serialize), +# not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = execnet.dumps(val) - val2 = execnet.loads(dumped) + dumped = _serialize.dumps(val) + val2 = _serialize.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(execnet.dumps(val)) + p.write_bytes(_serialize.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = execnet.load(m) + val2 = _serialize.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - execnet.dump(f, val) + _serialize.dump(f, val) read = BytesIO(f.getvalue()) - val2 = execnet.load(read) + val2 = _serialize.load(read) assert val == val2 +def test_serializer_not_public() -> None: + # dumps/loads/dump/load stay internal to execnet._serialize. ``dumps`` + # is still *reachable* as a temporary pytest-xdist shim, but it is not + # part of the surface -- see test_namespaces.py. + for name in ("loads", "dump", "load"): + assert not hasattr(execnet, name), name + for name in ("dumps", "loads", "dump", "load"): + assert name not in execnet.__all__, name + + def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: - bchr = gateway_base.bchr - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = execnet.dumps(42) - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) + bchr = _serialize.bchr + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(1)) + dumped = _serialize.dumps(42) + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(2)) + pytest.raises(execnet.DataFormatError, lambda: _serialize.loads(dumped)) def test_errors_on_execnet() -> None: @@ -68,81 +82,48 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def test_subprocess_interaction(anypython: str) -> None: - line = gateway_io.popen_bootstrapline - compile(line, "xyz", "exec") - args = [str(anypython), "-c", line] - popen = subprocess.Popen( - args, - bufsize=0, - universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - assert popen.stdin is not None - assert popen.stdout is not None +def standalone_protocol_source() -> str: + """The wire-protocol core as a self-contained script. - def send(line: str) -> None: - assert popen.stdin is not None - popen.stdin.write(line) - popen.stdin.flush() + ``_errors``, ``_message`` and ``_serialize`` are the layers a peer needs + to frame and serialize; between them the only runtime execnet imports are + of each other, so concatenating their sources -- dropping those imports + and keeping a single file-leading future import -- yields a script that + runs on any interpreter, which is what these checks verify. The + ``if TYPE_CHECKING:`` imports are left in place; they never execute. + """ + lines = ["from __future__ import annotations\n"] + for module in (_errors, _message, _serialize): + for line in inspect.getsource(module).splitlines(keepends=True): + if line.startswith(("from __future__ import annotations", "from ._")): + continue + lines.append(line) + return "".join(lines) - def receive() -> str: - assert popen.stdout is not None - return popen.stdout.readline() - try: - source = inspect.getsource(read_write_loop) + "read_write_loop()" - send(repr(source) + "\n") - s = receive() - assert s == "ok\n" - send("hello\n") - s = receive() - assert s == "received: hello\n" - send("world\n") - s = receive() - assert s == "received: world\n" - send("\n") # terminate loop - finally: - popen.stdin.close() - popen.stdout.close() - popen.wait() +IO_MESSAGE_EXTRA_SOURCE = """ +from io import BytesIO +class BufIO: + def __init__(self): + self.buf = BytesIO() -def read_write_loop() -> None: - sys.stdout.write("ok\n") - sys.stdout.flush() - while 1: - try: - line = sys.stdin.readline() - if not line.strip(): - break - sys.stdout.write("received: %s" % line) - sys.stdout.flush() - except (OSError, EOFError): - break + def write(self, data): + self.buf.write(data) + def read(self, numbytes): + data = self.buf.read(numbytes) + if len(data) < numbytes: + raise EOFError("expected %d bytes" % numbytes) + return data -IO_MESSAGE_EXTRA_SOURCE = """ -import sys -backend = sys.argv[1] -from io import BytesIO -import tempfile -temp_out = BytesIO() -temp_in = BytesIO() -io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) for i, handler in enumerate(Message._types): print ("checking", i, handler) for data in "hello", "hello".encode('ascii'): + io = BufIO() msg1 = Message(i, i, dumps(data)) msg1.to_io(io) - x = io.outfile.getvalue() - io.outfile.truncate(0) - io.outfile.seek(0) - io.infile.seek(0) - io.infile.write(x) - io.infile.seek(0) + io.buf.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) @@ -177,63 +158,15 @@ def checker(anypython: str, tmp_path: Path) -> Checker: return Checker(python=anypython, path=tmp_path) -def test_io_message(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend - ) - print(out.stdout) - assert "all passed" in out.stdout - - -def test_popen_io(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) - + f""" -io = init_popen_io(get_execmodel({execmodel.backend!r})) -io.write(b"hello") -s = io.read(1) -assert s == b"x" -""", - input="x", - ) - print(out.stderr) - assert "hello" in out.stdout - - -def test_popen_io_readloop(execmodel: ExecModel) -> None: - sio = BytesIO(b"test") - io = Popen2IO(sio, sio, execmodel) - real_read = io._read - - def newread(numbytes: int) -> bytes: - if numbytes > 1: - numbytes = numbytes - 1 - return real_read(numbytes) # type: ignore[no-any-return] - - io._read = newread - result = io.read(3) - assert result == b"tes" - - -def test_rinfo_source(checker: Checker) -> None: - out = checker.run_check( - f""" -class Channel: - def send(self, data): - assert eval(repr(data), {{}}) == data -channel = Channel() -{inspect.getsource(gateway.rinfo_source)} -print ('all passed') -""" - ) - +def test_io_message(checker: Checker) -> None: + out = checker.run_check(standalone_protocol_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - inspect.getsource(gateway_base) + standalone_protocol_source() + """ class Arg(Exception): pass @@ -252,19 +185,20 @@ class Arg(Exception): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull( - execmodel: ExecModel, capfd: pytest.CaptureFixture[str] -) -> None: - # Backup and restore stdin state, and rely on capfd to handle - # this for stdout and stderr. +def test_stdio_disposition_devnull(capfd: pytest.CaptureFixture[str]) -> None: + # apply_stdio(devnull) points fd 0/1 at the null device: writes and + # reads on them must go nowhere. Back up and restore the real fds. + from execnet import _trio_worker + orig_stdin = sys.stdin - orig_stdin_fd = os.dup(0) + orig_stdout = sys.stdout + orig_fd0 = os.dup(0) + orig_fd1 = os.dup(1) try: - # The returned Popen2IO instance can be garbage collected - # prematurely since we don't hold a reference here, but we - # tolerate this because it is intended to leave behind a - # sane state afterwards. - gateway_base.init_popen_io(execmodel) + read_fd, write_fd = _trio_worker._dup_protocol_fds() + os.close(read_fd) + os.close(write_fd) + _trio_worker.apply_stdio(stdin="devnull", stdout="devnull") os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() @@ -272,8 +206,31 @@ def test_stdouterrin_setnull( assert not err finally: sys.stdin = orig_stdin - os.dup2(orig_stdin_fd, 0) - os.close(orig_stdin_fd) + sys.stdout = orig_stdout + os.dup2(orig_fd0, 0) + os.dup2(orig_fd1, 1) + os.close(orig_fd0) + os.close(orig_fd1) + + +@pytest.mark.skipif("not hasattr(os, 'dup')") +def test_stdio_disposition_stdout_to_stderr(capfd: pytest.CaptureFixture[str]) -> None: + # The stdio transport's default: remote output lands on stderr rather + # than the null device, so it stays visible without touching the wire. + from execnet import _trio_worker + + orig_stdout = sys.stdout + orig_fd1 = os.dup(1) + try: + _trio_worker.apply_stdio(stdout="stderr") + os.write(1, b"to-stderr") + out, err = capfd.readouterr() + assert not out + assert "to-stderr" in err + finally: + sys.stdout = orig_stdout + os.dup2(orig_fd1, 1) + os.close(orig_fd1) class PseudoChannel: @@ -296,7 +253,7 @@ def close(self, errortext: str | None = None) -> None: def test_exectask(execmodel: ExecModel) -> None: io = BytesIO() io.execmodel = execmodel # type: ignore[attr-defined] - gw = gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] + gw = _gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] ch = PseudoChannel() gw.executetask((ch, ("raise ValueError()", None, {}))) # type: ignore[arg-type] assert "ValueError" in str(ch._closed[0]) @@ -318,16 +275,112 @@ def test_wire_protocol(self) -> None: assert isinstance(repr(msg), str) +class TestFrameDecoder: + def _messages(self) -> list[Message]: + return [ + Message(Message.CHANNEL_DATA, 1, b"x" * 20), + Message(Message.STATUS, 42, b""), + Message(Message.CHANNEL_DATA, 7, b"y"), + ] + + def test_single_feed_yields_all(self) -> None: + decoder = _message.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got = list(decoder.feed(blob)) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) + def test_adversarial_chunk_splits(self, chunksize: int) -> None: + decoder = _message.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got: list[Message] = [] + for start in range(0, len(blob), chunksize): + got.extend(decoder.feed(blob[start : start + chunksize])) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + def test_close_mid_frame_raises(self) -> None: + decoder = _message.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() + assert list(decoder.feed(blob[:-2])) == [] + with pytest.raises(EOFError, match="mid-frame"): + decoder.close() + + def test_feed_buffers_even_when_not_iterated(self) -> None: + decoder = _message.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() + decoder.feed(blob[:4]) # result deliberately not iterated + (msg,) = decoder.feed(blob[4:]) + assert (msg.msgcode, msg.channelid, msg.data) == ( + Message.CHANNEL_DATA, + 5, + b"data", + ) + + def test_memory_stream_roundtrip(self) -> None: + """Protocol-level: frames sent over a trio memory stream pair arrive + intact through the receive_some + FrameDecoder loop.""" + import trio + import trio.testing + + messages = self._messages() + + async def main() -> list[Message]: + ours, theirs = trio.testing.memory_stream_pair() + received: list[Message] = [] + + async def sender() -> None: + for m in messages: + await theirs.send_all(m.pack()) + await theirs.send_eof() + + async def receiver() -> None: + decoder = _message.FrameDecoder() + while True: + data = await ours.receive_some(4096) + if not data: + decoder.close() + break + received.extend(decoder.feed(data)) + + async with trio.open_nursery() as nursery: + nursery.start_soon(sender) + nursery.start_soon(receiver) + return received + + received = trio.run(main) + assert [(m.msgcode, m.channelid, m.data) for m in received] == [ + (m.msgcode, m.channelid, m.data) for m in messages + ] + + class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: + _trio_session = None + _new_wakener = staticmethod(_boundary.ThreadWakener) + + def _check_event_loop(self, what: str) -> None: + pass + def _trace(self, *args) -> None: pass def _send(self, *k) -> None: pass + def _bind_channel(self, channel) -> None: + pass + + def _release_channel(self, id) -> None: + pass + FakeGateway.execmodel = execmodel # type: ignore[attr-defined] return ChannelFactory(FakeGateway()) # type: ignore[arg-type] @@ -355,13 +408,13 @@ def test_channel_makefile_incompatmode(self, fac) -> None: class TestSourceOfFunction: def test_lambda_unsupported(self) -> None: - pytest.raises(ValueError, gateway._source_of_function, lambda: 1) + pytest.raises(ValueError, _exec_source._source_of_function, lambda: 1) def test_wrong_prototype_fails(self) -> None: def prototype(wrong) -> None: pass - pytest.raises(ValueError, gateway._source_of_function, prototype) + pytest.raises(ValueError, _exec_source._source_of_function, prototype) def test_function_without_known_source_fails(self) -> None: # this one won't be able to find the source @@ -369,7 +422,7 @@ def test_function_without_known_source_fails(self) -> None: exec("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) with pytest.raises(ValueError): - gateway._source_of_function(mess["fail"]) + _exec_source._source_of_function(mess["fail"]) def test_function_with_closure_fails(self) -> None: mess: dict[str, Any] = {} @@ -378,13 +431,13 @@ def closure(channel: object) -> None: print(mess) with pytest.raises(ValueError): - gateway._source_of_function(closure) + _exec_source._source_of_function(closure) def test_source_of_nested_function(self) -> None: def working(channel: object) -> None: pass - send_source = gateway._source_of_function(working).lstrip("\r\n") + send_source = _exec_source._source_of_function(working).lstrip("\r\n") expected = "def working(channel: object) -> None:\n pass\n" assert send_source == expected @@ -393,7 +446,7 @@ class TestGlobalFinder: def check(self, func) -> list[str]: src = textwrap.dedent(inspect.getsource(func)) code = func.__code__ - return gateway._find_non_builtin_globals(src, code) + return _exec_source._find_non_builtin_globals(src, code) def test_local(self) -> None: def f(a, b, c): @@ -420,7 +473,7 @@ def test_function_with_global_fails(self) -> None: def func(channel) -> None: sys - pytest.raises(ValueError, gateway._source_of_function, func) + pytest.raises(ValueError, _exec_source._source_of_function, func) def test_method_call(self) -> None: # method names are reason @@ -433,7 +486,7 @@ def f(channel): @skip_win_pypy def test_remote_exec_function_with_kwargs( - anypython: str, makegateway: Callable[[str], gateway.Gateway] + anypython: str, makegateway: Callable[[str], execnet.Gateway] ) -> None: def func(channel, data) -> None: channel.send(data) @@ -446,17 +499,17 @@ def func(channel, data) -> None: assert result == 1 -def test_remote_exc__no_kwargs(makegateway: Callable[[], gateway.Gateway]) -> None: +def test_remote_exc__no_kwargs(makegateway: Callable[[], execnet.Gateway]) -> None: gw = makegateway() with pytest.raises(TypeError): - gw.remote_exec(gateway_base, kwarg=1) + gw.remote_exec(_message, kwarg=1) with pytest.raises(TypeError): gw.remote_exec("pass", kwarg=1) @skip_win_pypy def test_remote_exec_inspect_stack( - makegateway: Callable[[], gateway.Gateway], + makegateway: Callable[[], execnet.Gateway], ) -> None: gw = makegateway() ch = gw.remote_exec( diff --git a/testing/test_boundary.py b/testing/test_boundary.py new file mode 100644 index 00000000..ce2a5302 --- /dev/null +++ b/testing/test_boundary.py @@ -0,0 +1,82 @@ +"""The boundary kit: Mailbox and OneShot on the thread wakener.""" + +from __future__ import annotations + +import queue +import threading + +import pytest + +from execnet import TimeoutError as ExecnetTimeoutError +from execnet._boundary import Mailbox +from execnet._boundary import OneShot + + +class TestMailbox: + def test_put_get_fifo(self) -> None: + box: Mailbox[int] = Mailbox() + for n in range(3): + box.put(n) + assert [box.get(), box.get(), box.get()] == [0, 1, 2] + + def test_get_nowait_empty(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(queue.Empty): + box.get_nowait() + + def test_get_timeout(self) -> None: + # the carriers speak execnet's TimeoutError, so Channel.receive + # does not have to translate the builtin one + box: Mailbox[int] = Mailbox() + with pytest.raises(ExecnetTimeoutError): + box.get(timeout=0.01) + + def test_get_blocks_until_put_from_other_thread(self) -> None: + box: Mailbox[str] = Mailbox() + threading.Timer(0.05, box.put, args=["item"]).start() + assert box.get(timeout=5.0) == "item" + + def test_get_after_stale_wakeup(self) -> None: + # A consumed notify leaves the wakener set; the drain pattern must + # still block (and then receive) rather than spin or miss items. + box: Mailbox[int] = Mailbox() + box.put(1) + assert box.get() == 1 + threading.Timer(0.05, box.put, args=[2]).start() + assert box.get(timeout=5.0) == 2 + + +class TestOneShot: + def test_set_then_wait(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(42) + assert shot.is_set() + assert shot.wait() == 42 + # a resolved OneShot stays readable + assert shot.wait(timeout=0.01) == 42 + + def test_wait_timeout(self) -> None: + shot: OneShot[int] = OneShot() + with pytest.raises(ExecnetTimeoutError): + shot.wait(timeout=0.01) + assert not shot.is_set() + + def test_set_error_reraises(self) -> None: + shot: OneShot[None] = OneShot() + shot.set_error(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + shot.wait() + + def test_wait_blocks_until_set_from_other_thread(self) -> None: + shot: OneShot[str] = OneShot() + threading.Timer(0.05, shot.set, args=["done"]).start() + assert shot.wait(timeout=5.0) == "done" + + def test_single_resolution_enforced(self) -> None: + # a real error, not an assert: it must survive python -O + shot: OneShot[int] = OneShot() + shot.set(1) + with pytest.raises(RuntimeError, match="already resolved"): + shot.set(2) + with pytest.raises(RuntimeError, match="already resolved"): + shot.set_error(RuntimeError("boom")) diff --git a/testing/test_channel.py b/testing/test_channel.py index d4712277..8bb292a0 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -4,12 +4,13 @@ from __future__ import annotations +import queue import time import pytest -from execnet.gateway import Gateway -from execnet.gateway_base import Channel +from execnet import Gateway +from execnet._channel import Channel needs_early_gc = pytest.mark.skipif("not hasattr(sys, 'getrefcount')") needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") @@ -214,8 +215,6 @@ def test_channel_callback_stays_active(self, gw: Gateway) -> None: def check_channel_callback_stays_active( self, gw: Gateway, earlyfree: bool = True ) -> Channel | None: - if gw.spec.execmodel == "gevent": - pytest.xfail("investigate gevent failure") # with 'earlyfree==True', this tests the "sendonly" channel state. l: list[int] = [] channel = gw.remote_exec( @@ -272,14 +271,14 @@ def test_channel_endmarker_callback(self, gw: Gateway) -> None: assert l[3] == 999 def test_channel_endmarker_callback_error(self, gw: Gateway) -> None: - q = gw.execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" raise ValueError() """ ) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert err @@ -306,6 +305,50 @@ def f(item): channel.send(1) channel.waitclose() + @needs_early_gc + def test_callback_channel_collected_after_close(self, gw: Gateway) -> None: + # A callback channel is kept alive only by its consumer task: while it + # is consuming it survives with no user reference, and once the stream + # closes and the callback has run the object becomes collectable. + import gc + import weakref + + received: list[int] = [] + channel = gw.remote_exec("channel.send(1); channel.send(2)") + channel.setcallback(received.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.05) + assert received == [1, 2] + assert ref() is None # consumer finished -> no strong refs -> reclaimed + + def test_callbacks_run_off_the_loop_thread(self, gw: Gateway) -> None: + # A slow callback on one channel must not block delivery to another: + # callbacks run in threadpool threads, not inline on the loop. + import threading + + release = threading.Event() + fast_ran = threading.Event() + + def slow(item: object) -> None: + release.wait(TESTTIMEOUT) + + chan_slow = gw.remote_exec("channel.send(1); channel.receive()") + chan_fast = gw.remote_exec("channel.send(1)") + chan_slow.setcallback(slow) + chan_fast.setcallback(lambda item: fast_ran.set()) + + # the fast callback fires even while the slow one is still blocking + assert fast_ran.wait(TESTTIMEOUT) + release.set() + chan_slow.send(0) # let the remote finish and close + chan_slow.waitclose(TESTTIMEOUT) + chan_fast.waitclose(TESTTIMEOUT) + class TestChannelFile: def test_channel_file_write(self, gw: Gateway) -> None: diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py new file mode 100644 index 00000000..f5637efb --- /dev/null +++ b/testing/test_channel_stress.py @@ -0,0 +1,121 @@ +"""Hypothesis stress tests for the channel callback (consumer-task) machinery. + +These hammer ``setcallback`` -- the loop-task-plus-threadpool consumer -- with +randomised traffic to check the invariants that must hold no matter the timing: + +* every item reaches the callback, exactly once and in send order; +* many callback channels run concurrently without cross-talk; +* switching to a callback after some ``receive()`` calls loses nothing; +* the endmarker is always delivered last; +* a callback channel with no user reference is kept alive by its consumer. + +Use ``--stress=N`` to raise the number of examples per test (default: a quick +profile registered in ``conftest.pytest_configure``). +""" + +from __future__ import annotations + +import gc +import weakref + +import pytest + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from execnet import Gateway # noqa: E402 + +# High --stress levels replay one test many times; lift the per-test timeout +# well above the default so that only a real hang (bounded by TESTTIMEOUT on +# every blocking call below) fails, not sheer example count. +pytestmark = pytest.mark.timeout(600) + +TESTTIMEOUT = 10.0 + +# Varied payloads: unbounded ints exercise both the short and long serializer +# paths (and their boundaries), on top of the callback machinery itself. +payload_strategy = st.one_of( + st.integers(), + st.text(max_size=20), + st.booleans(), + st.none(), +) +items_strategy = st.lists(payload_strategy, max_size=40) + + +def _echo(channel, items): + """Remote: send every item, in order, then let the channel close.""" + for item in items: + channel.send(item) + + +class TestCallbackStress: + # popen only: fast to spawn and the callback path is transport-independent + gwtype = "popen" + + @given(data=items_strategy) + def test_callback_receives_all_in_order(self, gw: Gateway, data: list[int]) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + channel.waitclose(TESTTIMEOUT) + assert collected == data + + @given(batches=st.lists(items_strategy, min_size=1, max_size=6)) + def test_many_channels_stay_ordered_and_isolated( + self, gw: Gateway, batches: list[list[int]] + ) -> None: + results: list[list[int]] = [[] for _ in batches] + channels = [] + for index, items in enumerate(batches): + channel = gw.remote_exec(_echo, items=items) + channel.setcallback(results[index].append) + channels.append(channel) + for channel in channels: + channel.waitclose(TESTTIMEOUT) + assert results == batches + + @given(data=st.data()) + def test_receive_then_switch_loses_nothing( + self, gw: Gateway, data: st.DataObject + ) -> None: + items = data.draw(items_strategy) + split = data.draw(st.integers(min_value=0, max_value=len(items))) + channel = gw.remote_exec(_echo, items=items) + first = [channel.receive(TESTTIMEOUT) for _ in range(split)] + rest: list[int] = [] + channel.setcallback(rest.append) + channel.waitclose(TESTTIMEOUT) + assert first + rest == items + + @given(data=items_strategy) + def test_endmarker_is_always_last(self, gw: Gateway, data: list[int]) -> None: + endmarker = object() + collected: list[object] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append, endmarker=endmarker) + channel.waitclose(TESTTIMEOUT) + assert collected == [*data, endmarker] + + @pytest.mark.skipif( + "not hasattr(sys, 'getrefcount')", reason="needs refcount GC semantics" + ) + @given(data=st.lists(payload_strategy, min_size=1, max_size=40)) + def test_callback_channel_kept_alive_then_collected( + self, gw: Gateway, data: list[object] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + import time + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.02) + assert collected == data + assert ref() is None # consumer finished -> reclaimed diff --git a/testing/test_cli.py b/testing/test_cli.py new file mode 100644 index 00000000..ba9d279a --- /dev/null +++ b/testing/test_cli.py @@ -0,0 +1,394 @@ +"""The ``execnet`` command line, and the transports it exposes. + +``execnet worker`` is the launch contract between a coordinator and the +process it starts. These tests drive it the way a coordinator does -- +including the transports that keep the protocol off the worker's stdio. +""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +from typing import Any + +import pytest + +import execnet +from execnet import _cli +from execnet import _provision + +posix_only = pytest.mark.skipif( + sys.platform.startswith("win"), reason="needs POSIX fd passing / unix sockets" +) + +TESTTIMEOUT = 30.0 + + +def worker_config(**overrides: object) -> str: + config: dict[str, object] = { + "id": "cli-test-worker", + "profile": "thread", + "execmodel": "thread", + "wait": "thread", + "coordinator_version": execnet.__version__, + } + config.update(overrides) + return json.dumps(config) + + +class TestInfo: + def test_info_reports_what_a_coordinator_needs(self) -> None: + out = subprocess.run( + [sys.executable, "-m", "execnet", "info"], + capture_output=True, + text=True, + check=True, + ) + info = json.loads(out.stdout) + assert info["execnet"] == execnet.__version__ + assert info["trio"] is not None + assert info["executable"] + assert "stdio" in info["protocols"] + + def test_info_matches_the_in_process_view(self) -> None: + assert _cli.interpreter_info()["execnet"] == execnet.__version__ + + def test_probe_uses_info(self) -> None: + _provision.target_info.cache_clear() + info = _provision.target_info(sys.executable) + assert info is not None + assert info["execnet"] == execnet.__version__ + assert _provision.target_has_execnet(sys.executable) + + def test_probe_rejects_an_interpreter_without_execnet(self, tmp_path) -> None: + # a python that cannot import execnet fails the probe, which is what + # sends it down the uv-provisioning path + _provision.target_info.cache_clear() + fake = tmp_path / "fake-python" + fake.write_text("#!/bin/sh\nexit 1\n") + fake.chmod(0o755) + assert _provision.target_info(str(fake)) is None + assert not _provision.target_has_execnet(str(fake)) + + +class TestArgumentGrammar: + def test_protocol_flags_are_mutually_exclusive(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args( + ["worker", "--protocol-fd", "3", "--protocol-connect", "unix:/x"] + ) + + def test_config_sources_are_mutually_exclusive(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args( + ["worker", "--config", "{}", "--config-fd", "0"] + ) + + @pytest.mark.parametrize(("value", "expected"), [("3", (3,)), ("4,5", (4, 5))]) + def test_protocol_fd_accepts_one_fd_or_a_pair( + self, value: str, expected: tuple[int, ...] + ) -> None: + ns = _cli._build_parser().parse_args(["worker", "--protocol-fd", value]) + assert ns.protocol_fd == expected + + def test_protocol_fd_rejects_nonsense(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args(["worker", "--protocol-fd", "a,b,c"]) + + @pytest.mark.parametrize( + ("address", "expected"), + [ + ("unix:/tmp/x.sock", ("unix", "/tmp/x.sock")), + ("localhost:8888", ("tcp", ("localhost", 8888))), + (":8888", ("tcp", ("localhost", 8888))), + ], + ) + def test_parse_address(self, address: str, expected: tuple[str, Any]) -> None: + from execnet._trio_worker import parse_address + + assert parse_address(address) == expected + + def test_parse_address_rejects_a_bare_path(self) -> None: + from execnet._trio_worker import parse_address + + with pytest.raises(ValueError, match="unix:/path or host:port"): + parse_address("/tmp/x.sock") + + +@posix_only +class TestProtocolFd: + """A worker serving over an inherited socket, driven by hand.""" + + def test_socketpair_roundtrip(self) -> None: + ours, theirs = socket.socketpair() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(theirs.fileno()), + "--config", + worker_config(), + ], + pass_fds=(theirs.fileno(),), + ) + theirs.close() + try: + assert ours.recv(1) == b"1" # the ready handshake + finally: + ours.close() + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + + def test_a_plain_pipe_fd_is_rejected(self) -> None: + # one fd has to be bidirectional; a pipe end needs the read,write form + read_fd, write_fd = os.pipe() + try: + out = subprocess.run( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(read_fd), + "--config", + worker_config(), + ], + pass_fds=(read_fd,), + capture_output=True, + text=True, + timeout=TESTTIMEOUT, + check=False, + ) + finally: + os.close(read_fd) + os.close(write_fd) + assert out.returncode != 0 + assert "not a socket" in out.stderr + + +class TestConfigSources: + @posix_only + def test_config_fd_keeps_it_out_of_argv(self) -> None: + ours, theirs = socket.socketpair() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(theirs.fileno()), + "--config-fd", + "0", + ], + pass_fds=(theirs.fileno(),), + stdin=subprocess.PIPE, + ) + theirs.close() + try: + assert proc.stdin is not None + proc.stdin.write(worker_config().encode()) + proc.stdin.close() + assert ours.recv(1) == b"1" + finally: + ours.close() + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + + def test_config_file(self, tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text(worker_config()) + ns = _cli._build_parser().parse_args(["worker", "--config-file", str(path)]) + assert _cli._load_config(ns)["id"] == "cli-test-worker" + + def test_no_config_source_is_an_error(self) -> None: + ns = _cli._build_parser().parse_args(["worker"]) + with pytest.raises(SystemExit, match="--config"): + _cli._load_config(ns) + + +class TestTransportSelection: + def test_defaults_per_platform(self) -> None: + spec = execnet.XSpec("popen") + expected = "stdio" if sys.platform.startswith("win") else "socket" + assert _provision.resolve_transport(spec) == expected + + def test_explicit_wins(self) -> None: + assert _provision.resolve_transport( + execnet.XSpec("popen//transport=stdio") + ) == ("stdio") + + def test_unknown_is_rejected(self) -> None: + with pytest.raises(ValueError, match="unknown transport"): + _provision.resolve_transport( + execnet.XSpec("popen//transport=carrier-pigeon") + ) + + @posix_only + def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec( + "import sys; channel.send(sys.argv)", + ) + argv = channel.receive(TESTTIMEOUT) + assert "--protocol-fd" in argv + finally: + group.terminate(timeout=5.0) + + def test_stdio_transport_still_works(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//transport=stdio") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + + +class TestWorkerStdio: + """Whose stdio is it? The code the worker runs, unless told otherwise.""" + + @posix_only + def test_socket_transport_inherits_stdio(self, capfd) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + gateway.remote_exec("print('INHERITED-STDOUT')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, _ = capfd.readouterr() + assert "INHERITED-STDOUT" in out + + def test_stdio_transport_folds_stdout_onto_stderr(self, capfd) -> None: + # the protocol owns fd 1 here, so remote output cannot go there -- + # but it lands on stderr rather than being discarded + group = execnet.Group() + try: + gateway = group.makegateway("popen//transport=stdio") + gateway.remote_exec("print('FOLDED-ONTO-STDERR')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, err = capfd.readouterr() + assert "FOLDED-ONTO-STDERR" not in out + assert "FOLDED-ONTO-STDERR" in err + + @posix_only + def test_stdin_can_be_sent_to_devnull(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//stdin=devnull") + channel = gateway.remote_exec( + "import os; channel.send(os.readlink('/proc/self/fd/0'))" + ) + assert channel.receive(TESTTIMEOUT) == "/dev/null" + finally: + group.terminate(timeout=5.0) + + @posix_only + def test_stdout_can_be_silenced(self, capfd) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//stdout=devnull") + gateway.remote_exec("print('SILENCED')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, err = capfd.readouterr() + assert "SILENCED" not in out + assert "SILENCED" not in err + + +@posix_only +class TestListenTransport: + def test_worker_listens_and_reports_its_address(self) -> None: + directory = tempfile.mkdtemp(prefix="execnet-cli-test-") + path = os.path.join(directory, "gw.sock") + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-listen", + f"unix:{path}", + "--config", + worker_config(), + ], + stdout=subprocess.PIPE, + ) + try: + assert proc.stdout is not None + announced = json.loads(proc.stdout.readline()) + assert announced == {"listening": path} + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(TESTTIMEOUT) + client.connect(path) + try: + assert client.recv(1) == b"1" + finally: + client.close() + finally: + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + shutil.rmtree(directory, ignore_errors=True) + + +class TestServerCommand: + def test_server_is_the_socketserver(self) -> None: + parser = _cli._build_parser() + ns = parser.parse_args(["server", "127.0.0.1:0", "--once"]) + assert ns.hostport == "127.0.0.1:0" + assert ns.once is True + + def test_socketserver_alias_warns(self, monkeypatch: pytest.MonkeyPatch) -> None: + called: list[list[str]] = [] + monkeypatch.setattr(_cli, "main", called.append) + with pytest.warns(DeprecationWarning, match="execnet server"): + _cli.socketserver_main([":0", "--once"]) + assert called == [["server", ":0", "--once"]] + + +class TestRemoteCommand: + """What ends up on the remote host's command line.""" + + def test_config_is_not_in_the_remote_argv(self) -> None: + # env: values are secrets often enough; the remote argv is readable + # by every user on that host via ps + spec = execnet.XSpec("ssh=host//id=gw0//env:TOKEN=s3cr3t") + spec.profile = "thread" + command = _provision.ssh_remote_command( + spec, "--protocol-connect", "unix:/tmp/x.sock", config_on_stdin=True + ) + assert "s3cr3t" not in command + assert "--config-fd 0" in command + + def test_launch_command_frames_nothing_in_band(self) -> None: + spec = execnet.XSpec("ssh=host//id=gw0") + spec.profile = "thread" + command = _provision.ssh_remote_command(spec) + # the wheel travels on its own connection now + assert "head -c" not in command + assert "mktemp -d" not in command + + def test_dialback_argv_forwards_a_unix_socket(self) -> None: + from execnet import _trio_gateway + + spec = execnet.XSpec("ssh=host//id=gw0") + spec.profile = "thread" + argv = _trio_gateway._ssh_argv( + spec, "worker-cmd", forward=("/tmp/remote.sock", "/tmp/local.sock") + ) + assert "-R" in argv + assert argv[argv.index("-R") + 1] == "/tmp/remote.sock:/tmp/local.sock" + # a stale remote socket must not block the bind + assert "StreamLocalBindUnlink=yes" in argv diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 5343e7d5..ed06be4e 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -1,8 +1,8 @@ -from execnet import gateway_base +from execnet import _serialize def test_opcodes() -> None: - data = vars(gateway_base.opcode) + data = vars(_serialize.opcode) computed = {k: v for k, v in data.items() if "__" not in k} assert computed == { "BUILDTUPLE": b"@", @@ -18,13 +18,13 @@ def test_opcodes() -> None: "NEWDICT": b"J", "NEWLIST": b"K", "NONE": b"L", - "PY2STRING": b"M", - "PY3STRING": b"N", + # b"M" (py2 str) and b"S" (py2 unicode) are retired along with + # Python2 support -- the bytes stay unused rather than reassigned. + "STRING": b"N", "SET": b"O", "SETITEM": b"P", "STOP": b"Q", "TRUE": b"R", - "UNICODE": b"S", # added in 1.4 # causes a regression since it was ordered in # between CHANNEL and FALSE as "C" moving the other items diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py new file mode 100644 index 00000000..8fd121d8 --- /dev/null +++ b/testing/test_execmodel_trio.py @@ -0,0 +1,183 @@ +"""The pure-async worker profile (profile=trio). + +One single thread in the worker: the trio loop owns the main thread and +exec'd sources run as tasks on it, talking through AsyncChannels. Sync +sources are rejected before they can starve the loop. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest +import trio as trio_lib + +import execnet +import execnet.trio +from execnet import Gateway + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: + return makegateway("popen//profile=trio") + + +class TestSyncCoordinator: + def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec("await channel.send(await channel.receive() + 1)") + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_async_function_source(self, trio_gw: Gateway) -> None: + async def source(channel, delta) -> None: + value = await channel.receive() + await channel.send(value + delta) + + channel = trio_gw.remote_exec(source, delta=5) + channel.send(37) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_iteration_and_eof(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + for i in range(3): + await channel.send(i * 2) + """ + ) + assert list(channel) == [0, 2, 4] + + def test_single_thread_on_main(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + import threading + await channel.send( + ( + threading.active_count(), + threading.current_thread() is threading.main_thread(), + ) + ) + """ + ) + active, on_main = channel.receive(TESTTIMEOUT) + assert on_main + assert active == 1 + + def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: + # two execs run as tasks on one loop: the first parks in receive + # while the second completes -- no threads involved. + blocked = trio_gw.remote_exec("await channel.send(await channel.receive())") + side = trio_gw.remote_exec("await channel.send('side')") + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_sync_source_rejected(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec("x = 40 + 2") + with pytest.raises(channel.RemoteError, match="sync source"): + channel.receive(TESTTIMEOUT) + + def test_sync_function_rejected(self, trio_gw: Gateway) -> None: + def source(channel) -> None: + pass + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="must be async"): + channel.receive(TESTTIMEOUT) + + def test_remote_error_traceback(self, trio_gw: Gateway) -> None: + async def source(channel) -> None: + raise ValueError(17) + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="ValueError"): + channel.receive(TESTTIMEOUT) + + def test_status_and_rinfo(self, trio_gw: Gateway) -> None: + status = trio_gw.remote_status() + assert status.profile == "trio" + # legacy STATUS key, kept for pytest-xdist + assert status.execmodel == "trio" + rinfo = trio_gw._rinfo() + assert rinfo.pid + assert rinfo.version_info + + +def test_unknown_profile_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown profile"): + makegateway("popen//profile=nope") + # the pre-3.0 spelling routes to the same validation + with pytest.raises(ValueError, match="unknown profile"): + makegateway("popen//execmodel=nope") + + +def test_execmodel_is_an_accepted_alias_for_profile( + makegateway: Callable[[str], Gateway], +) -> None: + gw = makegateway("popen//execmodel=trio") + assert gw.spec.profile == "trio" + assert gw.spec.execmodel == "trio" + with pytest.raises(ValueError, match="duplicate key"): + execnet.XSpec("popen//execmodel=trio//profile=thread") + + +def test_trio_native_coordinator() -> None: + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen//profile=trio") + channel = await gateway.remote_exec( + "await channel.send(await channel.receive() * 2)" + ) + await channel.send(21) + with trio_lib.fail_after(TESTTIMEOUT): + assert await channel.receive() == 42 + + trio_lib.run(main) + + +def test_async_coordinator_defaults_to_a_thread_worker() -> None: + # An async coordinator does not imply an async worker: the worker's + # shape is its own choice, so the default stays the thread profile. + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec( + "import threading;" + " channel.send(threading.current_thread() is" + " threading.main_thread())" + ) + with trio_lib.fail_after(TESTTIMEOUT): + # a sync source at all proves this is not the trio profile, + # which rejects them + assert await channel.receive() is True + + trio_lib.run(main) + + +def test_makegateway_does_not_rewrite_the_callers_profile( + makegateway: Callable[[str], Gateway], +) -> None: + # pytest-xdist reuses one XSpec across gateways and re-reads + # spec.execmodel to decide whether it still needs its + # "execmodel=main_thread_only//" prefix. Normalizing the value it set + # made the second use build a spec with a duplicate key, which broke + # crashed-worker replacement. Filling in a *missing* value is fine; + # rewriting one the caller set is not. + spec = execnet.XSpec("execmodel=main_thread_only//popen") + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway(spec) # type: ignore[arg-type] + assert spec.execmodel == "main_thread_only" + assert spec.profile == "main_thread_only" + # ... while the worker still gets a profile it has a strategy for + assert gw.remote_status().profile == "thread" + + +def test_makegateway_fills_in_a_missing_profile() -> None: + group = execnet.Group() + try: + spec = execnet.XSpec("popen") + group.makegateway(spec) + assert spec.profile == "thread" + finally: + group.terminate(timeout=5.0) diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 634f237a..ac845b38 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -9,15 +9,15 @@ import shutil import signal import sys +import time from collections.abc import Callable from textwrap import dedent import pytest import execnet -from execnet import gateway_base -from execnet import gateway_io -from execnet.gateway import Gateway +from execnet import Gateway +from execnet import _trace TESTTIMEOUT = 10.0 # seconds needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") @@ -169,7 +169,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -178,7 +178,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest.run_me) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -286,6 +286,50 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test_hybrid_primary_then_overflow( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # classic thread-model shape: the first exec claims the true main + # thread; while it is busy, further execs overflow to worker + # threads; once released the main thread is claimable again. + gw = makegateway("popen//execmodel=thread") + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = gw.remote_exec(report) + assert first.receive(TESTTIMEOUT) is True + second = gw.remote_exec(report) + assert second.receive(TESTTIMEOUT) is False + second.send(None) + second.waitclose(TESTTIMEOUT) + first.send(None) + first.waitclose(TESTTIMEOUT) + # the primary slot frees shortly after the exec finishes + for _ in range(50): + probe = gw.remote_exec(report) + on_main = probe.receive(TESTTIMEOUT) + probe.send(None) + probe.waitclose(TESTTIMEOUT) + if on_main: + break + time.sleep(0.05) + assert on_main + + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: + # info is a native protocol request: it must work (and not claim + # an exec slot) while an exec occupies the worker -- under + # main_thread_only an info-by-remote_exec used to either steal + # the main thread or trip the concurrency deadlock guard. + channel = gw.remote_exec("channel.send(channel.receive())") + try: + rinfo = gw._rinfo(update=True) + assert rinfo.pid != os.getpid() + finally: + channel.send("done") + assert channel.receive(TESTTIMEOUT) == "done" + class TestPopenGateway: gwtype = "popen" @@ -302,9 +346,9 @@ def test_chdir_separation( assert x.lower() == str(tmp_path).lower() def test_remoteerror_readable_traceback(self, gw: Gateway) -> None: - with pytest.raises(gateway_base.RemoteError) as e: + with pytest.raises(execnet.RemoteError) as e: gw.remote_exec("x y").waitclose() - assert "gateway_base" in e.value.formatted + assert "_gateway_base" in e.value.formatted def test_many_popen(self, makegateway: Callable[[str], Gateway]) -> None: num = 4 @@ -381,20 +425,19 @@ def test_socket_gw_host_not_found(makegateway: Callable[[str], Gateway]) -> None class TestSshPopenGateway: gwtype = "ssh" - def test_sshconfig_config_parsing( - self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] + def test_ssh_trio_args_include_config( + self, monkeypatch: pytest.MonkeyPatch ) -> None: - l = [] + from execnet import _provision + from execnet import _trio_host + monkeypatch.setattr( - gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) + _provision, "ssh_remote_command", lambda spec, *a, **kw: "worker-cmd" ) - with pytest.raises(AttributeError): - makegateway("ssh=xyz//ssh_config=qwe") - - assert len(l) == 1 - popen_args = l[0] - i = popen_args.index("-F") - assert popen_args[i + 1] == "qwe" + args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + assert args[args.index("-F") + 1] == "qwe" + assert "xyz" in args + assert args[-1] == "worker-cmd" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh @@ -477,9 +520,7 @@ def test_popen_filetracing( monkeypatch.setenv("EXECNET_DEBUG", "1") gw = makegateway("popen") # hack out the debuffilename - fn = gw.remote_exec( - "import execnet;channel.send(execnet.gateway_base.fn)" - ).receive() + fn = gw.remote_exec("import execnet;channel.send(execnet._trace.fn)").receive() assert isinstance(fn, str) workerfile = pathlib.Path(fn) assert workerfile.exists() @@ -509,7 +550,7 @@ def test_popen_stderr_tracing( gw.exit() def test_no_tracing_by_default(self): - assert gateway_base.trace == gateway_base.notrace, ( + assert _trace.trace == _trace.notrace, ( "trace does not to default to empty tracing" ) @@ -530,57 +571,24 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - expected_args = [*expected_args, "-u", "-c", gateway_io.popen_bootstrapline] - args = gateway_io.popen_args(execnet.XSpec(spec)) - assert args == expected_args + from execnet import _trio_gateway + args = _trio_gateway.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + assert args[: len(expected_args)] == expected_args + assert args[len(expected_args) :][:4] == ["-u", "-m", "execnet", "worker"] -@pytest.mark.parametrize( - "interleave_getstatus", - [ - pytest.param(True, id="interleave-remote-status"), - pytest.param( - False, - id="no-interleave-remote-status", - marks=pytest.mark.xfail( - reason="https://github.com/pytest-dev/execnet/issues/123", - ), - ), - ], -) -def test_regression_gevent_hangs( - group: execnet.Group, interleave_getstatus: bool -) -> None: - pytest.importorskip("gevent") - gw = group.makegateway("popen//execmodel=gevent") - - print(gw.remote_status()) - def sendback(channel) -> None: - channel.send(1234) - - ch = gw.remote_exec(sendback) - if interleave_getstatus: - print(gw.remote_status()) - assert ch.receive(timeout=0.5) == 1234 - - -def test_assert_main_thread_only( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] +def test_first_remote_exec_claims_the_main_thread( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") - + # The `thread` profile hands the first request the worker main thread + # -- the GUI/signal-safety property `main_thread_only` existed for. + # FIFO admission makes that one deterministic; a sequential *re*-exec + # races the claim release (see HybridExec), so only the first is + # asserted here. + gw = makegateway("profile=thread//popen") try: - # Submit multiple remote_exec requests in quick succession and - # assert that all tasks execute in the main thread. It is - # necessary to call receive on each channel before the next - # remote_exec call, since the channel will raise an error if - # concurrent remote_exec requests are submitted as in - # test_main_thread_only_concurrent_remote_exec_deadlock. - for i in range(10): + for _ in range(1): ch = gw.remote_exec( """ import time, threading @@ -588,7 +596,6 @@ def test_assert_main_thread_only( channel.send(threading.current_thread() is threading.main_thread()) """ ) - try: res = ch.receive() finally: @@ -604,43 +611,29 @@ def test_assert_main_thread_only( gw.join() -def test_main_thread_only_concurrent_remote_exec_deadlock( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] +def test_main_thread_only_is_deprecated_and_overflows( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") + # `main_thread_only` now maps to `thread`. Where it used to refuse a + # second concurrent remote_exec with a deadlock error, the request now + # overflows to a pool thread -- the first one still gets the main + # thread, which is what the profile was for. + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway("profile=main_thread_only//popen") channels = [] try: - # Submit multiple remote_exec requests in quick succession and - # assert that MAIN_THREAD_ONLY_DEADLOCK_TEXT is raised if - # concurrent remote_exec requests are submitted for the - # main_thread_only execmodel (as compensation for the lack of - # back pressure in remote_exec calls which do not attempt to - # block until the remote main thread is idle). - for i in range(2): + for _ in range(2): channels.append( gw.remote_exec( """ import threading channel.send(threading.current_thread() is threading.main_thread()) - # Wait forever, ensuring that the deadlock case triggers. - channel.gateway.execmodel.Event().wait() + channel.receive() """ ) ) - - expected_results = ( - True, - execnet.gateway_base.MAIN_THREAD_ONLY_DEADLOCK_TEXT, - ) - for expected, ch in zip(expected_results, channels, strict=True): - try: - res = ch.receive() - except execnet.RemoteError as e: - res = e.formatted - assert res == expected + # both run: first on the main thread, second on an overflow thread + assert [ch.receive(TESTTIMEOUT) for ch in channels] == [True, False] finally: for ch in channels: ch.close() diff --git a/testing/test_gevent.py b/testing/test_gevent.py new file mode 100644 index 00000000..5d8a84c3 --- /dev/null +++ b/testing/test_gevent.py @@ -0,0 +1,168 @@ +"""The execnet.gevent facade: greenlet-parking blocking waits. + +Opt-in: requires the ``gevent`` dependency group (``uv sync --group +gevent``); skipped when gevent is not installed. No monkey-patching is +needed -- the wakener parks the waiting greenlet while the trio host +thread keeps running the protocol. +""" + +from __future__ import annotations + +import threading + +import pytest + +gevent = pytest.importorskip("gevent") + +import execnet # noqa: E402 +import execnet.gevent # noqa: E402 +from execnet._boundary import Flag # noqa: E402 +from execnet._boundary import Mailbox # noqa: E402 +from execnet._boundary import make_wakener # noqa: E402 + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def gevent_gw(): + group = execnet.gevent.Group() + try: + yield group.makegateway("popen") + finally: + group.terminate(timeout=5.0) + + +class TestGeventWakener: + def test_mailbox_wakes_greenlet_from_foreign_thread(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + threading.Timer(0.05, box.put, args=["item"]).start() + result = gevent.spawn(box.get, 5.0) + assert result.get(timeout=TESTTIMEOUT) == "item" + + def test_notify_before_first_wait_is_not_lost(self) -> None: + flag = Flag(make_wakener("gevent")) + flag.set() + assert flag.wait(timeout=1.0) + + def test_wait_parks_greenlet_not_hub(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + box.put("done") + + waiter = gevent.spawn(box.get, 5.0) + gevent.spawn(other) + # if get() blocked the hub, other() could never run and put() + assert waiter.get(timeout=TESTTIMEOUT) == "done" + assert progressed == [0, 1, 2, 3, 4] + + +class TestGeventGateway: + def test_receive_parks_greenlet_not_hub(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(channel.receive())") + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + # sending from a greenlet blocks-until-written on the + # gevent wakener, parking only this greenlet + channel.send("hello") + + waiter = gevent.spawn(channel.receive, TESTTIMEOUT) + gevent.spawn(other) + # if receive() blocked the hub, other() could never send and + # the remote echo could never arrive -> this would hang + assert waiter.get(timeout=TESTTIMEOUT) == "hello" + assert progressed == [0, 1, 2, 3, 4] + + def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(1)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 + gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) + + def test_makegateway_parks_greenlet_not_hub(self) -> None: + # management ops (makegateway/terminate) from a greenlet must not + # stall the hub: they wait on a OneShot with a gevent wakener. + group = execnet.gevent.Group() + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + + try: + ticker = gevent.spawn(other) + maker = gevent.spawn(group.makegateway, "popen") + gw = maker.get(timeout=TESTTIMEOUT) + channel = gw.remote_exec("channel.send(42)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 + ticker.get(timeout=TESTTIMEOUT) + assert progressed == [0, 1, 2, 3, 4] + finally: + gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + + +class TestGeventWorkerProfile: + """execmodel=gevent: exec'd code runs as greenlets on the main-thread hub.""" + + @pytest.fixture + def worker_gw(self): + group = execnet.Group() + try: + yield group.makegateway("popen//execmodel=gevent") + finally: + group.terminate(timeout=5.0) + + def test_execs_are_greenlets_on_main_thread(self, worker_gw) -> None: + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = worker_gw.remote_exec(report) + second = worker_gw.remote_exec(report) + # both run concurrently on the one main thread: greenlets + assert first.receive(TESTTIMEOUT) is True + assert second.receive(TESTTIMEOUT) is True + first.send(None) + second.send(None) + first.waitclose(TESTTIMEOUT) + second.waitclose(TESTTIMEOUT) + + def test_execs_cooperate_via_gevent(self, worker_gw) -> None: + # the first exec parks in channel.receive() (gevent wakener) while + # the second completes -- with a blocked hub this would deadlock. + blocked = worker_gw.remote_exec("channel.send(channel.receive())") + side = worker_gw.remote_exec( + """ + import gevent + gevent.sleep(0.01) + channel.send('side') + """ + ) + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_status_reports_gevent(self, worker_gw) -> None: + assert worker_gw.remote_status().execmodel == "gevent" + + +def test_provisioning_adds_gevent_requirement() -> None: + from execnet import XSpec + from execnet._provision import _extra_with_tokens + from execnet._provision import worker_cli_arg + + spec = XSpec("popen//id=g1//execmodel=gevent") + config = worker_cli_arg(spec) + assert '"wait": "gevent"' in config + assert _extra_with_tokens(config) == ["--with", "gevent"] + plain = worker_cli_arg(XSpec("popen//id=g2//execmodel=thread")) + assert _extra_with_tokens(plain) == [] diff --git a/testing/test_host.py b/testing/test_host.py new file mode 100644 index 00000000..f5678a31 --- /dev/null +++ b/testing/test_host.py @@ -0,0 +1,197 @@ +"""The Trio host thread: sharing, explicit override, and loop-misuse guards. + +``execnet.trio`` runs gateways directly in the caller's own nursery; every +other surface drives a :class:`execnet.Host`. One is shared per process, +and blocking on it from inside a running event loop is an error rather +than a hang. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import threading + +import pytest +import trio + +import execnet +from execnet._host import Host +from execnet._host import default_host + +TESTTIMEOUT = 30.0 + + +def host_thread_names() -> list[str]: + return [t.name for t in threading.enumerate() if t.name.startswith("execnet-host")] + + +class TestSharedHost: + def test_groups_share_the_default_host(self) -> None: + a = execnet.Group() + b = execnet.Group() + assert a.host is b.host is default_host() + + def test_many_groups_run_one_thread(self) -> None: + groups = [execnet.Group() for _ in range(3)] + try: + for group in groups: + group.makegateway("popen") + assert len(host_thread_names()) == 1 + for group in groups: + channel = group[0].remote_exec("channel.send(1)") + assert channel.receive(TESTTIMEOUT) == 1 + finally: + for group in groups: + group.terminate(timeout=5.0) + + def test_explicit_host_is_isolated_and_closes(self) -> None: + host = Host(name="execnet-host-isolated") + group = execnet.Group(host=host) + assert group.host is host + assert group.host is not default_host() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + assert "execnet-host-isolated" in host_thread_names() + finally: + group.terminate(timeout=5.0) + host.close() + assert not host.running + assert "execnet-host-isolated" not in host_thread_names() + + def test_host_context_manager_closes(self) -> None: + with Host(name="execnet-host-ctx") as host: + group = execnet.Group(host=host) + group.makegateway("popen") + group.terminate(timeout=5.0) + assert not host.running + + def test_starting_is_lazy(self) -> None: + host = Host(name="execnet-host-lazy") + execnet.Group(host=host) + # constructing a group must not cost a thread + assert not host.running + assert "execnet-host-lazy" not in host_thread_names() + + @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") + def test_forked_child_gets_a_fresh_default_host(self) -> None: + # the child inherits a Host object whose thread does not exist + # there, so the first use must build a new one + default_host()._ensure_started() + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # pragma: no cover - runs in the child + code = 1 + try: + os.close(read_fd) + child_host = default_host() + group = execnet.Group() + gateway = group.makegateway("popen") + got = gateway.remote_exec("channel.send(3)").receive(TESTTIMEOUT) + group.terminate(timeout=5.0) + code = 0 if (got == 3 and child_host.running) else 1 + finally: + os.write(write_fd, bytes([code])) + os._exit(0) + os.close(write_fd) + try: + result = os.read(read_fd, 1) + finally: + os.close(read_fd) + os.waitpid(pid, 0) + assert result == b"\x00" + + +class TestEventLoopGuard: + """Blocking on the host from inside a running loop must not hang.""" + + def test_makegateway_inside_asyncio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + execnet.Group().makegateway("popen") + + asyncio.run(main()) + + def test_makegateway_inside_trio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match=r"execnet\.trio"): + execnet.Group().makegateway("popen") + + trio.run(main) + + def test_channel_receive_inside_asyncio_raises(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(1)") + + async def main() -> None: + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + channel.receive(TESTTIMEOUT) + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + channel.send(1) + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + channel.waitclose(TESTTIMEOUT) + + asyncio.run(main()) + # still usable from a plain thread afterwards + assert channel.receive(TESTTIMEOUT) == 1 + finally: + group.terminate(timeout=5.0) + + def test_worker_channels_are_not_guarded(self) -> None: + # exec'd code may run its own event loop and talk to its channel + # from inside it -- that is the caller's own loop to block + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec( + """ + import asyncio + + async def main(): + channel.send(channel.receive() + 1) + + asyncio.run(main()) + """ + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + + def test_a_worker_thread_is_still_fine(self) -> None: + group = execnet.Group() + result: list[object] = [] + + def work() -> None: + gateway = group.makegateway("popen") + result.append(gateway.remote_exec("channel.send(5)").receive(TESTTIMEOUT)) + + async def main() -> None: + # inside a loop, but the blocking call happens off it + await asyncio.to_thread(work) + + try: + asyncio.run(main()) + assert result == [5] + finally: + group.terminate(timeout=5.0) + + def test_no_asyncio_import_no_cost(self) -> None: + # the probe must not import asyncio/trio into a program that has + # neither; it goes through sys.modules first + code = ( + "import sys, execnet;" + " execnet._host.check_not_in_event_loop('x');" + " print('asyncio' in sys.modules, 'trio' in sys.modules)" + ) + import subprocess + + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.split() == ["False", "False"] diff --git a/testing/test_multi.py b/testing/test_multi.py index 12e0ed3d..156879b3 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -5,18 +5,19 @@ from __future__ import annotations import gc +import threading from collections.abc import Callable from time import sleep import pytest import execnet +from execnet import Gateway +from execnet import Group from execnet import XSpec -from execnet.gateway import Gateway -from execnet.gateway_base import Channel -from execnet.gateway_base import ExecModel -from execnet.multi import Group -from execnet.multi import safe_terminate +from execnet._channel import Channel +from execnet._execmodel import ExecModel +from execnet._multi import safe_terminate class TestMultiChannelAndGateway: @@ -69,7 +70,7 @@ def test_Group_execmodel_setting(self) -> None: gm._gateways.append(1) # type: ignore[arg-type] try: with pytest.raises(ValueError): - gm.set_execmodel("eventlet") + gm.set_execmodel("main_thread_only") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop() @@ -231,6 +232,24 @@ def test_terminate_with_proxying(self) -> None: group.makegateway("popen//via=master//id=worker") group.terminate(1.0) + def test_via_foreign_python(self) -> None: + # A python= sub-spec through a via master: the master resolves the + # interpreter locally (this interpreter has execnet, so the sub runs + # the worker module directly, no uv provisioning). + import sys + + group = Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"popen//python={sys.executable}//via=master//id=sub" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(1.0) + @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate(execmodel: ExecModel) -> None: @@ -290,12 +309,12 @@ def test_safe_terminate_does_not_hang_when_kill_blocks( out, so a blocking killfunc made Group.terminate() hang indefinitely (seen via pytest-xdist teardown). """ - kill_started = execmodel.Event() - release_kill = execmodel.Event() + kill_started = threading.Event() + release_kill = threading.Event() other_killed: list[int] = [] def term_slow() -> None: - execmodel.sleep(10) + sleep(10) def kill_hang() -> None: kill_started.set() diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py new file mode 100644 index 00000000..63a707b3 --- /dev/null +++ b/testing/test_namespaces.py @@ -0,0 +1,220 @@ +"""The public namespaces, and the deprecation shims for the private modules. + +There is one namespace per concurrency library the caller drives execnet +from: top-level ``execnet.*`` is an alias surface over ``execnet.sync`` +(threads), ``execnet.trio`` and ``execnet.aio`` expose the async-native core +and its asyncio bridge, and ``execnet.gevent`` the greenlet-parking blocking +surface. Everything else in the package is private -- the pre-Trio module +names survive only as warning shims. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import subprocess +import sys +import warnings + +import pytest + +import execnet +import execnet.aio +import execnet.sync +import execnet.trio + +#: the only modules that may be reachable without a leading underscore +PUBLIC_NAMESPACES = ("aio", "gevent", "sync", "trio") + +#: those importable without an optional dependency (execnet.gevent needs gevent) +ALWAYS_IMPORTABLE = tuple(n for n in PUBLIC_NAMESPACES if n != "gevent") + +#: pre-Trio module names kept as deprecated forwarding shims +SHIMS = ("gateway", "gateway_base", "multi", "rsync", "rsync_remote", "xspec") + + +def test_top_level_names_alias_execnet_sync() -> None: + for name in execnet.sync.__all__: + assert getattr(execnet, name) is getattr(execnet.sync, name), name + + +def test_top_level_all_matches_sync_surface() -> None: + # the top level is the sync surface plus the two package-level names: + # the version, and can_send (a wire-format fact, not a gateway API) + assert set(execnet.__all__) == set(execnet.sync.__all__) | { + "__version__", + "can_send", + } + + +@pytest.mark.parametrize( + "namespace", + [execnet, *[importlib.import_module(f"execnet.{n}") for n in ALWAYS_IMPORTABLE]], +) +def test_namespace_all_resolves(namespace: object) -> None: + missing = [n for n in namespace.__all__ if not hasattr(namespace, n)] # type: ignore[attr-defined] + assert not missing + + +def test_gevent_namespace_all_resolves() -> None: + pytest.importorskip("gevent") + namespace = importlib.import_module("execnet.gevent") + assert not [n for n in namespace.__all__ if not hasattr(namespace, n)] + # the facade is the sync surface with greenlet parking wired in + assert namespace.Group._wait_backend == "gevent" + assert issubclass(namespace.Group, execnet.Group) + + +def test_no_unexpected_public_modules() -> None: + found = { + info.name + for info in pkgutil.iter_modules(execnet.__path__) + if not info.name.startswith("_") + } + assert found == set(PUBLIC_NAMESPACES) | set(SHIMS) + + +def test_trio_namespace_exposes_async_core() -> None: + from execnet import _trio_gateway + + assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup + assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway + assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel + assert execnet.trio.open_gateway is _trio_gateway.open_gateway + # error types are shared with the sync surface; the standalone serializer + # is intentionally not exposed on any public namespace + assert execnet.trio.RemoteError is execnet.RemoteError + assert not hasattr(execnet.trio, "dumps") + + +def test_trio_namespace_hides_raw_plumbing() -> None: + # the raw-channel/stream layer is internal routing detail: reachable from + # execnet._trio_gateway, not advertised on the public namespace + for name in ("ByteStream", "RawChannel", "RawChannelStream", "serve_gateway"): + assert name not in execnet.trio.__all__, name + + +def test_can_send_lives_only_on_the_top_level() -> None: + assert execnet.can_send({"a": [1, 2.0, b"x", None, (True, frozenset({3}))]}) + assert not execnet.can_send(object()) + # the wire contract does not vary by surface, so it is not mirrored + for namespace in (execnet.sync, execnet.trio, execnet.aio): + assert "can_send" not in namespace.__all__, namespace.__name__ + + +def test_dumps_is_a_temporary_xdist_shim() -> None: + # FOLLOW-UP: delete this test together with execnet._XDIST_COMPAT once + # pytest-xdist stops probing with ``execnet.dumps`` / ``except DumpError`` + # and uses ``execnet.can_send`` instead. + from execnet import _serialize + + assert execnet._XDIST_COMPAT == ("dumps",) + assert execnet.dumps is _serialize.dumps + # reachable, but never advertised + assert "dumps" not in execnet.__all__ + assert "dumps" not in dir(execnet) + + +def test_dumps_shim_does_not_warn() -> None: + # xdist reaches this from serialize_warning_message -- once per warning + # a *user's* test raises, from inside pytest's warning-recording hook. + # A warning there is attributed to that test, which cannot act on it, + # and warning on every access made recording one warning record + # another, unbounded, wedging the run. + with warnings.catch_warnings(): + warnings.simplefilter("error") + for _ in range(3): + execnet.dumps # noqa: B018 + + +def test_boundary_kit_is_private() -> None: + # There is no third-party event-loop extension point: the two wait + # backends are threads and gevent, and every other concurrency library + # gets a facade instead of a wakener. + assert not hasattr(execnet, "portal") + for name in ("Wakener", "Mailbox", "OneShot", "LoopPortal"): + assert not hasattr(execnet, name), name + from execnet import _boundary + + assert not hasattr(_boundary, "register_wakener") + assert _boundary.make_wakener("thread") is not None + with pytest.raises(ValueError, match="unknown wait backend"): + _boundary.make_wakener("nope") # type: ignore[arg-type] + + +def test_lazy_submodule_attribute_access() -> None: + # After ``import execnet`` alone, execnet.trio is reachable as an + # attribute (PEP 562) without having been imported. + out = subprocess.run( + [ + sys.executable, + "-c", + "import execnet; print(execnet.trio.AsyncGroup.__name__)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "AsyncGroup" + + +def test_import_execnet_does_not_import_trio() -> None: + # The blocking surface must stay importable without loading the trio + # event loop machinery (it loads lazily on first gateway / namespace + # use). + out = subprocess.run( + [ + sys.executable, + "-c", + "import sys, execnet; print('trio' in sys.modules)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "False" + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_reachable_as_package_attribute(shim: str) -> None: + # pytest-xdist reaches these as ``execnet.gateway_base.X`` after a plain + # ``import execnet``; that used to work because the import chain pulled + # them in, and must keep working for as long as the shims exist. + out = subprocess.run( + [sys.executable, "-c", f"import execnet; print(execnet.{shim}.__name__)"], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == f"execnet.{shim}" + + +def shim_attributes() -> list[tuple[str, str, str]]: + """``(shim, attribute, private module)`` for every forwarded name.""" + cases = [] + for shim in SHIMS: + module = importlib.import_module(f"execnet.{shim}") + for name, target in module._MOVED.items(): + cases.append((shim, name, target)) + return cases + + +@pytest.mark.parametrize(("shim", "name", "target"), shim_attributes(), ids=str) +def test_shim_warns_and_forwards(shim: str, name: str, target: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + private = importlib.import_module(f"execnet{target}") + if not hasattr(private, name): + # ``trace``/``notrace``/``fn`` exist only under a given EXECNET_DEBUG + pytest.skip(f"execnet{target}.{name} not defined in this configuration") + with pytest.warns(DeprecationWarning, match=f"execnet.{shim} is private"): + value = getattr(module, name) + assert value is getattr(private, name) + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_rejects_unknown_attribute(shim: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with pytest.raises(AttributeError, match="no attribute 'nonexistent'"): + getattr(module, "nonexistent") # noqa: B009 diff --git a/testing/test_provision.py b/testing/test_provision.py new file mode 100644 index 00000000..0d708b93 --- /dev/null +++ b/testing/test_provision.py @@ -0,0 +1,104 @@ +"""Unit tests for coordinator-side uv worker provisioning.""" + +from __future__ import annotations + +import json +import re + +import pytest + +import execnet +from execnet import _provision + +released = re.fullmatch(r"\d+\.\d+\.\d+", execnet.__version__) is not None + + +def test_worker_cli_arg_carries_config() -> None: + spec = execnet.XSpec("popen//id=gw5//execmodel=thread") + config = json.loads(_provision.worker_cli_arg(spec)) + assert config["id"] == "gw5-worker" + assert config["execmodel"] == "thread" + assert config["coordinator_version"] == execnet.__version__ + + +def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9") + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command = _provision.ssh_remote_command(spec) + assert "execnet==9.9.9" in command + assert "head -c" not in command # nothing is framed into the launch + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_ssh_remote_command_dev_uses_a_delivered_wheel() -> None: + # The wheel travels out of band now (its own connection, before the + # launch), so the launch command just points uv at where it landed -- + # no byte accounting, no `exec` to keep an fd alive. + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command = _provision.ssh_remote_command(spec) + wheel = _provision.ssh_wheel(spec) + assert wheel is not None + assert wheel.read_bytes()[:2] == b"PK" # a wheel is a zip archive + assert _provision.remote_wheel_path(wheel) in command + assert "head -c" not in command + assert "mktemp -d" not in command + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_wheel_delivery_command_expands_home_and_drains_stdin() -> None: + wheel = _provision.ssh_wheel(execnet.XSpec("ssh=host//id=gw0")) + assert wheel is not None + command = _provision.wheel_delivery_command(wheel) + # $HOME must stay expandable: quoting it as a literal would create a + # directory actually named "~" + assert '"$HOME"' in command + assert "~" not in command + # the coordinator always streams the wheel, so the cached branch has + # to consume stdin too or it hands the coordinator an EPIPE + assert "cat > /dev/null" in command + + +def test_vagrant_ssh_argv() -> None: + argv = _provision.vagrant_ssh_argv("default", None, "run-worker") + assert argv == ["vagrant", "ssh", "default", "--", "-C", "run-worker"] + argv = _provision.vagrant_ssh_argv("default", "/tmp/cfg", "run-worker") + assert argv == [ + "vagrant", + "ssh", + "default", + "--", + "-C", + "-F", + "/tmp/cfg", + "run-worker", + ] + + +def test_sub_spawn_argv_plain_popen() -> None: + import sys + + argv, delivery = _provision.sub_spawn_argv({"config": "{}"}) + assert argv == [ + sys.executable, + "-u", + "-m", + "execnet", + "worker", + "--config", + "{}", + ] + assert delivery is None + + +def test_sub_spawn_argv_vagrant_released() -> None: + request = { + "config": "{}", + "vagrant_ssh": "default", + "requirement": "execnet==9.9.9", + } + argv, delivery = _provision.sub_spawn_argv(request) + assert argv[:5] == ["vagrant", "ssh", "default", "--", "-C"] + assert "execnet==9.9.9" in argv[-1] + assert delivery is None diff --git a/testing/test_rsync.py b/testing/test_rsync.py index 2112439f..8190efc7 100644 --- a/testing/test_rsync.py +++ b/testing/test_rsync.py @@ -7,8 +7,8 @@ import pytest import execnet +from execnet import Gateway from execnet import RSync -from execnet.gateway import Gateway @pytest.fixture(scope="module") diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 06a84cd2..ae88f80d 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,9 @@ import execnet -# We use the execnet folder in order to avoid triggering a missing apipkg. -pyimportdir = os.fspath(Path(execnet.__file__).parent) +# The package parent: the serializer is imported as execnet._serialize +# (it needs its sibling execnet._errors; only stdlib beyond that). +pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) class PythonWrapper: @@ -24,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet._serialize as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -40,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet._serialize as serializer from io import BytesIO data = {data!r} io = BytesIO(data) @@ -125,6 +126,26 @@ def test_long(load, dump) -> None: assert v == really_big +@pytest.mark.parametrize( + "value", + [ + "2147483647", # int4 max: short path + "-2147483648", # int4 min: short path + "2147483648", # just over max: long path + "-2147483649", # just under min: long path (used to crash in struct.pack) + "9223372036854775807324234", + "-9223372036854775807324234", + ], +) +def test_int_boundaries(value, dump, load) -> None: + # regression: negative ints below the signed int4 minimum must take the + # arbitrary-precision long path instead of overflowing the 4-byte pack. + p = dump(value) + tp, v = load(p) + assert tp == "int" + assert v == value + + def test_bytes(dump, load) -> None: p = dump("b'hi'") tp, v = load(p) @@ -166,6 +187,7 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: assert s == "(1, (), 3)" -def test_py2_string_loads() -> None: - """Regression test for #267.""" - assert execnet.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" +def test_py2_string_opcode_is_retired() -> None: + """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" + with pytest.raises(execnet.DataFormatError, match="unknown opcode"): + execnet._serialize.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py new file mode 100644 index 00000000..2b6d60dd --- /dev/null +++ b/testing/test_socketserver_cli.py @@ -0,0 +1,58 @@ +"""Test the ``execnet-socketserver`` console entry point end to end. + +The Trio socketserver binds a port and spawns a ``python -m execnet._trio_worker`` +subprocess per connection (no inline code execution); the coordinator connects +over a Trio TCP stream. +""" + +from __future__ import annotations + +import shutil +import subprocess +from collections.abc import Iterator + +import pytest + +import execnet + +SERVER = shutil.which("execnet-socketserver") + +pytestmark = pytest.mark.skipif( + SERVER is None, reason="execnet-socketserver console script not installed" +) + + +@pytest.fixture +def socketserver_port() -> Iterator[int]: + assert SERVER is not None + proc = subprocess.Popen( + [SERVER, ":0"], # ephemeral port; it prints the one it bound + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + assert proc.stdout is not None + port = None + while True: + line = proc.stdout.readline() + if not line: + pytest.fail("execnet-socketserver exited before binding") + if "listening on" in line: + port = int(line.split()[-1]) + break + yield port + finally: + proc.kill() + proc.wait(timeout=5) + + +def test_socketserver_cli_roundtrip(socketserver_port: int) -> None: + group = execnet.Group() + try: + gw = group.makegateway(f"socket=127.0.0.1:{socketserver_port}//id=sock") + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py new file mode 100644 index 00000000..764ad12a --- /dev/null +++ b/testing/test_ssh_local.py @@ -0,0 +1,180 @@ +"""Local ssh-connect tests backed by an in-process asyncssh server. + +The coordinator shells out to the system ``ssh`` client, which connects to an +asyncssh server running in its own asyncio-loop thread; the server runs each +requested command as a subprocess with binary-safe stdio passthrough (the +execnet Message protocol needs raw bytes). +""" + +from __future__ import annotations + +import asyncio +import shutil +import sys +import threading +from collections.abc import Iterator +from pathlib import Path + +import asyncssh +import pytest + +import execnet + +pytestmark = pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" +) + +# Committed, intentionally-insecure test keys (see sshkeys/README.md). +SSHKEYS = Path(__file__).parent / "sshkeys" +HOST_KEY = SSHKEYS / "insecure_host_ed25519" +CLIENT_KEY = SSHKEYS / "insecure_client_ed25519" +CLIENT_PUBKEY = SSHKEYS / "insecure_client_ed25519.pub" + + +class _ForwardingSSHServer(asyncssh.SSHServer): # type: ignore[misc] + """Accepts ``-R`` unix-socket forwards, which the socket transport needs. + + execnet's ssh worker dials back to the coordinator over a unix socket + that ``ssh -R`` forwards; asyncssh refuses such requests unless the + server opts in, and returning True asks it to do the standard + forwarding. + """ + + def unix_server_requested(self, listen_path: str) -> bool: + return True + + +class SSHServerThread: + """asyncssh server on an ephemeral port, driven from its own asyncio thread.""" + + def __init__(self, client_key_path: str) -> None: + # OpenSSH refuses a world-readable private key; git does not preserve + # 0600, so the caller hands us a temp copy already chmod'd 0600. + self.client_key_path = client_key_path + self._loop: asyncio.AbstractEventLoop | None = None + self._server: asyncssh.SSHAcceptor | None = None + self.port: int | None = None + self._ready = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + async def _handle(self, process: asyncssh.SSHServerProcess) -> None: + proc = await asyncio.create_subprocess_shell( + process.command or "", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) + process.exit(await proc.wait()) + # Drain asyncssh's redirect cleanup coroutines (Queue.join et al): + # they are stored un-awaited and only run inside wait_closed(); + # skipping this leaks them and GC prints RuntimeWarnings. + await process.wait_closed() + + async def _serve(self) -> None: + self._server = await asyncssh.listen( + "127.0.0.1", + 0, + server_host_keys=[str(HOST_KEY)], + authorized_client_keys=str(CLIENT_PUBKEY), + server_factory=_ForwardingSSHServer, + process_factory=self._handle, + encoding=None, # binary stdio + ) + self.port = self._server.get_port() + self._ready.set() + await self._server.wait_closed() + + def _run(self) -> None: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + try: + self._loop.run_until_complete(self._serve()) + finally: + # Drain asyncssh's shutdown coroutines so GC does not surface an + # "un-awaited coroutine" RuntimeWarning. + pending = asyncio.all_tasks(self._loop) + for task in pending: + task.cancel() + self._loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.close() + + def start(self) -> None: + self._thread.start() + assert self._ready.wait(timeout=10), "ssh server did not start" + + def stop(self) -> None: + if self._loop is not None and self._server is not None: + self._loop.call_soon_threadsafe(self._server.close) + self._thread.join(timeout=5) + + def write_ssh_config(self, path: str) -> None: + """Write an ssh config with a ``testhost`` alias pointing at this server.""" + with open(path, "w") as f: + f.write( + "Host testhost\n" + " HostName 127.0.0.1\n" + f" Port {self.port}\n" + " User testuser\n" + f" IdentityFile {self.client_key_path}\n" + " IdentitiesOnly yes\n" + " StrictHostKeyChecking no\n" + " UserKnownHostsFile /dev/null\n" + " LogLevel ERROR\n" + ) + + +@pytest.fixture +def ssh_server(tmp_path) -> Iterator[SSHServerThread]: + # OpenSSH rejects the committed key's checkout permissions; use a 0600 copy. + client_key = tmp_path / "client_ed25519" + client_key.write_bytes(CLIENT_KEY.read_bytes()) + client_key.chmod(0o600) + server = SSHServerThread(str(client_key)) + server.start() + yield server + server.stop() + + +@pytest.fixture +def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: + path = str(tmp_path / "ssh_config") + ssh_server.write_ssh_config(path) + return path + + +def test_ssh_roundtrip(ssh_config: str) -> None: + # The worker is provisioned over ssh with uv. + group = execnet.Group() + try: + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}//id=ssh" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) + + +def test_ssh_via_roundtrip(ssh_config: str) -> None: + """An ssh sub-gateway spawned by a popen master (GATEWAY_START_SUB relay). + + The master runs the ssh client; for a dev coordinator the wheel travels + coordinator -> master (in the spawn request) -> remote (ssh stdin preamble). + """ + group = execnet.Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}" + "//via=master//id=sshvia" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) diff --git a/testing/test_termination.py b/testing/test_termination.py index ca119304..03d6bbb8 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -1,18 +1,19 @@ import os import pathlib +import queue import shutil import signal import subprocess import sys from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor import pytest from test_gateway import TESTTIMEOUT import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool +from execnet import Gateway +from execnet._execmodel import ExecModel execnetdir = pathlib.Path(execnet.__file__).parent.parent @@ -23,7 +24,7 @@ def test_exit_blocked_worker_execution_gateway( - anypython: str, makegateway: Callable[[str], Gateway], pool: WorkerPool + anypython: str, makegateway: Callable[[str], Gateway], executor: ThreadPoolExecutor ) -> None: gateway = makegateway("popen//python=%s" % anypython) gateway.remote_exec( @@ -37,8 +38,7 @@ def doit() -> int: gateway.exit() return 17 - reply = pool.spawn(doit) - x = reply.get(timeout=5.0) + x = executor.submit(doit).result(timeout=5.0) assert x == 17 @@ -48,7 +48,7 @@ def test_endmarker_delivery_on_remote_killterm( if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("test and execnet not compatible to greenlets yet") gw = makegateway("popen") - q = execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" import os, time @@ -60,7 +60,7 @@ def test_endmarker_delivery_on_remote_killterm( assert isinstance(pid, int) os.kill(pid, signal.SIGTERM) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert isinstance(err, EOFError) @@ -113,10 +113,8 @@ def test_terminate_implicit_does_trykill( pytester: pytest.Pytester, anypython: str, capfd: pytest.CaptureFixture[str], - pool: WorkerPool, + executor: ThreadPoolExecutor, ) -> None: - if pool.execmodel.backend not in ("thread", "main_thread_only"): - pytest.xfail("only os threading model supported") if sys.version_info >= (3, 12): pytest.xfail( "since python3.12 this test triggers RuntimeError: can't create new thread at interpreter shutdown" @@ -149,8 +147,7 @@ def flush(self): # sync with start-up assert popen.stdout is not None popen.stdout.readline() - reply = pool.spawn(popen.communicate) - reply.get(timeout=50) + executor.submit(popen.communicate).result(timeout=50) _out, err = capfd.readouterr() lines = [x for x in err.splitlines() if "*sys-package" not in x] assert not lines or "Killed" in err diff --git a/testing/test_threadpool.py b/testing/test_threadpool.py deleted file mode 100644 index 510ae020..00000000 --- a/testing/test_threadpool.py +++ /dev/null @@ -1,222 +0,0 @@ -import os -from pathlib import Path - -import pytest - -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool - - -def test_execmodel(execmodel: ExecModel, tmp_path: Path) -> None: - assert execmodel.backend - p = tmp_path / "somefile" - p.write_text("content") - fd = os.open(p, os.O_RDONLY) - f = execmodel.fdopen(fd, "r") - assert f.read() == "content" - f.close() - - -def test_execmodel_basic_attrs(execmodel: ExecModel) -> None: - m = execmodel - assert callable(m.start) - assert m.get_ident() - - -def test_simple(pool: WorkerPool) -> None: - reply = pool.spawn(lambda: 42) - assert reply.get() == 42 - - -def test_some(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - num = 4 - - def f(i: int) -> None: - q.put(i) - while q.qsize(): - execmodel.sleep(0.01) - - for i in range(num): - pool.spawn(f, i) - for i in range(num): - q.get() - # assert len(pool._running) == 4 - assert pool.waitall(timeout=1.0) - # execmodel.sleep(1) helps on windows? - assert len(pool._running) == 0 - - -def test_running_semnatics(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def first() -> None: - q.get() - - reply = pool.spawn(first) - assert reply.running - assert pool.active_count() == 1 - q.put(1) - assert pool.waitall() - assert pool.active_count() == 0 - assert not reply.running - - -def test_waitfinish_on_reply(pool: WorkerPool) -> None: - l = [] - reply = pool.spawn(lambda: l.append(1)) - reply.waitfinish() - assert l == [1] - reply = pool.spawn(lambda: 0 / 0) - reply.waitfinish() # no exception raised - pytest.raises(ZeroDivisionError, reply.get) - - -@pytest.mark.xfail(reason="WorkerPool does not implement limited size") -def test_limited_size(execmodel: ExecModel) -> None: - pool = WorkerPool(execmodel, size=1) # type: ignore[call-arg] - q = execmodel.queue.Queue() - q2 = execmodel.queue.Queue() - q3 = execmodel.queue.Queue() - - def first() -> None: - q.put(1) - q2.get() - - pool.spawn(first) - assert q.get() == 1 - - def second() -> None: - q3.put(3) - - # we spawn a second pool to spawn the second function - # which should block - pool2 = WorkerPool(execmodel) - pool2.spawn(pool.spawn, second) - assert not pool2.waitall(1.0) - assert q3.qsize() == 0 - q2.put(2) - assert pool2.waitall() - assert pool.waitall() - - -def test_get(pool: WorkerPool) -> None: - def f() -> int: - return 42 - - reply = pool.spawn(f) - result = reply.get() - assert result == 42 - - -def test_get_timeout(execmodel: ExecModel, pool: WorkerPool) -> None: - def f() -> int: - execmodel.sleep(0.2) - return 42 - - reply = pool.spawn(f) - with pytest.raises(IOError): - reply.get(timeout=0.01) - - -def test_get_excinfo(pool: WorkerPool) -> None: - def f() -> None: - raise ValueError("42") - - reply = pool.spawn(f) - with pytest.raises(ValueError): - reply.get(1.0) - with pytest.raises(ValueError): - reply.get(1.0) - - -def test_waitall_timeout(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def f() -> None: - q.get() - - reply = pool.spawn(f) - assert not pool.waitall(0.01) - q.put(None) - reply.get(timeout=1.0) - assert pool.waitall(timeout=0.1) - - -@pytest.mark.skipif(not hasattr(os, "dup"), reason="no os.dup") -def test_pool_clean_shutdown( - pool: WorkerPool, capfd: pytest.CaptureFixture[str] -) -> None: - q = pool.execmodel.queue.Queue() - - def f() -> None: - q.get() - - pool.spawn(f) - assert not pool.waitall(timeout=1.0) - pool.trigger_shutdown() - with pytest.raises(ValueError): - pool.spawn(f) - - def wait_then_put() -> None: - pool.execmodel.sleep(0.1) - q.put(1) - - pool.execmodel.start(wait_then_put) - assert pool.waitall() - _out, err = capfd.readouterr() - assert err == "" - - -def test_primary_thread_integration(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - with pytest.raises(ValueError): - WorkerPool(execmodel=execmodel, hasprimary=True) - return - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - - def func() -> None: - queue.put(execmodel.get_ident()) - - pool.spawn(func) - ident1 = queue.get() - ident2 = queue.get() - assert ident1 == ident2 - pool.terminate() - - -def test_primary_thread_integration_shutdown(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - pytest.skip("can only run with threading") - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - queue.get() - - queue2 = execmodel.queue.Queue() - - def get_two() -> None: - queue.put(execmodel.get_ident()) - queue2.get() - - reply = pool.spawn(get_two) - # make sure get_two is running and blocked on queue2 - queue.get() - # then shut down - pool.trigger_shutdown() - # and let get_two finish - queue2.put(1) - reply.get() - assert pool.waitall(5.0) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py new file mode 100644 index 00000000..9473df89 --- /dev/null +++ b/testing/test_trio_gateway.py @@ -0,0 +1,405 @@ +"""Protocol tests for the trio-native async gateway core (RawChannel level). + +Two AsyncGateways are wired together over an in-memory stream pair (the +transport harness, as in ``TestFrameDecoder``); the assertions cover execnet +semantics: payload routing by channel id, the close/EOF/sendonly state +machine, RemoteError propagation, and gateway termination. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest +import trio +import trio.testing + +from execnet import _errors +from execnet._errors import RemoteError +from execnet._message import Message +from execnet._serialize import dumps_internal +from execnet._serialize import loads_internal +from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import AsyncGroup +from execnet._trio_gateway import open_gateway + + +@asynccontextmanager +async def gateway_pair() -> AsyncIterator[tuple[AsyncGateway, AsyncGateway]]: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + left = AsyncGateway(left_stream, id="left", _startcount=1) + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(left._serve) + await nursery.start(right._serve) + try: + yield left, right + finally: + await left.aclose() + await right.aclose() + + +def test_payload_boundaries_are_preserved() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"first payload") + await sender.send_bytes(b"second") + assert await receiver.receive_bytes() == b"first payload" + assert await receiver.receive_bytes() == b"second" + + trio.run(main) + + +def test_send_eof_makes_peer_sendonly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"data") + await sender.send_eof() + assert await receiver.receive_bytes() == b"data" + with pytest.raises(EOFError): + await receiver.receive_bytes() + # the receiver of an EOF may still send back + await receiver.send_bytes(b"reply") + assert await sender.receive_bytes() == b"reply" + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"after eof") + + trio.run(main) + + +def test_close_drains_then_blocks_both_directions() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"x") + await sender.aclose() + # payloads sent before the close still drain + assert await receiver.receive_bytes() == b"x" + with pytest.raises(EOFError): + await receiver.receive_bytes() + with pytest.raises(OSError, match="cannot send"): + await receiver.send_bytes(b"y") + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"z") + + trio.run(main) + + +def test_close_with_error_raises_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.aclose(error="boom happened") + with pytest.raises(RemoteError, match="boom happened"): + await receiver.receive_bytes() + + trio.run(main) + + +def test_async_iteration_yields_payloads_until_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + payloads = [b"a", b"bb", b"ccc"] + for payload in payloads: + await sender.send_bytes(payload) + await sender.send_eof() + assert [data async for data in receiver] == payloads + + trio.run(main) + + +def test_terminate_closes_peer_cleanly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = right.open_raw_channel() + await left.terminate() + await right.wait_closed() + assert right._error is None + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_status_reply_travels_on_raw_channel() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_raw_channel() + await left._send(Message.STATUS, channel.id) + status = loads_internal(await channel.receive_bytes()) + assert status["execmodel"] == "trio" + assert status["numexecuting"] == 0 + # the peer closes the status channel after the reply + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_unsupported_message_is_rejected_with_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_raw_channel() + await left._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal(("code", None, None, {})), + ) + with pytest.raises(RemoteError, match="unsupported message"): + await channel.receive_bytes() + + trio.run(main) + + +def test_send_after_gateway_close_raises() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_raw_channel() + await left.aclose() + with pytest.raises(OSError, match="cannot send"): + await channel.send_bytes(b"x") + with pytest.raises(OSError, match="already closed"): + left.open_raw_channel() + + trio.run(main) + + +def test_peer_disappearing_surfaces_eof_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + channel = right.open_raw_channel() + # peer vanishes without a termination message + await left_stream.aclose() + await right.wait_closed() + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_mid_frame_eof_is_an_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + frame = Message(Message.CHANNEL_DATA, 1, b"payload").pack() + await left_stream.send_all(frame[:5]) + await left_stream.aclose() + await right.wait_closed() + assert isinstance(right._error, EOFError) + assert "mid-frame" in str(right._error) + + trio.run(main) + + +def test_channel_serializes_builtin_items() -> None: + items = [42, "text", b"bytes", [1, 2], ("a", 1), {"key": [True, None]}, {1, 2}] + + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + for item in items: + await sender.send(item) + await sender.send_eof() + assert [item async for item in receiver] == items + + trio.run(main) + + +def test_channel_receive_timeout() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_channel() + with pytest.raises(_errors.TimeoutError): + await channel.receive(timeout=0.05) + + trio.run(main) + + +def test_channel_close_with_error_and_wait_closed() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.aclose(error="exec exploded") + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.receive() + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.wait_closed() + + trio.run(main) + + +def test_channel_wait_closed_on_clean_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.send(1) + await sender.send_eof() + await receiver.wait_closed() + # items sent before the EOF still drain after waitclose + assert await receiver.receive() == 1 + + trio.run(main) + + +def test_channel_objects_travel_over_the_wire() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + carrier = left.open_channel() + right_carrier = right.open_channel(carrier.id) + extra = left.open_channel() + await carrier.send({"reply-to": extra}) + received = await right_carrier.receive() + remote_extra = received["reply-to"] + assert remote_extra.id == extra.id + await remote_extra.send("over the transferred channel") + assert await extra.receive() == "over the transferred channel" + + trio.run(main) + + +def _remote_add(channel, a, b) -> None: # type: ignore[no-untyped-def] + channel.send(a + b) + + +class TestPopenAsyncGateway: + """Integration: an AsyncGateway serving a real popen worker inside + the user's own trio run (no host thread).""" + + def test_remote_exec_roundtrip(self) -> None: + async def main() -> None: + async with open_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() + 1)" + ) + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + trio.run(main) + + def test_remote_exec_function_with_kwargs(self) -> None: + async def main() -> None: + async with open_gateway() as gateway: + channel = await gateway.remote_exec(_remote_add, a=40, b=2) + assert await channel.receive() == 42 + + trio.run(main) + + def test_remote_error_propagates(self) -> None: + async def main() -> None: + async with open_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError('kaboom')") + with pytest.raises(RemoteError, match="kaboom"): + await channel.receive() + + trio.run(main) + + def test_exec_finish_closes_channel_ending_iteration(self) -> None: + async def main() -> None: + async with open_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(3): channel.send(i)" + ) + assert [item async for item in channel] == [0, 1, 2] + + trio.run(main) + + def test_concurrent_remote_execs(self) -> None: + async def main() -> None: + async with open_gateway() as gateway: + results = [] + + async def run_one(value: int) -> None: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 10)" + ) + await channel.send(value) + results.append(await channel.receive()) + + async with trio.open_nursery() as nursery: + for value in range(5): + nursery.start_soon(run_one, value) + assert sorted(results) == [0, 10, 20, 30, 40] + + trio.run(main) + + +class TestAsyncGroup: + def test_multiple_gateways_with_auto_ids(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + first = await group.makegateway() + second = await group.makegateway() + assert {first.id, second.id} == {"gw0", "gw1"} + for gateway in (first, second): + channel = await gateway.remote_exec("channel.send(42)") + assert await channel.receive() == 42 + + trio.run(main) + + def test_group_exit_terminates_and_reaps_workers(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + await group.makegateway() + await group.makegateway() + processes = list(group._processes.values()) + # workers exited on GATEWAY_TERMINATE, nobody had to kill them + assert [process.returncode for process in processes] == [0, 0] + + trio.run(main) + + def test_terminate_kills_hung_worker_within_bound(self) -> None: + async def main() -> None: + async with AsyncGroup(termination_timeout=1.0) as group: + gateway = await group.makegateway() + await gateway.remote_exec("import time\nwhile True: time.sleep(1)") + processes = list(group._processes.values()) + assert all(process.returncode is not None for process in processes) + + trio.run(main) + + def test_via_gateway_relays_through_master(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + master = await group.makegateway("popen//id=master") + sub = await group.makegateway("popen//via=master") + master_channel = await master.remote_exec( + "import os; channel.send(os.getpid())" + ) + sub_channel = await sub.remote_exec( + "import os; channel.send(os.getpid())" + ) + master_pid = await master_channel.receive() + sub_pid = await sub_channel.receive() + # a real second process, reached through the master's relay + assert sub_pid != master_pid + echo = await sub.remote_exec("channel.send(channel.receive() * 2)") + await echo.send(21) + assert await echo.receive() == 42 + + trio.run(main) + + def test_unsupported_spec_is_rejected(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + with pytest.raises(ValueError, match="unsupported spec"): + await group.makegateway("id=notype") + + trio.run(main) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f837b07a..f81afaf5 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -10,11 +10,9 @@ import pytest import execnet +from execnet import Gateway from execnet import XSpec -from execnet.gateway import Gateway -from execnet.gateway_io import popen_args -from execnet.gateway_io import ssh_args -from execnet.gateway_io import vagrant_ssh_args +from execnet import _provision skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -64,27 +62,25 @@ def test_ssh_options(self) -> None: def test_execmodel(self) -> None: spec = XSpec("execmodel=thread") assert spec.execmodel == "thread" - spec = XSpec("execmodel=eventlet") - assert spec.execmodel == "eventlet" + spec = XSpec("execmodel=main_thread_only") + assert spec.execmodel == "main_thread_only" def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") - spec.ssh_config = "/home/user/ssh_config" - assert ssh_args(spec)[:6] == ["ssh", "-C", "-F", spec.ssh_config, "-p", "22100"] + args = _provision.ssh_argv("-p 22100 user@host", "/home/user/ssh_config", "cmd") + assert args[:6] == ["ssh", "-C", "-F", "/home/user/ssh_config", "-p", "22100"] + assert spec.ssh is not None def test_vagrant_options(self) -> None: - spec = XSpec("vagrant_ssh=default//python=python3") - assert vagrant_ssh_args(spec)[:-1] == ["vagrant", "ssh", "default", "--", "-C"] + args = _provision.vagrant_ssh_argv("default", None, "cmd") + assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - spec = XSpec("popen//python=sudo python3") - assert popen_args(spec) == [ - "sudo", - "python3", - "-u", - "-c", - "import sys;exec(eval(sys.stdin.readline()))", - ] + from execnet import _trio_gateway + + spec = XSpec("popen//python=sudo python3//id=gw0") + args = _trio_gateway.popen_module_args(spec) + assert args[:6] == ["sudo", "python3", "-u", "-m", "execnet", "worker"] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") @@ -123,6 +119,16 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) + def test_wait_backend_comes_from_the_facade( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # not a spec key: the blocking surface decides how *it* parks, and + # a thread-shaped worker profile parks on threads either way + gw = makegateway("popen") + assert gw._wait_backend == "thread" + channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") + assert channel.receive() == "thread" + @skip_win_pypy def test_popen_default(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("") diff --git a/uv.lock b/uv.lock index 893f1543..311eb0d8 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncssh" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/7f/2d79247bacc562104f312d27efe541673aa177feac89de291bd61bca52be/asyncssh-2.24.0.tar.gz", hash = "sha256:4064c590e59ce2e8d82a2f66d35f3120d765828b4df5e3dbfb07b4a8c24686c9", size = 550148, upload-time = "2026-06-27T20:34:44.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/29/908ce0ca5e8cae76662e354a0f08df552d6d221844748b9e5ca06051cc44/asyncssh-2.24.0-py3-none-any.whl", hash = "sha256:9abd46300adcb6d4b73269b34c53cd0d17a138b9a22b5b38008ce7d5808734b7", size = 381237, upload-time = "2026-06-27T20:34:43.198Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -136,6 +158,8 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/e9/6d7724983b3d5a0908dbf74f64038ade77c18646ff6636ec7894fd392ce1/cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0", size = 183837, upload-time = "2026-07-06T21:32:09.655Z" }, + { url = "https://files.pythonhosted.org/packages/69/aa/24580a278de21fd7322635556334d9b535f1cbc00b0a3919447cdf464c65/cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd", size = 184226, upload-time = "2026-07-06T21:32:11.196Z" }, { url = "https://files.pythonhosted.org/packages/88/a9/02cae418ec4beb282ace11958d9d4737793439d561fadc7e6d56f2e2b354/cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46", size = 211107, upload-time = "2026-07-06T21:32:12.328Z" }, { url = "https://files.pythonhosted.org/packages/3b/30/c806937ed5e4c2c7ac30d9d6b76b5dc57ff8b75d83800d9bb11a8253cf2a/cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2", size = 218733, upload-time = "2026-07-06T21:32:13.67Z" }, { url = "https://files.pythonhosted.org/packages/f9/cf/398272b8bbfd58aa314fda5a7f1cdbb26d1d78ae324a11211521315dd1f0/cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd", size = 205543, upload-time = "2026-07-06T21:32:15.148Z" }, @@ -146,6 +170,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/75/74dfb7c3fc6ebbd408038476bd4c1d7e925c62614e7b9c534ecc34218288/cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd", size = 220341, upload-time = "2026-07-06T21:32:21.9Z" }, { url = "https://files.pythonhosted.org/packages/70/b6/9003c33a3e7d2c1306f5962e646457dcfe5a8cd8fce6bbe02d7af25db783/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f", size = 174578, upload-time = "2026-07-06T21:32:23.073Z" }, { url = "https://files.pythonhosted.org/packages/8a/26/710688310447531c7a22f857c7f79d9855ec18b03e04494ced723fb37e2f/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da", size = 185071, upload-time = "2026-07-06T21:32:24.671Z" }, + { url = "https://files.pythonhosted.org/packages/d3/67/85c89a59ba36a671e79638f44d466749f08179266a57e4f2ffdf92174072/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc", size = 183845, upload-time = "2026-07-06T21:32:26.32Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dd/e3b0baa2d3d6a857ac72b7efbf18e32e487c9cdafcc13049ad765495b15e/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7", size = 184186, upload-time = "2026-07-06T21:32:28.025Z" }, { url = "https://files.pythonhosted.org/packages/65/68/9f3ef890cf3c6ab97bd531c5677f67613d302165d16f8142b2811782a614/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93", size = 211892, upload-time = "2026-07-06T21:32:29.565Z" }, { url = "https://files.pythonhosted.org/packages/22/d7/1a74539db16d8bfd839ff1515948948efbb162e574650fd3d846896eea95/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2", size = 218793, upload-time = "2026-07-06T21:32:30.951Z" }, { url = "https://files.pythonhosted.org/packages/ec/d1/9a5b7169499e8e8d8e636de70b97ac7c9447104d2ff1a2cd94790cea5162/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c", size = 205737, upload-time = "2026-07-06T21:32:32.216Z" }, @@ -157,6 +183,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, @@ -169,6 +197,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, @@ -181,6 +211,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, @@ -190,6 +222,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, @@ -201,6 +235,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, @@ -210,6 +246,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, @@ -261,6 +299,7 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, @@ -272,6 +311,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, @@ -283,6 +325,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, @@ -294,10 +339,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] @@ -333,11 +382,18 @@ wheels = [ [[package]] name = "execnet" source = { editable = "." } +dependencies = [ + { name = "trio" }, +] [package.optional-dependencies] -testing = [ +gevent = [ { name = "gevent" }, +] +testing = [ + { name = "asyncssh" }, { name = "hatch" }, + { name = "hypothesis" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-timeout" }, @@ -346,23 +402,34 @@ testing = [ ] [package.dev-dependencies] +dev = [ + { name = "pytest-xdist" }, +] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] [package.metadata] requires-dist = [ - { name = "gevent", marker = "extra == 'testing'" }, + { name = "asyncssh", marker = "extra == 'testing'" }, + { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, + { name = "hypothesis", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, { name = "tox", marker = "extra == 'testing'" }, + { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] -provides-extras = ["testing"] +provides-extras = ["gevent", "testing"] [package.metadata.requires-dev] +dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] +gevent = [{ name = "gevent", specifier = ">=26.7.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -376,7 +443,7 @@ wheels = [ [[package]] name = "gevent" -version = "26.5.0" +version = "26.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, @@ -384,106 +451,139 @@ dependencies = [ { name = "zope-event" }, { name = "zope-interface" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/b7/01a5880e01702f39fb09e3616c624054a0dc9a82561a865f3b1eff4bfc80/gevent-26.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2ba673dcbf7747513b58fa64ca7e9d6a828bc5c604d1552d23db89006d7911df", size = 2181491, upload-time = "2026-05-20T20:35:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fe/035ec5fa58a886740a744380118f03a90ac2da3f6c9cba248f28074ce40a/gevent-26.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:271b1474d81bb33036631adb16a35e5a1ee9dc414b05c999d6b01dc839a89975", size = 2212161, upload-time = "2026-05-20T20:43:25.678Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ea/ea87c08931c9e4c6c40bb05a2cb19c2d6f93fe6e0052f9152ea5ade6d037/gevent-26.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cd3dc60581687e2618286108f8e2f820d8446be4b34131065011c066e911d39c", size = 1768295, upload-time = "2026-05-20T21:17:29.438Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1d0e7287ae55700a8d25153ac736896bd9bcc3f85a12d374ef398db4b33c/gevent-26.5.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:dc7fa28b2d627f8e87595f39043b6dec71e8e7fb97e685e5506c47cf3ff8cb2e", size = 1862627, upload-time = "2026-05-20T21:15:59.365Z" }, - { url = "https://files.pythonhosted.org/packages/3a/4c/7f5ed67e52dfdef4ff91ae1a6fb28186d52e2496962edc8f17bdea9ab2c0/gevent-26.5.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:68c5fc21cef80268cdff88a4ae6c025fabb019b071f6f8ee4d20a7bccbddb873", size = 1804690, upload-time = "2026-05-20T21:30:51.713Z" }, - { url = "https://files.pythonhosted.org/packages/4c/75/0f5da6ca045f8a052203e1810058029f4b682507a789b413cac7d28bae28/gevent-26.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d325502eb0695708ef8c899f605573ed6847f3961f8159627dba267fbf3ce457", size = 2119054, upload-time = "2026-05-20T20:35:22.678Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f1/fcff7f7fad2bb33f3742db6b2145825a2191c0cd31d75789b0741fd28faf/gevent-26.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a11daf3a588b932c8bf965fb18444c69aff48badec88435e988cf8d67137075a", size = 1778784, upload-time = "2026-05-20T21:16:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/98/57/151314f00bdc6ba77333febb3e9dc97fdf94d79426559b4fa8332f0c2b6e/gevent-26.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1101b5ef82a3fb178550cfd80f32293dc8dd2f3d0828292223ebba29d6f76e33", size = 2145373, upload-time = "2026-05-20T20:43:27.255Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b5/7a02f711db62cbed1c1a00e1f9ff50eef95ccc78d4c04a0f93636655d1b7/gevent-26.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:5233109ad4f3af16393ba9888f238919a05ce15ce68d6831ac8a0da8dfb750ae", size = 1696576, upload-time = "2026-05-20T20:15:49.62Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/5022adc310697ef25c6fb22eb9bf0ebcad3427b51776e882709de9a8b6d7/gevent-26.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:3be804565168ffacebeb21af9f1cd689831a89f0f12fc0c3f423c730c3c9eb31", size = 1552095, upload-time = "2026-05-20T20:16:54.81Z" }, - { url = "https://files.pythonhosted.org/packages/37/0b/1a530b2db55c97cc0cf44116201f538f3033c04c1d2aca143979b412f4be/gevent-26.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:e80ad2a8a1e8bdaa5605e3bf4929e0cebf9ea7b8237c83362f7257698bb14280", size = 2929714, upload-time = "2026-05-20T20:13:24.656Z" }, - { url = "https://files.pythonhosted.org/packages/b9/df/32fe851ed5f68493f354e09b19bdebae0de1185be4db0b2988e71e737fd3/gevent-26.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fe42c037253580a3386fce275f8a2a845e540f5a729916934a732f13d42e72cc", size = 1784838, upload-time = "2026-05-20T21:17:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9a/21332674f9a10e8cdf13b41b52e9d663647a1c6e1dc3c62b07c0aeefd360/gevent-26.5.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:9f463c7d6f69d13b6fe8e3b832a6175a6e95328a940f38495d25496d1ae8ad88", size = 1880440, upload-time = "2026-05-20T21:16:00.881Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b1/5f8a4196113cf7f3fdd987b483f7e6b10c28ea3930c4727e31ba8cce51b6/gevent-26.5.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:96d5e96b1b14a4c1023dcfcc114533217f13febc3b6169254f23fc18d19fee29", size = 1831592, upload-time = "2026-05-20T21:30:53.832Z" }, - { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, - { url = "https://files.pythonhosted.org/packages/e4/32/602c499d54472f64e5cdf6013aeab5ce6aa6fed005387e8b4f2d22f5dc8d/gevent-26.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f519139354d5ca7625df9ddb1b2ffada885c14abc5b4dbae3682e967ddf79669", size = 1796906, upload-time = "2026-05-20T21:16:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/22/d5/4620797bbd9c88f4541188efc138b0d615f9834db540da36a2249ee929c5/gevent-26.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e49ce0de007dfd7412edbc2b5d41cce33b049bb1b7086f50be5a09e601bde603", size = 1699995, upload-time = "2026-05-20T20:15:39.311Z" }, - { url = "https://files.pythonhosted.org/packages/cb/83/ac3477dfc0f9fd80c88110102c73cefc35dcded2b248544f45a8fa5412df/gevent-26.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c5ff29495a2eed2a244de8150f21893d6c1b15d8b4b5719ab4bbfa06db1e28f", size = 1547433, upload-time = "2026-05-20T20:15:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/7d/47/5b992ab9c8037633cfd0fe698a97a878f59d8eb53c381e91e9a1a76fd215/gevent-26.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9b4d3f34c913d1a6bec6d030365a517f3b527a9773b12e58cf56c3339bbe96e6", size = 2952523, upload-time = "2026-05-20T20:13:04.698Z" }, - { url = "https://files.pythonhosted.org/packages/74/11/c7dfc773eb43331a682efed610b49df6e976331f1b0e1c592a0c35d29872/gevent-26.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1d8da4e799431feeb4c9e441ac7431f0baabb9106976790d884289d08ac08359", size = 1787044, upload-time = "2026-05-20T21:17:32.845Z" }, - { url = "https://files.pythonhosted.org/packages/ae/28/9812933dac93560f46910a9e834805fe76f822c408bd1c20cdf299d7c311/gevent-26.5.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:51becdb4c30a8f45c1c028ad7a97bf5a1ed141f74b159a31aa9cc6aa1e6263a6", size = 1882342, upload-time = "2026-05-20T21:16:02.645Z" }, - { url = "https://files.pythonhosted.org/packages/96/4b/514f248f69b2230b69b0bb17f4158b0b05dd4b2cb469a60ab206e9fe7496/gevent-26.5.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:c42bbcd3d453b08ad8915fd3feaf3d44a3562cdf1c7b208f9837149711e16d9d", size = 1834136, upload-time = "2026-05-20T21:30:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/53/67/f5f30716efca99b6200ae89a9303a7e94dae085b7de6f6d0033c52a37f4b/gevent-26.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bd3445e4fbeeb46690ed8efe94b8d1d46b14aa04af8866ae7a8da5997828d1c6", size = 2115349, upload-time = "2026-05-20T20:35:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/09/d8/60e8809bde7986e6c4e6d106080b3603fa09b3bb0255fed1a4d8282e3ca2/gevent-26.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b573d5b2826edc705f31f07da6889ad483a6a0d64944ebd8d32205f7c5bf46fb", size = 1799443, upload-time = "2026-05-20T21:16:41.928Z" }, - { url = "https://files.pythonhosted.org/packages/f8/41/b388b2b1f0a026ea30687e51ddf81dbb783dfb55fac0a16708d2821d99e5/gevent-26.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d53b1b28f2082a151bded2850b53f6baed02f742d2a1584029e8bd42d457fb4", size = 2141117, upload-time = "2026-05-20T20:43:30.694Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f3/ac9a4b0de487e390c5d53a908a9347c0df0102de2bbf3e8603087769191d/gevent-26.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:23569ce0c254eb821fc3dcfe250843dde8b3180b09bae9e222e41aa3fa4885b7", size = 1699862, upload-time = "2026-05-20T20:15:33.642Z" }, - { url = "https://files.pythonhosted.org/packages/2a/cf/1ef1fc9b390563c0f97702f94a557d1649b7bbb5724f9b86c2122747e92f/gevent-26.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:40cdcdb2e404b6c82b82a4576bdb33958f23fc2deb0d933e9e022b362001e647", size = 1545341, upload-time = "2026-05-20T20:16:26.229Z" }, - { url = "https://files.pythonhosted.org/packages/17/55/7d98d3888e7bb9ad4656420dec69232ecbbea48792aff9295d0ad7cf8435/gevent-26.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:75a0050e4b87f08ddee7e56f59e6014cd7fcdc3153046c09a847940515d12c85", size = 2968223, upload-time = "2026-05-20T20:13:17.223Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b4/e8e116fcbcb9dc0bf3acc50037f86e1204c217c8ed5defde68be11b3aab6/gevent-26.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fd1a0b83a04e19378d9466ae0ee2b5937cf1d7fbfdcb916b2aea82179a208574", size = 1793926, upload-time = "2026-05-20T21:17:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/7b267e9754b661defb93542e97731a4df21f8a40dc0f6c853faa717cf124/gevent-26.5.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:4c964c15076e76391d523ec24202f579a2535f7e301a40efb1656ae046d3eb69", size = 1887632, upload-time = "2026-05-20T21:16:04.158Z" }, - { url = "https://files.pythonhosted.org/packages/5c/50/b47d29e99449bd13b557ffa451401dc13d397a9923f562ef90a4e8514502/gevent-26.5.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:45d5438d1c84da5df7e832434627624709543630977332bb4e2d05ecca362cc9", size = 1838688, upload-time = "2026-05-20T21:30:57.979Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/5b54ccff11bc7d7bebd40a24571ccc115d5cdae4f6c32ab457b43b436e42/gevent-26.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:354f35924113abc954819216c2a6ee16751958c615681e0490946e31b437bd2f", size = 2120351, upload-time = "2026-05-20T20:35:32.699Z" }, - { url = "https://files.pythonhosted.org/packages/9c/70/30fd325c30e04b1e5174c61945e17421d53ddb2450366cc52cef234f8c4b/gevent-26.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a47cd2d32f6404212d374ad8014a3491d7477dcf0cc09c5a2308ad6d325fd663", size = 1806684, upload-time = "2026-05-20T21:16:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e8/fbf911ac3f9524ecfaed174d100fde671904ab8db92ceaf07faaebd13386/gevent-26.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:032157cebdedb84f2f52cdd980f2f5f2623eed6a8f083aadf44b44c47f628642", size = 2146606, upload-time = "2026-05-20T20:43:32.216Z" }, - { url = "https://files.pythonhosted.org/packages/9e/4d/284fcbbfde66fd978c2980c1fbe0eabd586af6e4b728649e9cf459e8b38f/gevent-26.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c414935ba5fc88359110968851d3616f119082c937390d00a1c0f4f59be814f", size = 1722497, upload-time = "2026-05-20T20:16:44.274Z" }, - { url = "https://files.pythonhosted.org/packages/15/d2/9f66eb53434704402be0ba733bf3320bf589671a4b76fac52a7d6077e972/gevent-26.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:2a0f5993a04b95a35b3a118b1a58ba272833f9b547b774001dea29f90620882f", size = 1574249, upload-time = "2026-05-20T20:15:50.873Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d5/b4c50adb761878e3c96642b9f79bf44cee3120f3df55cd40876f51d89866/gevent-26.5.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2e117df896a2660c9ebd4e2b5afc02dfd6e2ddf9b495e787e67c72d105432b09", size = 2971993, upload-time = "2026-05-20T20:12:50.845Z" }, - { url = "https://files.pythonhosted.org/packages/03/83/71c2a945e80198422d1d93dbe67355f249fb456b451bf9201199d3ef6a1a/gevent-26.5.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:af5ffe9c11ffb8a39b6bef2e8b722aa2043ae4980977915c6aa8c68b4bc26e46", size = 1796658, upload-time = "2026-05-20T21:17:35.968Z" }, - { url = "https://files.pythonhosted.org/packages/42/96/548ca77aed5cb9a44e855a6c23ebceeb3554a0ea9ca0c01c311878899a3e/gevent-26.5.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:7da34aef7e87c43dd3662e5785e79ed505c01399a7cb42876d2d8925969fd75f", size = 1891473, upload-time = "2026-05-20T21:16:05.657Z" }, - { url = "https://files.pythonhosted.org/packages/f6/4f/f48bd47d5287afb0fbcc56165f3ed47583f1803bad401653fe27e71ade2d/gevent-26.5.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:1c6293a7046bcc6f3d8972a74b19cd7a4cfd02d3881edf0fcf827aa514bd247b", size = 1841429, upload-time = "2026-05-20T21:30:59.907Z" }, - { url = "https://files.pythonhosted.org/packages/a0/72/1925215fc720d2561fa3ec8d4af5f098f8d0cbfa76a45fafed6e5ade7718/gevent-26.5.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d3bde0f140a275b2fa88e4b6516bda85551930e10bc2fd95e18c1b7d11cb780c", size = 2123895, upload-time = "2026-05-20T20:35:34.964Z" }, - { url = "https://files.pythonhosted.org/packages/83/59/0f584f6b1170c9a6abd9b70ccf5e9cc5ead34eabafabc0e21876ef0fe6f7/gevent-26.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e29fb4b17d9958ec8cb7f6339a111b29bc23f2c2efbef86189d1248bb4862d17", size = 1809047, upload-time = "2026-05-20T21:16:45.977Z" }, - { url = "https://files.pythonhosted.org/packages/82/88/61e854bfd98ac22eac78a97fc6db10de0f9ace46514072b435c217168729/gevent-26.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b2239df2f7570efa03736678f3f053bb1bdd22a8a16cd28a2feb7d32ea5f533f", size = 2150764, upload-time = "2026-05-20T20:43:33.781Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f5/af048b97433d7f9a7df7f5510b2c46918b7d073dcfb3bf6d0ef0e5a83dcc/gevent-26.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:aae214952fd38d27a42dc416bb70193962ec932384b63445d29bbb5817a1c042", size = 1722600, upload-time = "2026-05-20T20:19:56.81Z" }, - { url = "https://files.pythonhosted.org/packages/11/95/fb74a2299c6a2d78d9de12deaaac640ab5d2ef96a8e0f97a3ff84b9ca84b/gevent-26.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:f7067564f139e33bf26a31ee3b13d168d76eb99a44b85ced626652b158baa80c", size = 1574406, upload-time = "2026-05-20T20:17:12.125Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/60/878d0cdef05d952ac7f17ffe385143fb0f3720afce0f6ff5ddbf7aac0342/gevent-26.7.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:80e98fc808bd9cc5c911d78a443d214bf0c8f96c9fdd296893df7e40364d5f37", size = 2198781, upload-time = "2026-07-22T16:48:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/69/79/6ce781b60049060e9d89b3d0fe60940353adeb39856aaad5ee925fd127e9/gevent-26.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf4b946b47cc6fdbdf9221f891db9a44df92166435c027760ee7dbdfb4039adc", size = 2229318, upload-time = "2026-07-22T17:02:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/48898f35c2092d699b755b01918551358db72b554c63f553c8f027d2bf31/gevent-26.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:55ce0b7f87f9befcc788d77eb039b1de89a35f37afc31942e12c7ae090a563b8", size = 1700480, upload-time = "2026-07-22T16:26:16.794Z" }, + { url = "https://files.pythonhosted.org/packages/93/51/53370896942523c333699394ccad379d186648dbfb913f42ce094ddfb4b3/gevent-26.7.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f7143823ef99bc657534a2b6e8cbadedc910750cc0b4f4b4438a58d9fe43ab2", size = 1783048, upload-time = "2026-07-22T18:11:25.996Z" }, + { url = "https://files.pythonhosted.org/packages/ba/56/5a2cb36d75d3b626d6ffa116673b34442bf5afd6f7ab4b98e512c1b008d6/gevent-26.7.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:ca4019899830471910129968251c795c8aee59e225fd16326ae01c1f93f3cfa6", size = 1880257, upload-time = "2026-07-22T18:10:40.919Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ed/ee7eb2f03a38a4f33b0f327bab16d3158b3a794588a84dba739cbf4a3e68/gevent-26.7.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e4042da317a96d12110831cc404855f0c501a5a5aa476a7a18c3b480a5a59233", size = 1819378, upload-time = "2026-07-22T18:29:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/e2a202c03cff4f49bba54cc321c3afc29eee9d6fc452f4aa94d2f00e7d3d/gevent-26.7.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5c97ca98e1aae427a267eae0fbfe8d0884327e6b1cd51fc2ef6642b8b0b82701", size = 2136837, upload-time = "2026-07-22T16:48:29.939Z" }, + { url = "https://files.pythonhosted.org/packages/ac/64/4892fbca47aa4e06b86aef96d6146bd61175b23b7db431ec7937d73b2f72/gevent-26.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d5d1864bc3db92d1f82d1790395eda99f98b47fd9f7ec02c4e182d7828a8251", size = 1794058, upload-time = "2026-07-22T18:07:10.644Z" }, + { url = "https://files.pythonhosted.org/packages/21/23/90bb7d0c6f59d2973bb8f4bd3164be00e466823dea01568e0cb36f2afb77/gevent-26.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15fd2d88ed5370f8084079758758df91f26d2f68575e1ee76fce604ddba83e5e", size = 2159797, upload-time = "2026-07-22T17:02:13.229Z" }, + { url = "https://files.pythonhosted.org/packages/a3/67/4d1e315ee3052530fa8537e0cdf1f9c3a6606740f7372f420c7d12d5cf82/gevent-26.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:514bda3fff741d7e5ab108ee1d31550a7f4b2fd3dc6e3b6f38dfb8685efdafaa", size = 1682414, upload-time = "2026-07-22T16:26:22.18Z" }, + { url = "https://files.pythonhosted.org/packages/02/b1/d1b1de89677ee39e641ad8501ed72c2b99f1ddba1c14c462675f40b37a66/gevent-26.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:0f26f9a8c32ac0a73f6084c59b63deeacb350e7f1fee5301d95c5e0683a390d4", size = 1562794, upload-time = "2026-07-22T16:27:53.205Z" }, + { url = "https://files.pythonhosted.org/packages/2b/66/104590ad3a9e671b3ef77ad19c1cc50e7f1c8c220b27ddfaf34e5f88bd9c/gevent-26.7.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:92f256285fb43a57f152bd2e51a59cde1cd0b20869ae1e6da583b6beab88ed8a", size = 2953977, upload-time = "2026-07-22T16:23:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/350d87161378633714184828bdc57c66f9b525eca5249ad1294dc7f8cf58/gevent-26.7.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0e4fea187c5df7168b9538b4f543fcb0fcbaeb93be3d6cd499c324652c740704", size = 1800960, upload-time = "2026-07-22T18:11:27.242Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6c/ddfe298c2ecb1cfc72a03dd69ba751759f7e7835d3135f171b284eb09ed1/gevent-26.7.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:ce732fe08d0ea65de07eff6e46bade8ac6a6fdb65cc748c713f3d31ae122529e", size = 1900387, upload-time = "2026-07-22T18:10:42.973Z" }, + { url = "https://files.pythonhosted.org/packages/f9/89/2647bbbf1da35a1c271a54452e76065308cbaf684ee32a79f989ca4265f6/gevent-26.7.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:c25b3522072137aecf3389031039230190038f888e257f490b3897d0e0620f74", size = 1848046, upload-time = "2026-07-22T18:29:08.074Z" }, + { url = "https://files.pythonhosted.org/packages/31/52/4f9b4c536b5a0424e328d5a5466640d185020f1f0b08b2de0ca939cc0a0b/gevent-26.7.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:eaaa75c9014df3f8c310c64f53f1152af8c6be32e82734396bed91e1d0e6f35c", size = 2132200, upload-time = "2026-07-22T16:48:31.203Z" }, + { url = "https://files.pythonhosted.org/packages/42/88/1daffa63b257c381df68018e4d3da72b429955cf95a171a46d3220422c8d/gevent-26.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f1956032a9926ac9b4152b2a50bc5a2cc020722ec16928ccaf32e227ee0aae47", size = 1814237, upload-time = "2026-07-22T18:07:12.438Z" }, + { url = "https://files.pythonhosted.org/packages/73/4c/b996cdddca78eac435195cd97dff590c65a9e0052640bcb6f8b6f1e30b1d/gevent-26.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d989a1ad6cc54f5c69bb7304360f98b4fda80da2b773f1047db9fba61ae7379a", size = 2157938, upload-time = "2026-07-22T17:02:14.887Z" }, + { url = "https://files.pythonhosted.org/packages/08/fd/44419d7559a95e238ee45d29df30bad78a91d343cb27b13a6af552b907bb/gevent-26.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:e0c9ce2d80fc0f8894d748a1045ff26ad188e294bad656b29839271800827c85", size = 1685319, upload-time = "2026-07-22T16:26:13.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/7a/7df1762ccd9a40ce1ca626f50cb7a75758f2c930aabcc73c91cf00cb3448/gevent-26.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:959effe0c56cdee0bf761e5c4e78ab62880be147a2f2aa31112ca2f7e5754e53", size = 1558945, upload-time = "2026-07-22T16:26:56.737Z" }, + { url = "https://files.pythonhosted.org/packages/75/63/0fcfbe3f5696e56424f331ec41e0e447cea79c384848d6019e3f7f340f4b/gevent-26.7.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b1b89eb5566f75aa8b2bbdb0308e1ac8d9113ca7cff85b45366aea9faad639a1", size = 2976844, upload-time = "2026-07-22T16:24:39.275Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6c/ea2d0afbe760c18df5bd1631dbe5a73d840d9b141cb71e6810157c2ae28a/gevent-26.7.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:449857ce058183442e2d71d83ff0c587a3ddff631e93c6d19a6dffb4814eccad", size = 1802332, upload-time = "2026-07-22T18:11:29.155Z" }, + { url = "https://files.pythonhosted.org/packages/59/90/36f2258f1bfe8601224f6159066103b832c31e1451ce8dc2cd2408b6ecf4/gevent-26.7.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:8260a3f38b05fcf3c283417b18617562dbec74f5784f748e4ba3866789d7f3a4", size = 1901253, upload-time = "2026-07-22T18:10:44.402Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/86dd67e5c2dfab016a9c935b4338d6cf8f9bfa72dc5a0f3fb879d993127a/gevent-26.7.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:30894398d06747b433c8923a6a77ede61259ce6822a99f6c6e7fa0216ccb73c3", size = 1850489, upload-time = "2026-07-22T18:29:09.694Z" }, + { url = "https://files.pythonhosted.org/packages/f1/33/f5651942a5967483298b6ce6f45572d33120dd0fd01c8991d3fca5b1e8ee/gevent-26.7.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0b753522498118c9489753de7c612d4baed0edf384d9df2bf9492233ba1c20ff", size = 2129813, upload-time = "2026-07-22T16:48:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/c0/09/abe8217a8fcd3f0e94c9eec024a5499c0f267cb269ccea6c0e2e812319b9/gevent-26.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:055a643026dc28daff2be228555a2097937448cc9b58307edebcf81b9d78ff4b", size = 1815121, upload-time = "2026-07-22T18:07:13.988Z" }, + { url = "https://files.pythonhosted.org/packages/40/d6/dbae1cd2d27b62664cefa086035530eb21203d45b12f466272441c048c9c/gevent-26.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e1dc6a2712de67fd210e1f1a408601f6908b042f6420e188106f2f37f94ec71", size = 2155913, upload-time = "2026-07-22T17:02:16.35Z" }, + { url = "https://files.pythonhosted.org/packages/fc/42/90b662f4eb27d7727d4619d5c6be872117f1a9f187b243ec7f6fef988ce7/gevent-26.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:44e5280296129c0915addaefdb37d6e9bc124a77a433b1b1c8ddf1853c53f4e7", size = 1682483, upload-time = "2026-07-22T16:26:30.492Z" }, + { url = "https://files.pythonhosted.org/packages/e1/84/7297c56b9fff463c4ba2f685dbb913a855df903046dc68d14e8655a29ffe/gevent-26.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f08b1aa6729f794409ca137e25f671e0d9bbda4451200c5e28a769375365388", size = 1556053, upload-time = "2026-07-22T16:26:14.365Z" }, + { url = "https://files.pythonhosted.org/packages/c1/bb/ab60d496cbdc0293ebbd6c2070b34da0632bd7a2ca20163c17e18d2d2dc9/gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948", size = 2992503, upload-time = "2026-07-22T16:24:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/5c/35/75f27c06a82a5b22600aaccbd9567d89bb4091be43e96c02981f10aff23d/gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480", size = 1809173, upload-time = "2026-07-22T18:11:30.724Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/a6b32b4db3fa76bd8a070f0f46f5306123bf6336e7a0ca0cd2f9b99473df/gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6", size = 1906630, upload-time = "2026-07-22T18:10:45.836Z" }, + { url = "https://files.pythonhosted.org/packages/e1/87/832495d8fcc05ff7432f038b7c4decbd5632425a2cd5da2ce73cb2d800c4/gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0", size = 1855278, upload-time = "2026-07-22T18:29:11.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/2f36c0fa389fa2b7ceb5a8972b0e7da7bc770f9135315cf4246c607ca5fc/gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11", size = 2136155, upload-time = "2026-07-22T16:48:33.865Z" }, + { url = "https://files.pythonhosted.org/packages/b5/98/09f2cfaa23dbce48e3271e95b0d003f93acece6b5cfd40f4cebe3850d79b/gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0", size = 1822108, upload-time = "2026-07-22T18:07:15.397Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d8/05a294165c17569f04284ad3c889684c8780544885b4cdf77b1432947d0c/gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad", size = 2162814, upload-time = "2026-07-22T17:02:18.066Z" }, + { url = "https://files.pythonhosted.org/packages/59/89/58a545c4eda33e106d6887a0387adc2249abc14c779e3eb88bbfdf3768d6/gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a", size = 1706971, upload-time = "2026-07-22T16:27:02.263Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cd/413f293e54961e5c89c54235370e3603ec0f561e7ace8357980410efbf78/gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609", size = 1585078, upload-time = "2026-07-22T16:30:48.145Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/47f29f632aaa38aa12410f57f1732fc50bfd4d4006d2e7e022ce731cabc9/gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404", size = 2996208, upload-time = "2026-07-22T16:23:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b3/4620f1ce81ecec9890229806c73f07dd022e40f76a552bd430391e7316c4/gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17", size = 1811545, upload-time = "2026-07-22T18:11:32.224Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/d285212ffd5585d511299e13e61d76262def8e826e9f20c92cb85df406f2/gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce", size = 1910418, upload-time = "2026-07-22T18:10:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/c5d666e6065652918cfb6e6a3cf8f721d0e152c57cec217ad81792a1323b/gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2", size = 1857891, upload-time = "2026-07-22T18:29:13.13Z" }, + { url = "https://files.pythonhosted.org/packages/a1/67/e945ed458fa98b34572876bfd0d35fe4fa3f1159f43660b71d982b7cb63e/gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4", size = 2138121, upload-time = "2026-07-22T16:48:35.358Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/622468fa1a3c4cf51f20e14813f5cc1592fe6e44a42ccca9195a0b18c769/gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107", size = 1825114, upload-time = "2026-07-22T18:07:16.667Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7a/151a2afcacf487ca25faf8b1bdd6c5b4ace2f7c1e6b4eaffe0a5e6e1df61/gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e", size = 2165990, upload-time = "2026-07-22T17:02:19.817Z" }, ] [[package]] name = "greenlet" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/c1/a82edae11d46c0d83481aacaa1e578fea21d94a1ef400afd734d47ad95ad/greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", size = 185797, upload-time = "2025-05-09T19:47:35.066Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/66/910217271189cc3f32f670040235f4bf026ded8ca07270667d69c06e7324/greenlet-3.2.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", size = 267395, upload-time = "2025-05-09T14:50:45.357Z" }, - { url = "https://files.pythonhosted.org/packages/a8/36/8d812402ca21017c82880f399309afadb78a0aa300a9b45d741e4df5d954/greenlet-3.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", size = 625742, upload-time = "2025-05-09T15:23:58.293Z" }, - { url = "https://files.pythonhosted.org/packages/7b/77/66d7b59dfb7cc1102b2f880bc61cb165ee8998c9ec13c96606ba37e54c77/greenlet-3.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", size = 637014, upload-time = "2025-05-09T15:24:47.025Z" }, - { url = "https://files.pythonhosted.org/packages/36/a7/ff0d408f8086a0d9a5aac47fa1b33a040a9fca89bd5a3f7b54d1cd6e2793/greenlet-3.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", size = 632874, upload-time = "2025-05-09T15:29:20.014Z" }, - { url = "https://files.pythonhosted.org/packages/a1/75/1dc2603bf8184da9ebe69200849c53c3c1dca5b3a3d44d9f5ca06a930550/greenlet-3.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", size = 631652, upload-time = "2025-05-09T14:53:30.961Z" }, - { url = "https://files.pythonhosted.org/packages/7b/74/ddc8c3bd4c2c20548e5bf2b1d2e312a717d44e2eca3eadcfc207b5f5ad80/greenlet-3.2.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", size = 580619, upload-time = "2025-05-09T14:53:42.049Z" }, - { url = "https://files.pythonhosted.org/packages/7e/f2/40f26d7b3077b1c7ae7318a4de1f8ffc1d8ccbad8f1d8979bf5080250fd6/greenlet-3.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", size = 1109809, upload-time = "2025-05-09T15:26:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/c5/21/9329e8c276746b0d2318b696606753f5e7b72d478adcf4ad9a975521ea5f/greenlet-3.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", size = 1133455, upload-time = "2025-05-09T14:53:55.823Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1e/0dca9619dbd736d6981f12f946a497ec21a0ea27262f563bca5729662d4d/greenlet-3.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", size = 294991, upload-time = "2025-05-09T15:05:56.847Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9f/a47e19261747b562ce88219e5ed8c859d42c6e01e73da6fbfa3f08a7be13/greenlet-3.2.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", size = 268635, upload-time = "2025-05-09T14:50:39.007Z" }, - { url = "https://files.pythonhosted.org/packages/11/80/a0042b91b66975f82a914d515e81c1944a3023f2ce1ed7a9b22e10b46919/greenlet-3.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", size = 628786, upload-time = "2025-05-09T15:24:00.692Z" }, - { url = "https://files.pythonhosted.org/packages/38/a2/8336bf1e691013f72a6ebab55da04db81a11f68e82bb691f434909fa1327/greenlet-3.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", size = 640866, upload-time = "2025-05-09T15:24:48.153Z" }, - { url = "https://files.pythonhosted.org/packages/f8/7e/f2a3a13e424670a5d08826dab7468fa5e403e0fbe0b5f951ff1bc4425b45/greenlet-3.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", size = 636752, upload-time = "2025-05-09T15:29:23.182Z" }, - { url = "https://files.pythonhosted.org/packages/fd/5d/ce4a03a36d956dcc29b761283f084eb4a3863401c7cb505f113f73af8774/greenlet-3.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", size = 636028, upload-time = "2025-05-09T14:53:32.854Z" }, - { url = "https://files.pythonhosted.org/packages/4b/29/b130946b57e3ceb039238413790dd3793c5e7b8e14a54968de1fe449a7cf/greenlet-3.2.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", size = 583869, upload-time = "2025-05-09T14:53:43.614Z" }, - { url = "https://files.pythonhosted.org/packages/ac/30/9f538dfe7f87b90ecc75e589d20cbd71635531a617a336c386d775725a8b/greenlet-3.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", size = 1112886, upload-time = "2025-05-09T15:27:01.304Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/4b7deeb1a1e9c32c1b59fdca1cac3175731c23311ddca2ea28a8b6ada91c/greenlet-3.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", size = 1138355, upload-time = "2025-05-09T14:53:58.011Z" }, - { url = "https://files.pythonhosted.org/packages/c5/eb/7551c751a2ea6498907b2fcbe31d7a54b602ba5e8eb9550a9695ca25d25c/greenlet-3.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", size = 295437, upload-time = "2025-05-09T15:00:57.733Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a1/88fdc6ce0df6ad361a30ed78d24c86ea32acb2b563f33e39e927b1da9ea0/greenlet-3.2.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", size = 270413, upload-time = "2025-05-09T14:51:32.455Z" }, - { url = "https://files.pythonhosted.org/packages/a6/2e/6c1caffd65490c68cd9bcec8cb7feb8ac7b27d38ba1fea121fdc1f2331dc/greenlet-3.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", size = 637242, upload-time = "2025-05-09T15:24:02.63Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/088af2cedf8823b6b7ab029a5626302af4ca1037cf8b998bed3a8d3cb9e2/greenlet-3.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", size = 651444, upload-time = "2025-05-09T15:24:49.856Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0116ab876bb0bc7a81eadc21c3f02cd6100dcd25a1cf2a085a130a63a26a/greenlet-3.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", size = 646067, upload-time = "2025-05-09T15:29:24.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/17/bb8f9c9580e28a94a9575da847c257953d5eb6e39ca888239183320c1c28/greenlet-3.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", size = 648153, upload-time = "2025-05-09T14:53:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ee/7f31b6f7021b8df6f7203b53b9cc741b939a2591dcc6d899d8042fcf66f2/greenlet-3.2.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", size = 603865, upload-time = "2025-05-09T14:53:45.738Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2d/759fa59323b521c6f223276a4fc3d3719475dc9ae4c44c2fe7fc750f8de0/greenlet-3.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", size = 1119575, upload-time = "2025-05-09T15:27:04.248Z" }, - { url = "https://files.pythonhosted.org/packages/30/05/356813470060bce0e81c3df63ab8cd1967c1ff6f5189760c1a4734d405ba/greenlet-3.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", size = 1147460, upload-time = "2025-05-09T14:54:00.315Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/b2a26a309a04fb844c7406a4501331b9400e1dd7dd64d3450472fd47d2e1/greenlet-3.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", size = 296239, upload-time = "2025-05-09T14:57:17.633Z" }, - { url = "https://files.pythonhosted.org/packages/89/30/97b49779fff8601af20972a62cc4af0c497c1504dfbb3e93be218e093f21/greenlet-3.2.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", size = 269150, upload-time = "2025-05-09T14:50:30.784Z" }, - { url = "https://files.pythonhosted.org/packages/21/30/877245def4220f684bc2e01df1c2e782c164e84b32e07373992f14a2d107/greenlet-3.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", size = 637381, upload-time = "2025-05-09T15:24:12.893Z" }, - { url = "https://files.pythonhosted.org/packages/8e/16/adf937908e1f913856b5371c1d8bdaef5f58f251d714085abeea73ecc471/greenlet-3.2.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", size = 651427, upload-time = "2025-05-09T15:24:51.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/49/6d79f58fa695b618654adac64e56aff2eeb13344dc28259af8f505662bb1/greenlet-3.2.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", size = 645795, upload-time = "2025-05-09T15:29:26.673Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e6/28ed5cb929c6b2f001e96b1d0698c622976cd8f1e41fe7ebc047fa7c6dd4/greenlet-3.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", size = 648398, upload-time = "2025-05-09T14:53:36.61Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/b200194e25ae86bc57077f695b6cc47ee3118becf54130c5514456cf8dac/greenlet-3.2.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", size = 606795, upload-time = "2025-05-09T14:53:47.039Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c8/ba1def67513a941154ed8f9477ae6e5a03f645be6b507d3930f72ed508d3/greenlet-3.2.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", size = 1117976, upload-time = "2025-05-09T15:27:06.542Z" }, - { url = "https://files.pythonhosted.org/packages/c3/30/d0e88c1cfcc1b3331d63c2b54a0a3a4a950ef202fb8b92e772ca714a9221/greenlet-3.2.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", size = 1145509, upload-time = "2025-05-09T14:54:02.223Z" }, - { url = "https://files.pythonhosted.org/packages/90/2e/59d6491834b6e289051b252cf4776d16da51c7c6ca6a87ff97e3a50aa0cd/greenlet-3.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421", size = 296023, upload-time = "2025-05-09T14:53:24.157Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/8a73aace5a5335a1cba56d0da71b7bd93e450f17d372c5b7c5fa547557e9/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", size = 629911, upload-time = "2025-05-09T15:24:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/48/08/c8b8ebac4e0c95dcc68ec99198842e7db53eda4ab3fb0a4e785690883991/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", size = 635251, upload-time = "2025-05-09T15:24:52.205Z" }, - { url = "https://files.pythonhosted.org/packages/37/26/7db30868f73e86b9125264d2959acabea132b444b88185ba5c462cb8e571/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", size = 632620, upload-time = "2025-05-09T15:29:28.051Z" }, - { url = "https://files.pythonhosted.org/packages/10/ec/718a3bd56249e729016b0b69bee4adea0dfccf6ca43d147ef3b21edbca16/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", size = 628851, upload-time = "2025-05-09T14:53:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/d1c79286a76bc62ccdc1387291464af16a4204ea717f24e77b0acd623b99/greenlet-3.2.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", size = 593718, upload-time = "2025-05-09T14:53:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/cd/41/96ba2bf948f67b245784cd294b84e3d17933597dffd3acdb367a210d1949/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", size = 1105752, upload-time = "2025-05-09T15:27:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/68/3b/3b97f9d33c1f2eb081759da62bd6162159db260f602f048bc2f36b4c453e/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", size = 1125170, upload-time = "2025-05-09T14:54:04.082Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b7d17d66c8d0f578d2885a3d8f565e9e4725eacc9d3fdc946d0031c055c4/greenlet-3.2.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", size = 269899, upload-time = "2025-05-09T14:54:01.581Z" }, +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/9d/58f80897f4121f5c218bb931cf6d3b6514873f02ad0b729f744352926b9f/greenlet-3.5.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190", size = 293072, upload-time = "2026-07-22T11:38:14.299Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9f/b4bc9bbd6a7855cbd8ad8a83c874eeeca56c24de9132b3323f81c03a30ba/greenlet-3.5.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353", size = 609393, upload-time = "2026-07-22T12:26:37.964Z" }, + { url = "https://files.pythonhosted.org/packages/05/0e/744b5e063af127d2e3c74fe0f1aef15573064c83b6066883524f5b258b17/greenlet-3.5.4-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606", size = 622750, upload-time = "2026-07-22T12:28:59.546Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5c/53d6b94742a6f1ee1877c7ff76262c909e137f3f7383ce96a8ab78e1ae31/greenlet-3.5.4-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52", size = 629659, upload-time = "2026-07-22T12:43:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/d6/6b/d78ea2908e8e08985348f28ac396c2950be7ab66321dfe0054c73bd1f456/greenlet-3.5.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7", size = 622920, upload-time = "2026-07-22T11:51:06.83Z" }, + { url = "https://files.pythonhosted.org/packages/f2/34/957fc5577180ef2f57be82580ee1f59fdefad4f6c623c7d5e1b6980a76fb/greenlet-3.5.4-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c", size = 425580, upload-time = "2026-07-22T12:39:48.326Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/3ce7009c948920b01527f8d9da29f501a31ac3d98318829e981fd879b850/greenlet-3.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7", size = 1582262, upload-time = "2026-07-22T12:25:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4c/0408366102a33829f7bdd6a992dad75abbf75e86cc1e76caf19e57311d29/greenlet-3.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df", size = 1648906, upload-time = "2026-07-22T11:51:08.627Z" }, + { url = "https://files.pythonhosted.org/packages/13/52/ebfe8f6a1aeb8e430540b406c844ecc4e3367072b0192f69dcb85eeeec2b/greenlet-3.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616", size = 246036, upload-time = "2026-07-22T11:38:30.073Z" }, + { url = "https://files.pythonhosted.org/packages/61/16/71eefcf68267bbf06a9b6bff57d0b222e49432326e85d74348b67694b8d4/greenlet-3.5.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb", size = 294266, upload-time = "2026-07-22T11:37:56.142Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/a0b19adfc35d07e10acb626e9d22a3893b95f1309c42c4a20161dec16800/greenlet-3.5.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686", size = 613712, upload-time = "2026-07-22T12:26:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a121978b3337407d05a1ce5f79b4aa5998a43a9d8422f9726029b90b4471/greenlet-3.5.4-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7", size = 625582, upload-time = "2026-07-22T12:29:00.814Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4a/f301f1d85c69a86b90b5d581a73e8927bba4e79450037e6e2cbca05eb4fd/greenlet-3.5.4-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7", size = 633429, upload-time = "2026-07-22T12:43:42.073Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/080f16cf870e929e592f55767f01d6c98d2ee83bfdc36c3b892f2d0459ab/greenlet-3.5.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071", size = 624663, upload-time = "2026-07-22T11:51:08.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/26884072b0eb343a4d5fee903341bfe5171b32b7f14553886e2b6349135a/greenlet-3.5.4-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937", size = 428238, upload-time = "2026-07-22T12:39:49.973Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/8f3ca88370b817369008faeceeee85970adc16c92a70a3e5fe5fea495a57/greenlet-3.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72", size = 1585010, upload-time = "2026-07-22T12:25:02.539Z" }, + { url = "https://files.pythonhosted.org/packages/51/c2/45877154689709ebce9a0b83c2235e6ca0f31577889b02af308c8cc5f8fb/greenlet-3.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59", size = 1651283, upload-time = "2026-07-22T11:51:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7d/8711a75cb61d85246277c07ff6e1a6504621ba473d808c11ad225ffca43f/greenlet-3.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6", size = 246434, upload-time = "2026-07-22T11:43:15.557Z" }, + { url = "https://files.pythonhosted.org/packages/00/62/e290b3bce433da8f0324ac02da0b128d683482229f1a8b789fa47818a4cd/greenlet-3.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da", size = 244990, upload-time = "2026-07-22T11:39:22.626Z" }, + { url = "https://files.pythonhosted.org/packages/f3/04/81bd731d6d1e3a469d9a4c36f5eb069bcf0cbb2d5d342c9fec22245b91fc/greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4", size = 295909, upload-time = "2026-07-22T11:38:09.261Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dd/f5f22903a6ae70f5ea328ed0beaec92ad903f0e3b7d2845133b354abc4b8/greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17", size = 612011, upload-time = "2026-07-22T12:26:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/8e/10/92a4a88d12b915d74ea5b6d288e4afefda4771647caa34442c156f7a454f/greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a", size = 624299, upload-time = "2026-07-22T12:29:02.089Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f9/03e26be3487c5238e81f2b84714959a86ea8515a869828cf41f4fc54b34e/greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf", size = 629603, upload-time = "2026-07-22T12:43:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/0b14bb9db2989f32cd9fe7f76afedea01ee8bee3f87c07e69f24adfe7e63/greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f", size = 621541, upload-time = "2026-07-22T11:51:09.464Z" }, + { url = "https://files.pythonhosted.org/packages/57/6b/7c55ca72ef80d57c16c4a55210f82582622462dc4485799a30f4ec6f3372/greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f", size = 432554, upload-time = "2026-07-22T12:39:51.379Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/25e9a2d9eb6b2e8b7ca4e80a3a26cb887cce6c8e0a87c921164f11bc5574/greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d", size = 1581444, upload-time = "2026-07-22T12:25:03.818Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/4c9bf2e2c408dcc0556edce69efa9f802e82223573c53240136a086821f1/greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9", size = 1645842, upload-time = "2026-07-22T11:51:12.295Z" }, + { url = "https://files.pythonhosted.org/packages/b5/41/303ecb26a3a56122c0f4d4073ee078881847bd6b6f463ae0ec57ec20223b/greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3", size = 247169, upload-time = "2026-07-22T11:38:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/ef56864b4c35fcb3eb3b41b869f6cc46f4cd3f5e2c68e74acde8ac433951/greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0", size = 245565, upload-time = "2026-07-22T11:38:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, + { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, + { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, + { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, + { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, + { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, + { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, + { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, + { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, + { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, ] [[package]] @@ -582,6 +682,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z" }, ] +[[package]] +name = "hypothesis" +version = "6.161.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/94/d208ced653376e7e0a2f0429ee5be864dd0b59393b98a8b41a35ceb4d035/hypothesis-6.161.5.tar.gz", hash = "sha256:ba73a3c3b68e63a0bee5ea1a8a13efce60bcc7ee5fc7e71df2954db39c225b95", size = 486653, upload-time = "2026-07-25T14:39:34.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/58/e48aa878d119474631fc097d511b5e70807dfe56be4b244cce0275f1805e/hypothesis-6.161.5-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4c89e0c35d7cd70af2a0a5f0b5ad69d1898c369adf15115c6d4271d47bf1b280", size = 766970, upload-time = "2026-07-25T14:38:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ff/c83c1a4d5d3c4b6a4dc1fcb5069245171b7a30ad5dd31f44282e8881e089/hypothesis-6.161.5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1bb987e519a6d00675bab124c925000637e2e59384196b0ded5d108dc1851449", size = 762574, upload-time = "2026-07-25T14:38:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/d2/04/7a5ba16cd14340009d1d8aa46083bf3a3a55c5b0d1ccf553e6f6748ee0e8/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5990e9e2e145ff5369c95ad684bd6eee7ebd2d4de37d37eea4c6ca5e339e2a52", size = 1091754, upload-time = "2026-07-25T14:38:38.562Z" }, + { url = "https://files.pythonhosted.org/packages/70/69/031913f7408ebb41e31a9b20e5bca75c5a64ab151d57ca75e944a09ba6e0/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2a5c6400c58c9c3d616ad7177ada12ae577e5103b3b0df6e79920729250bf100", size = 1120372, upload-time = "2026-07-25T14:38:18.962Z" }, + { url = "https://files.pythonhosted.org/packages/b9/95/360884e86ab99099340fca781531286c26d40ce32c853097e76537cc0726/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71aa718e08bdfbacd1a5d8f86ffd55f1d86e64a57cec6a107144d883b522823e", size = 1141230, upload-time = "2026-07-25T14:39:03.757Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5b/6f3c9fbe9191432f33c9aaf951cf5a5416533848999f2e2c6a20ec00098b/hypothesis-6.161.5-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:66912038467e1af791f76336bc079d669f7c8bbf7aeb85bdd52e83259d885122", size = 1096611, upload-time = "2026-07-25T14:38:34.688Z" }, + { url = "https://files.pythonhosted.org/packages/32/c7/d126b9f66295fed5d5745f98ad92f485c98dee303dd67ea7a53fe4a903aa/hypothesis-6.161.5-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99d8b2380d0bba602df963d6e42d56bb50cf1734376c1b9b14dcab3cec21814e", size = 1133382, upload-time = "2026-07-25T14:38:20.27Z" }, + { url = "https://files.pythonhosted.org/packages/56/de/277a17091687f298079c197cadd806d64908c5a08c9c91bb27b834192503/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:19e57e621c6abd98123a91bd4b022bde7760ba709f1ca121f822d9aa291bd001", size = 1265592, upload-time = "2026-07-25T14:38:11.734Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/3f15b1a43cb70b223427ce3a9a6afbe430bf1754b3346ac546a3df38b96b/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b74cced62995ed2e0096fa63d0b7babd1fa08ce1944cff41134275516848708", size = 1393424, upload-time = "2026-07-25T14:38:46.738Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1d/a3cf1441550dbf5a362dcfa19c831721f3bb6a749eb53ecbdfc983959027/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:fd37a0257647288be8c27d56a78a31da2b65e3b6254511f03164f5e1c786187a", size = 1266197, upload-time = "2026-07-25T14:39:33.052Z" }, + { url = "https://files.pythonhosted.org/packages/57/e5/2616432d6144057b6c8f4409c95d95610bdbdf07fecb6618e98761861c24/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf9dfba054164e481f05774c70b65ea88ba735b986cdc44f5210ef40871a32e9", size = 1308330, upload-time = "2026-07-25T14:39:27.86Z" }, + { url = "https://files.pythonhosted.org/packages/f7/cd/9e9c7a88858f84447e44e45c8b9fd1058120b4d5fa9bab9c3ceb9b9cf96b/hypothesis-6.161.5-cp310-abi3-win32.whl", hash = "sha256:8763938787e6c98e461f8c11139981597d083e6915a9956db1c26cc609bc7b19", size = 652819, upload-time = "2026-07-25T14:38:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/03/23/aae37b5bb2014e0e01e372498f7f5977b9fd61a0179143854b7fb0538b1c/hypothesis-6.161.5-cp310-abi3-win_amd64.whl", hash = "sha256:79b5d069095a726dca5b6b7e4c5d268acc809a6595c46d7eee860708f960cb8f", size = 658970, upload-time = "2026-07-25T14:39:05.239Z" }, + { url = "https://files.pythonhosted.org/packages/70/77/fbda7005f891f534af0a74cb5acd7a67cb1a7fbee1ae170d176c7dcd5e0a/hypothesis-6.161.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c73c86afe87b1bf81af586402d41bff69d93c8c34517a81d3a88a1e28e8662e2", size = 767688, upload-time = "2026-07-25T14:39:13.418Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8d/f884e10029018504504df6100faf3795918e1dd14d4b6f4102c3fcb21acf/hypothesis-6.161.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b289d33f0154850baed802d088564bba83dae58bae6fb9157e498c43d758afd", size = 763399, upload-time = "2026-07-25T14:38:10.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/75d9c1f7ad8a52a14c5981c9ba97b1d4abfd263cd48ce7ffb036ce1f5a16/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60afc424627a4119d89f02425049a97e500e022aef6a06d76e987e266e3d720", size = 1092278, upload-time = "2026-07-25T14:38:51.187Z" }, + { url = "https://files.pythonhosted.org/packages/be/52/efc7c45352636fc71d24590893e52f0e8b4ca42984aea870b47831720fef/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:defa4f24aa62b3a6d07271a4d54f31c7bb07ce1f3fa86cc0705890a071a58301", size = 1141822, upload-time = "2026-07-25T14:38:48.298Z" }, + { url = "https://files.pythonhosted.org/packages/6b/48/697b194412fed03cbf9757771b9c3b51ca062e1d78fe2b5ddf833a25b7f2/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f5e8be93cf0da10b8a2f1044c12f2aa506e80c5071e84dffeec4311fddeb8d7", size = 1266206, upload-time = "2026-07-25T14:39:31.414Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b8/56c7996272a8310737c44378fe181befbe3ab41f72ee3d7180e32269cb7c/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:46aa94fa4107f97760b63c83142cd2fe208f38a6708736494cabfcf219b137cb", size = 1308570, upload-time = "2026-07-25T14:39:09.772Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ba/ed5baa948085a08ed38362e0e88ddad7a4a524b0a2a1581c8138ce7b307a/hypothesis-6.161.5-cp310-cp310-win_amd64.whl", hash = "sha256:44e25c35123a2b77ebd2df356e1a78bf2c9abbf875222ff252b3012801bfb143", size = 658822, upload-time = "2026-07-25T14:38:45.402Z" }, + { url = "https://files.pythonhosted.org/packages/9d/82/60f7213dd5262863646bbf31ca28e1dba68730080c14f39744d110733932/hypothesis-6.161.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6d55821d13890875d5a50a1433448c8a86f7f83e85103a527dad232d18c470e", size = 767446, upload-time = "2026-07-25T14:38:39.904Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/145fe198d155ac394cac067ae852074adbca2c015a7244fdb3df2f655c19/hypothesis-6.161.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecb5878b81beb1dfda4e573375a816b1ffa7931d7899a1a2d7216afa5e1efb4f", size = 763215, upload-time = "2026-07-25T14:38:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c0/0ea73426ba5b2504d4e3f4e6c7589f5808ad6eef4c922f15766c0537bbc2/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bff9062c1af02e391c0e48aa75f989393651c6e07f1b50eddeadaa8ca440f944", size = 1092115, upload-time = "2026-07-25T14:39:18.121Z" }, + { url = "https://files.pythonhosted.org/packages/dc/65/2a52daac1d39d0a1a88f96a602ff997e4665c5a9acc4d029e3168e536cd1/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:570ab5afc34dd7a78e4d1ab1a0ea4e026bbae6caee79dc04245d0347f1d548a7", size = 1141577, upload-time = "2026-07-25T14:38:55.841Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/77693dbb6766345d980fdfa41f7e7d3d0a676469d156c8e9455c1ebda67d/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7541433984c5b5fb4ad433a715808579cba16ebaca6a214a99d2f8c35ed49d", size = 1265879, upload-time = "2026-07-25T14:38:21.759Z" }, + { url = "https://files.pythonhosted.org/packages/ed/af/ea72a466210a43b35f3e9d86350aca94529a00d2aff698b4cdd490238955/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f3118cce6e66366e2c64389a5dd6e9bc49c697c366d62709e6fda0b3a8a7d997", size = 1308593, upload-time = "2026-07-25T14:38:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/50/8c/d0ae6738baed9061ca29b18de506f5bc8dc84649b216b55213474ec7da9b/hypothesis-6.161.5-cp311-cp311-win_amd64.whl", hash = "sha256:d8cbd7c938b191d5f9bf846a79e81484135a239cd45bf194993949645495ee6f", size = 658671, upload-time = "2026-07-25T14:38:54.453Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/71a53545f2732574bdd7a45a6e073043bce3447fec4aa722211ebf742f2e/hypothesis-6.161.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:37016b6b4842993b06ee453dda4dedac5ef328cefa6daa36372d261abf12db43", size = 768565, upload-time = "2026-07-25T14:38:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/e6/56/7f32ba1443d5819b324444e7d5ebcf207ba4a0f9219a1693a7994293fd3f/hypothesis-6.161.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d60a204b86936b29d8914f93a99c59a2ecd4e311be793bf32a3f94774d41e016", size = 760188, upload-time = "2026-07-25T14:38:07.894Z" }, + { url = "https://files.pythonhosted.org/packages/07/0d/699436929d2980103c9ddba153872ebed55cfcc13f0f78c1741ac6128205/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cee9f1a563830a9137b9d1505c9c4fac760bc43dfbfa5873bebefe36c8dca4ec", size = 1090570, upload-time = "2026-07-25T14:39:00.492Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a5/2c28b58d16443fd8ad63e4f287440291a42135073748e5e511084f32a6f3/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7fcf44f9876b7b4fdc40b607342219c634cf6a1acc708dc0ff88e8e48ae8ea2", size = 1140601, upload-time = "2026-07-25T14:39:21.403Z" }, + { url = "https://files.pythonhosted.org/packages/4e/fe/f0f87c38dc741687bace80553478f8c9796ba5b6e68793a80990d79ec5ac/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:59088f6458d2a6ef04724a2a639418c97dd8b8c280bfd222fcc5566854560caf", size = 1263341, upload-time = "2026-07-25T14:38:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/02e30bcdb3434f4eee8fdbebf5cf151fecd37d2a76f2fc19f2745c93ca38/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea8e7e6bed5407ab902026d8cac74d10cac894a496d82e261a0beaca499114dc", size = 1307604, upload-time = "2026-07-25T14:39:19.867Z" }, + { url = "https://files.pythonhosted.org/packages/1d/70/1f6349a244f13c5a383a62ca674d3175018c7fc30fbaad10faa859d5c2cf/hypothesis-6.161.5-cp312-cp312-win_amd64.whl", hash = "sha256:a4ad11f4eafd561a672cb9fa977d0f58ab4ae9cf4b160dd3c12c351089f3e2be", size = 656103, upload-time = "2026-07-25T14:38:57.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/07/86f98ebd945f3f8a821c1e7bf9b530902675145d93ed44fa56a309cc24fd/hypothesis-6.161.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3b4c308ff19741ab9f795cac61cce61227f55a1b9767ad525a34de8b01391d1d", size = 768455, upload-time = "2026-07-25T14:38:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/70/11/93c00ba5c77b886017ea8eaa42de8ca937e319032c4251b0540fc1838ae1/hypothesis-6.161.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef16fac46cd5675504a4d3824f443f6842b80fae8b6d15ecffa9c90e0fcf4522", size = 760099, upload-time = "2026-07-25T14:38:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/86/13/c8787cfae81c816976c1b3aaa43294c3abd6fd055a0147122edd7357a349/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4c6aab8dbabb98b9c5ca8da8fa9294d2e5173432a12abcb447e83e666fa430", size = 1090486, upload-time = "2026-07-25T14:39:26.243Z" }, + { url = "https://files.pythonhosted.org/packages/f8/be/48f4f56bfb24787aa27e5ae851d1aee5214ab6e95311cf8c88a56707941f/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f485e3c7ead1f76a8c060b16c6b50e5615264819e95b794c6fc6882a9cbfdc4", size = 1140433, upload-time = "2026-07-25T14:38:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/50/c119bfec24d267e4af5bdefda8fd490f7b3a7ee36a52d5ac0b88e07c10b3/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:75bb899b2db5c45dbfa6ef704c26259bfd27fb3418179c3bd5788b7f2ecce72d", size = 1263296, upload-time = "2026-07-25T14:38:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/81/89/eeb97a9684e3eaae801dd538058202b0b65c2aaecea0211e17d79f731170/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:894e5d9e2cd97798c3dc2731699098c7f800a7905572db1677e0d27d3b41f541", size = 1307324, upload-time = "2026-07-25T14:38:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/15/9d/e0c64a64721ba169dff5a6f12236d5102f8c76f661598d7837591ec7b375/hypothesis-6.161.5-cp313-cp313-win_amd64.whl", hash = "sha256:62452bb19d73496a74919d82afc6306bf9bd42b8cf1dfce21da3802c596a59b1", size = 656067, upload-time = "2026-07-25T14:38:37.157Z" }, + { url = "https://files.pythonhosted.org/packages/31/27/0c6884785ab6afb41305296b2a5fac2c5d1d26dba40e85c70909b8692fce/hypothesis-6.161.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b41648b547a233dd89bed37a3ce8c8298a454fd7133b27b0a37ced135eddfe", size = 768668, upload-time = "2026-07-25T14:39:15.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/17c63bba85201d5fa9db89841e8548c4fa6b240b2437c80d606a65889eb6/hypothesis-6.161.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:429d2179b01787a54f8ccd150c02077a6feb973f14ef31664227148c6a1fadff", size = 760240, upload-time = "2026-07-25T14:38:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/a2/df/1084fd695a1516120a5ea26c026d9f0a071fdf9efcba774c94c09332b372/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b23fc5441ef297d56797dd372889dec38ddf3662468772b347db2fd8fd43eced", size = 1090987, upload-time = "2026-07-25T14:39:06.646Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a3/bef99daeaf6d1d2db09790cd4ec9ce0cdcac52faaf1fd80b4eaba5d5ea5e/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f42dcc7fee0d4d56d011218b8c4d5afa6cdee5dbc690652d2a1a598d21969a3", size = 1140613, upload-time = "2026-07-25T14:38:26.284Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ce1f6e29d897c7dcb7de1665a9e30d1e00500933caec697a4338db9e7bd0/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2825fac0d0428cb5e7826347aa4e60106d28f32948cd58c837097f3bd96dbf6c", size = 1263802, upload-time = "2026-07-25T14:38:33.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/e5/7b33aae590bc457fffba70c4f19cc150e3ca356f8e15322f9224c5372a64/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e9f859ecfd3e00ebb8a36fbb36a8951513fd8e09103a56c645b102e0a51dd1bc", size = 1307642, upload-time = "2026-07-25T14:39:29.729Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8b/4156a9e70bf8c69f54954539ca805e7311048516b665c5f87bee2c1955b7/hypothesis-6.161.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b2647c2d5da341467c1ffaad0dd86d8612e27da2419e76b3b6ee79a33bba7d86", size = 600146, upload-time = "2026-07-25T14:38:17.815Z" }, + { url = "https://files.pythonhosted.org/packages/f9/db/48a518f3f2facd7b280592f81dbb0ba09109656be4ad3dbf0d95a02b4c02/hypothesis-6.161.5-cp314-cp314-win_amd64.whl", hash = "sha256:7c40dd1a3e99497d48a3cfd4c5d71bdb0cd70402a9e09eceb160f045edc92b3a", size = 655981, upload-time = "2026-07-25T14:39:16.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/92/1e9f9f44ef75fc052cc0e3700cfc7d52fd4af1ed9e8b2945d19956bb83cb/hypothesis-6.161.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6c1b7e1d508a3cc5c10ea7a7a4a7d3b0b56fe6291e8936bb9d052af530380b15", size = 767251, upload-time = "2026-07-25T14:38:52.608Z" }, + { url = "https://files.pythonhosted.org/packages/48/31/0c3f824bbb1fccc0338930c8ef855880065d9d58dddf9547a1b9ae4e664d/hypothesis-6.161.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b0926ba51452c24b92fbcbb30d28cc43fd6cd9e636ea2134fbc1cc2c9da109de", size = 758710, upload-time = "2026-07-25T14:38:09.156Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/5811dbb4c1dc0a772519fb9af9c4089128bceeb5deb4c36a887e1e731920/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba8d0346308b558bfbb49f965ed1d9d5bb65758f4530cb30ce1f9bd911a130b4", size = 1089583, upload-time = "2026-07-25T14:38:49.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/29/f9cbfd9d0aaea5370a7ceb966f6c8d47e9101ee9a6623606e9f564a59c6e/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c64079ef4633ab7b540f532107188d75999ee5b342116636312e09ada15783e", size = 1139525, upload-time = "2026-07-25T14:38:29.262Z" }, + { url = "https://files.pythonhosted.org/packages/99/27/f56a03be4ee21f25828e8fba8a502d6840826aea4ff784ff54cbd1f51175/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed3eede1c23689edb4773b4c2303ee91518e82056a0a4893fc8415b18e5c3e8b", size = 1261963, upload-time = "2026-07-25T14:38:31.931Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ea/72e11fe6832a68674271655be5cb6f035502dd328ea629b3ee510fb130a5/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82b1088ebcad5ab6d146caf9a1552019472a8a9302727f1be5939e30460a101b", size = 1306382, upload-time = "2026-07-25T14:38:13.011Z" }, + { url = "https://files.pythonhosted.org/packages/df/8b/bb039d11a805db0977904c245fa3dcf6d266808482d9dfc6dad3e5f1b256/hypothesis-6.161.5-cp314-cp314t-win_amd64.whl", hash = "sha256:bbe8705ff394a573624cd53f2192dad6255c86346704adf9873cb3acb9b940e5", size = 656131, upload-time = "2026-07-25T14:38:16.712Z" }, + { url = "https://files.pythonhosted.org/packages/21/71/0b28e9ac10f692d9b85a0df74615d308ec1e77140c61773ab107070b82d3/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:64167d94bcd4a15c1b0aa44c3176f019614898ff10907a2065c142ed34acf828", size = 768375, upload-time = "2026-07-25T14:39:23Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/f518bdc85ee5a9b9153aa01b3ed6a77304b8f573a461da1a6694a6992001/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:22bc641a290428cfd01c62b2b7fe7686007b104dfb5e8ed5420fd3d3a281ebb9", size = 764276, upload-time = "2026-07-25T14:39:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/af/3b/6c93b03adb5804b01854d2801a1f74705ea73d685cef7861484ad8804856/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35527a795992396b82293fec2294ba5b9c0768350c8cb89085e3f5f76ed7d08e", size = 1093089, upload-time = "2026-07-25T14:39:11.442Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/6ebfbd76534a71c7ffdd48e0d42373a14152e45714feabf0c25bd7d6b10a/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae81035628d2b80435320c43cb09f5b421bac4941bd8ea9b2778d278ae65996", size = 1142876, upload-time = "2026-07-25T14:39:24.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/b57cfd34f55c14a2e823284a796d4228c356e338e1195affc0546bca567b/hypothesis-6.161.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:20e3ab3b0dc300a84b58f41ddd7f5190757521282aad58701c4c76f28a3d981a", size = 659768, upload-time = "2026-07-25T14:39:08.166Z" }, +] + [[package]] name = "identify" version = "2.6.12" @@ -723,6 +896,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -866,6 +1051,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.5.0" @@ -960,21 +1158,30 @@ wheels = [ ] [[package]] -name = "setuptools" -version = "80.8.0" +name = "shellingham" +version = "1.5.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/d2/ec1acaaff45caed5c2dedb33b67055ba9d4e96b091094df90762e60135fe/setuptools-80.8.0.tar.gz", hash = "sha256:49f7af965996f26d43c8ae34539c8d99c5042fbff34302ea151eaa9c207cd257", size = 1319720, upload-time = "2025-05-20T14:02:53.503Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/29/93c53c098d301132196c3238c312825324740851d77a8500a2462c0fd888/setuptools-80.8.0-py3-none-any.whl", hash = "sha256:95a60484590d24103af13b686121328cc2736bee85de8936383111e421b9edc0", size = 1201470, upload-time = "2025-05-20T14:02:51.348Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] [[package]] -name = "shellingham" -version = "1.5.4" +name = "sniffio" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] [[package]] @@ -1072,6 +1279,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/3d/7ba55871e9d794d40b6c8424f2e5d1b267ea5d9a4bd2175e08b57960ba13/tox-4.58.0-py3-none-any.whl", hash = "sha256:dcae21f5f015f3a67658e35644cce0d1aa0dedcd06f3927f95d84e1717f6cea5", size = 223298, upload-time = "2026-07-21T13:10:34.731Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + [[package]] name = "trove-classifiers" version = "2025.5.9.12" @@ -1164,47 +1389,57 @@ wheels = [ [[package]] name = "zope-event" -version = "5.0" +version = "6.2" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c2/427f1867bb96555d1d34342f1dd97f8c420966ab564d58d18469a1db8736/zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd", size = 17350, upload-time = "2023-06-23T06:28:35.709Z" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/42/f8dbc2b9ad59e927940325a22d6d3931d630c3644dae7e2369ef5d9ba230/zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", size = 6824, upload-time = "2023-06-23T06:28:32.652Z" }, + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, ] [[package]] name = "zope-interface" -version = "7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/71/e6177f390e8daa7e75378505c5ab974e0bf59c1d3b19155638c7afbf4b2d/zope.interface-7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce290e62229964715f1011c3dbeab7a4a1e4971fd6f31324c4519464473ef9f2", size = 208243, upload-time = "2024-11-28T08:47:29.781Z" }, - { url = "https://files.pythonhosted.org/packages/52/db/7e5f4226bef540f6d55acfd95cd105782bc6ee044d9b5587ce2c95558a5e/zope.interface-7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05b910a5afe03256b58ab2ba6288960a2892dfeef01336dc4be6f1b9ed02ab0a", size = 208759, upload-time = "2024-11-28T08:47:31.908Z" }, - { url = "https://files.pythonhosted.org/packages/28/ea/fdd9813c1eafd333ad92464d57a4e3a82b37ae57c19497bcffa42df673e4/zope.interface-7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550f1c6588ecc368c9ce13c44a49b8d6b6f3ca7588873c679bd8fd88a1b557b6", size = 254922, upload-time = "2024-11-28T09:18:11.795Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d3/0000a4d497ef9fbf4f66bb6828b8d0a235e690d57c333be877bec763722f/zope.interface-7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ef9e2f865721553c6f22a9ff97da0f0216c074bd02b25cf0d3af60ea4d6931d", size = 249367, upload-time = "2024-11-28T08:48:24.238Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e5/0b359e99084f033d413419eff23ee9c2bd33bca2ca9f4e83d11856f22d10/zope.interface-7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f926f0dcb058211a3bb3e0e501c69759613b17a553788b2caeb991bed3b61d", size = 254488, upload-time = "2024-11-28T08:48:28.816Z" }, - { url = "https://files.pythonhosted.org/packages/7b/90/12d50b95f40e3b2fc0ba7f7782104093b9fd62806b13b98ef4e580f2ca61/zope.interface-7.2-cp310-cp310-win_amd64.whl", hash = "sha256:144964649eba4c5e4410bb0ee290d338e78f179cdbfd15813de1a664e7649b3b", size = 211947, upload-time = "2024-11-28T08:48:18.831Z" }, - { url = "https://files.pythonhosted.org/packages/98/7d/2e8daf0abea7798d16a58f2f3a2bf7588872eee54ac119f99393fdd47b65/zope.interface-7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1909f52a00c8c3dcab6c4fad5d13de2285a4b3c7be063b239b8dc15ddfb73bd2", size = 208776, upload-time = "2024-11-28T08:47:53.009Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2a/0c03c7170fe61d0d371e4c7ea5b62b8cb79b095b3d630ca16719bf8b7b18/zope.interface-7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80ecf2451596f19fd607bb09953f426588fc1e79e93f5968ecf3367550396b22", size = 209296, upload-time = "2024-11-28T08:47:57.993Z" }, - { url = "https://files.pythonhosted.org/packages/49/b4/451f19448772b4a1159519033a5f72672221e623b0a1bd2b896b653943d8/zope.interface-7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:033b3923b63474800b04cba480b70f6e6243a62208071fc148354f3f89cc01b7", size = 260997, upload-time = "2024-11-28T09:18:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/5aa4461c10718062c8f8711161faf3249d6d3679c24a0b81dd6fc8ba1dd3/zope.interface-7.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a102424e28c6b47c67923a1f337ede4a4c2bba3965b01cf707978a801fc7442c", size = 255038, upload-time = "2024-11-28T08:48:26.381Z" }, - { url = "https://files.pythonhosted.org/packages/9f/aa/1a28c02815fe1ca282b54f6705b9ddba20328fabdc37b8cf73fc06b172f0/zope.interface-7.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25e6a61dcb184453bb00eafa733169ab6d903e46f5c2ace4ad275386f9ab327a", size = 259806, upload-time = "2024-11-28T08:48:30.78Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2c/82028f121d27c7e68632347fe04f4a6e0466e77bb36e104c8b074f3d7d7b/zope.interface-7.2-cp311-cp311-win_amd64.whl", hash = "sha256:3f6771d1647b1fc543d37640b45c06b34832a943c80d1db214a37c31161a93f1", size = 212305, upload-time = "2024-11-28T08:49:14.525Z" }, - { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, - { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3b/e309d731712c1a1866d61b5356a069dd44e5b01e394b6cb49848fa2efbff/zope.interface-7.2-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:3e0350b51e88658d5ad126c6a57502b19d5f559f6cb0a628e3dc90442b53dd98", size = 208961, upload-time = "2024-11-28T08:48:29.865Z" }, - { url = "https://files.pythonhosted.org/packages/49/65/78e7cebca6be07c8fc4032bfbb123e500d60efdf7b86727bb8a071992108/zope.interface-7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15398c000c094b8855d7d74f4fdc9e73aa02d4d0d5c775acdef98cdb1119768d", size = 209356, upload-time = "2024-11-28T08:48:33.297Z" }, - { url = "https://files.pythonhosted.org/packages/11/b1/627384b745310d082d29e3695db5f5a9188186676912c14b61a78bbc6afe/zope.interface-7.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:802176a9f99bd8cc276dcd3b8512808716492f6f557c11196d42e26c01a69a4c", size = 264196, upload-time = "2024-11-28T09:18:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f6/54548df6dc73e30ac6c8a7ff1da73ac9007ba38f866397091d5a82237bd3/zope.interface-7.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb23f58a446a7f09db85eda09521a498e109f137b85fb278edb2e34841055398", size = 259237, upload-time = "2024-11-28T08:48:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/b6/66/ac05b741c2129fdf668b85631d2268421c5cd1a9ff99be1674371139d665/zope.interface-7.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a71a5b541078d0ebe373a81a3b7e71432c61d12e660f1d67896ca62d9628045b", size = 264696, upload-time = "2024-11-28T08:48:41.161Z" }, - { url = "https://files.pythonhosted.org/packages/0a/2f/1bccc6f4cc882662162a1158cda1a7f616add2ffe322b28c99cb031b4ffc/zope.interface-7.2-cp313-cp313-win_amd64.whl", hash = "sha256:4893395d5dd2ba655c38ceb13014fd65667740f09fa5bb01caa1e6284e48c0cd", size = 212472, upload-time = "2024-11-28T08:49:56.587Z" }, +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/43/9cd98bee951d23848de690ba2809f87e3b22c67c370987acc960da15ad37/zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe", size = 210951, upload-time = "2026-05-26T06:49:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/8f1a29966bcf863e3a2121edcafb81c55715de7886bcc9544749cc79e7da/zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c", size = 211309, upload-time = "2026-05-26T06:49:02.732Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9f/37e564eaaf85e3abc1ada40a79fa43f2ab45bdb67431b0ec0fe29e4763e2/zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37", size = 254881, upload-time = "2026-05-26T06:49:04.303Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/e6501d8ea7a2cac3217e03f404e1f98c1df7191d83cfe86b1895fbba5dac/zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a", size = 259811, upload-time = "2026-05-26T06:49:06.373Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/bfa25ef480b02af6e9452c478483fec75e87c9e2b60c407fd0b1f6054b9c/zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58", size = 260358, upload-time = "2026-05-26T06:49:08.317Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/2b518072fea76242da64451d501c69b7b5ccdef9b57fead584ccf1c180d5/zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170", size = 214822, upload-time = "2026-05-26T06:49:10.441Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/83ad110fb847413affe71609bb50e59e1aa082e1236030122227c7c283d3/zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057", size = 211426, upload-time = "2026-05-26T06:49:12.595Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a7/6b6e0c31ac240cb9fc015ae9ed45ca54be886c18fcf7bfa2377a4d7a8785/zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f", size = 211850, upload-time = "2026-05-26T06:49:14.474Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/7599ecabcf80ce4fef2e1ef3c5ac0d4696b61f03f724cc44022f4d226af9/zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376", size = 260711, upload-time = "2026-05-26T06:49:16.568Z" }, + { url = "https://files.pythonhosted.org/packages/03/3e/1774b0ee46ccbb5498ee3c33ece40315b6ef58bc71957be94bd345340bc1/zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f", size = 265277, upload-time = "2026-05-26T06:49:18.656Z" }, + { url = "https://files.pythonhosted.org/packages/b6/09/e533b2ffabaae4e5d5730d6768a591cf335defe8e37bec2ad905d09be656/zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2", size = 266369, upload-time = "2026-05-26T06:49:20.174Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/3ebe6a4c122b2d5340db45cbe7e490663d3228b172710ec71060cd5d541e/zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955", size = 215161, upload-time = "2026-05-26T06:49:21.799Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/056ad97af5b16db1975ee98ec7ab03d2ce3f3355efad904ced1dbce0e39f/zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646", size = 213481, upload-time = "2026-05-26T06:49:25.085Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" }, + { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, + { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" }, + { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" }, + { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" }, + { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" }, ]