From 78eba7ae4cc400b15f58b05638ea603aa03027f3 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 10:15:03 +0200 Subject: [PATCH] Add isolated context mode: CPython in a child OS process A mode where Python code cannot take the node down or run forever: one child process per context over a Unix socket, signals with SIGKILL as backstop, rlimits and cgroups, restart on crash. Same public API; the context process is a gen_statem serving one caller at a time. Validated on macOS and FreeBSD. Version 4.2.0. --- CHANGELOG.md | 39 + README.md | 13 + c_src/py_nif.c | 29 + docs/interrupts.md | 19 +- docs/isolated.md | 267 +++++++ docs/security.md | 17 + docs/workers.md | 5 + priv/_erlang_impl/_etf.py | 477 ++++++++++++ priv/_erlang_impl/_isolated.py | 824 ++++++++++++++++++++ priv/py_isolated_child.py | 222 ++++++ rebar.config | 2 + src/erlang_python.app.src | 2 +- src/py.erl | 15 + src/py_context.erl | 118 ++- src/py_isolated.erl | 1193 +++++++++++++++++++++++++++++ src/py_nif.erl | 6 + test/py_isolated_SUITE.erl | 874 +++++++++++++++++++++ test/py_isolated_async_SUITE.erl | 516 +++++++++++++ test/py_isolated_soak_SUITE.erl | 277 +++++++ test/py_isolated_stress_SUITE.erl | 162 ++++ test/py_isolated_vm_SUITE.erl | 477 ++++++++++++ test/py_test_isolated.py | 374 +++++++++ 22 files changed, 5920 insertions(+), 8 deletions(-) create mode 100644 docs/isolated.md create mode 100644 priv/_erlang_impl/_etf.py create mode 100644 priv/_erlang_impl/_isolated.py create mode 100644 priv/py_isolated_child.py create mode 100644 src/py_isolated.erl create mode 100644 test/py_isolated_SUITE.erl create mode 100644 test/py_isolated_async_SUITE.erl create mode 100644 test/py_isolated_soak_SUITE.erl create mode 100644 test/py_isolated_stress_SUITE.erl create mode 100644 test/py_isolated_vm_SUITE.erl create mode 100644 test/py_test_isolated.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ddd25..ad58ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## 4.2.0 (2026-08-29) + +### Added + +- **`isolated` context mode** - `py_context:new(#{mode => isolated})` runs + CPython in a child OS process per context, with the same `call/eval/exec`, + callback, `erlang.send`/`whereis`, worker-loop and pool API as the embedded + modes. It is the first mode with a hard bound: `py_context:interrupt/1` + stops a blocking C call (a signal in the child) and `SIGKILL` is the + backstop after `kill_after` ms; `py_context:kill/1` kills at once. `rlimits` + (`as`, `cpu`, `nofile`) and a cgroup v2 directory bound the child; a + segfault in a C extension returns `{error, {child_exited, {signal, 11}}}` + and the node survives. The child restarts on crash within a budget + (`restart`, `max_restarts`, `restart_period`); `py_context:child_info/1` + reports its OS pid. Children are reaped by the VM and exit when the BEAM + dies (socket EOF watchdog, `PR_SET_PDEATHSIG` on Linux, `PROC_PDEATHSIG_CTL` + on FreeBSD). `cgroup` is refused outside Linux; rlimits apply everywhere: + `as` is kernel-enforced on Linux and FreeBSD and enforced by an RSS + watchdog in the child on macOS (`{child_exited, {memory_limit, Bytes}}`). + Validated on macOS (arm64) and FreeBSD 14.3 (OTP 28, Python 3.11). +- **`py_context:pass_fd/2`** - hands a file descriptor to an isolated child + over the control socket (`SCM_RIGHTS`), so `erlang.server.serve` works out + of process: Erlang binds once, N killable children accept. +- **Pure-Python ETF codec** (`priv/_erlang_impl/_etf.py`) with the type + mapping of `py_convert.c`; the child needs no C extension. Integers beyond + 64 bits round-trip exactly in isolated mode. +- `py:python_executable/0`, `py:kill/1`, `py_nif:os_kill/2`. +- `py_isolated` is a `gen_statem` (states `idle`, `{busy, Id}`, `looping`, + `stopping_loop`, `{restarting, Reason}`): `sys:get_state/1` and + `sys:trace/2` work on isolated contexts, requests arriving during a + restart are served by the new child, and `py_context:kill/1` returns once + the new child is up. +- Timeouts on an isolated context cancel their own request only (queued + requests are dropped, the executing one is interrupted); the kill backstop + is bound to that request, so a busy shared context is never killed because + another caller gave up. Soak-tested: callback storms, interrupt/kill + storms, loop churn, 60 s mixed workload with resource counters checked. +- Guide: `docs/isolated.md`, with what each of the three modes guarantees. + ## 4.1.0 (2026-08-15) ### Added diff --git a/README.md b/README.md index d98c347..66b2ae8 100644 --- a/README.md +++ b/README.md @@ -603,6 +603,7 @@ When creating Python contexts, you can choose the execution mode: |------|----------------|-------------| | `worker` | Any | Dedicated pthread per context, main interpreter namespace (default) | | `owngil` | 3.14+ | Dedicated pthread + subinterpreter with its own GIL, true parallelism | +| `isolated` | Any | CPython in a child OS process: killable, rlimit-bounded, crash-contained | ```erlang %% Default: worker mode (recommended) @@ -612,8 +613,18 @@ When creating Python contexts, you can choose the execution mode: %% OWN_GIL mode for true parallelism (Python 3.14+ required) %% Each context runs in its own pthread with independent GIL {ok, Ctx} = py_context:new(#{mode => owngil}). + +%% Isolated mode: a child process per context. A stuck call is killed, a +%% segfault only takes the child down, rlimits bound memory and CPU. +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000, + rlimits => #{as => 512 * 1024 * 1024}}). ``` +**Isolated mode** is the only mode with a hard bound: `py_context:interrupt/1` +stops a blocking C call, and `SIGKILL` is the backstop. It costs a process per +context (about 16 MB and 40 ms to start) and roughly twice the call latency. +See [Isolated Contexts](docs/isolated.md). + **Worker mode is recommended** because it works with any Python version and automatically benefits from free-threaded Python (3.13t+) when available. Each context owns a dedicated pthread, providing stable thread affinity for libraries with thread-local state (numpy, torch, tensorflow). **Why OWN_GIL requires Python 3.14+**: Some C extensions (e.g., `_decimal`, `numpy`) have global state bugs in sub-interpreters on Python 3.12/3.13. These are fixed in Python 3.14. @@ -629,6 +640,7 @@ py:execution_mode(). %% => worker | owngil |------|----------------|-------------| | `worker` (default) | Any | One pthread per context; true parallelism on free-threaded 3.13t+ | | `owngil` | 3.14+ | Per-interpreter GIL, true parallelism across contexts | +| `isolated` | Any | One OS process per context, parallel and failure-isolated | ## Error Handling @@ -651,6 +663,7 @@ py:execution_mode(). %% => worker | owngil - [Logging and Tracing](docs/logging.md) - [Asyncio Event Loop](docs/asyncio.md) - Erlang-native asyncio with TCP/UDP support - [Worker Loops](docs/workers.md) - Long-lived loops in owngil contexts, serving on sockets Erlang owns +- [Isolated Contexts](docs/isolated.md) - Python in a child process: kill, rlimits, crash containment - [Reactor](docs/reactor.md) - FD-based protocol handling - [Security](docs/security.md) - Sandbox and blocked operations - [Changelog](https://github.com/benoitc/erlang-python/releases) diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 527b738..cb8f459 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -36,6 +36,14 @@ * - py_callback.c: Callback system and asyncio support */ +/* pthread_timedjoin_np (used to bound the owngil worker join on Linux) + * is declared by only under _GNU_SOURCE. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include #include "py_nif.h" #include "py_util.h" #include "py_event_loop.h" @@ -8079,6 +8087,26 @@ static void unload(ErlNifEnv *env, void *priv_data) { /* Other cleanup handled by finalize */ } +/** + * @brief Send a signal to an OS process (kill(2)). + * + * Used by isolated contexts to SIGKILL their child. The caller holds the + * child's port open until exit_status arrives, so the pid cannot have been + * recycled. + */ +static ERL_NIF_TERM nif_os_kill(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + (void)argc; + int pid, sig; + if (!enif_get_int(env, argv[0], &pid) || !enif_get_int(env, argv[1], &sig) || pid <= 0) { + return enif_make_badarg(env); + } + if (kill((pid_t)pid, sig) == 0) { + return ATOM_OK; + } + return enif_make_tuple2(env, ATOM_ERROR, + enif_make_atom(env, errno == ESRCH ? "esrch" : errno == EPERM ? "eperm" : "einval")); +} + static ErlNifFunc nif_funcs[] = { /* Initialization */ {"init", 0, nif_py_init, 0}, @@ -8220,6 +8248,7 @@ static ErlNifFunc nif_funcs[] = { {"create_test_pipe", 0, nif_create_test_pipe, 0}, {"close_test_fd", 1, nif_close_test_fd, 0}, {"dup_fd", 1, nif_dup_fd, 0}, + {"os_kill", 2, nif_os_kill, 0}, {"write_test_fd", 2, nif_write_test_fd, 0}, {"read_test_fd", 2, nif_read_test_fd, 0}, /* TCP test helpers */ diff --git a/docs/interrupts.md b/docs/interrupts.md index 84d4465..20461e5 100644 --- a/docs/interrupts.md +++ b/docs/interrupts.md @@ -65,12 +65,29 @@ context to deal with that: ok = py_context:destroy(Ctx). ``` +## Interrupting a blocking C call: isolated mode + +The limits below apply to the embedded modes. An `isolated` context runs +Python in a child process, where an interrupt is a signal that lands inside +`time.sleep`, a socket read or any other blocking call, and `SIGKILL` is the +backstop if the signal is ignored: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000}), +{error, timeout} = py_context:eval(Ctx, <<"__import__('time').sleep(60)">>, #{}, 200), +%% Usable at once, no 60 s wait +{ok, 4} = py_context:eval(Ctx, <<"2+2">>, #{}, 5000). +``` + +`py_context:kill/1` kills at once. See [Isolated Contexts](isolated.md). + ## Limits - CPython delivers an async exception at the next bytecode boundary. Code blocked inside a C call (`time.sleep`, a numpy kernel, a socket read) is not interrupted until that call returns. The call still times out on the - Erlang side; the context becomes usable once the C call finishes. + Erlang side; the context becomes usable once the C call finishes. Only + `isolated` mode interrupts such a call. - An interrupt targets the context, not an individual request. Interrupting a context that just finished one call and started another stops the new one. - `py:call/3,4` and `py:eval/1,2` use `infinity` by default. Pass an explicit diff --git a/docs/isolated.md b/docs/isolated.md new file mode 100644 index 0000000..b737bf2 --- /dev/null +++ b/docs/isolated.md @@ -0,0 +1,267 @@ +# Isolated Contexts + +This guide covers `isolated` mode, where a context's CPython interpreter runs +in a child OS process instead of inside the BEAM. You need it when Python +code must not be able to take the node down or run forever: user-supplied +scripts, C extensions you do not control, work that needs a hard time or +memory bound. The public API is the one you already use with `worker` and +`owngil`; you switch by configuration. + +## What the three modes guarantee + +| | `worker` | `owngil` | `isolated` | +|---|---|---|---| +| Where Python runs | BEAM process, one pthread per context | BEAM process, one pthread and one interpreter per context | Child OS process per context | +| Interrupt a Python loop | yes (`KeyboardInterrupt` at the next bytecode) | yes | yes | +| Interrupt a blocking C call (`time.sleep`, socket read, numpy kernel) | no, only when the call returns | no | yes (signal), then `SIGKILL` | +| Hard bound on a call | no: a stuck call keeps its thread until it returns | no | yes: `kill_after` then `SIGKILL` | +| Memory cap | no | obmalloc accounting, C extensions not counted | `RLIMIT_AS` (Linux, FreeBSD), RSS watchdog (macOS), cgroups v2 (Linux) | +| CPU bound | no | no | `RLIMIT_CPU` | +| Segfault in a C extension | kills the node | kills the node | kills the child, caller gets `{error, {child_exited, {signal, 11}}}` | +| Python state after a crash | n/a | n/a | lost; the child restarts and the context stays usable | +| Call latency (`eval("1+1")`, p50, same machine) | 16 us | ~20 us | 25 us | +| Memory per context | shared interpreter | one interpreter | one process, ~16 MB RSS bare | +| Startup | microseconds | milliseconds | ~40 ms | +| Zero-copy `py_buffer`, channels, `erlang.schedule`, object refs | yes | yes | no (see Limits) | + +## Start a context + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated}), +{ok, 4} = py_context:eval(Ctx, <<"2+2">>), +{ok, 4.0} = py_context:call(Ctx, math, sqrt, [16]), +ok = py_context:exec(Ctx, <<"x = 41">>), +{ok, 42} = py_context:eval(Ctx, <<"x + 1">>), +ok = py_context:stop(Ctx). +``` + +A pool works the same way, and `py:call/4` routes to it: + +```erlang +{ok, _} = py_context_router:start_pool(sandbox, 4, isolated), +{ok, 4.0} = py:call(sandbox, math, sqrt, [16]). +``` + +Options of `py_context:new/1` specific to this mode: + +| Option | Default | Meaning | +|---|---|---| +| `python` | interpreter matching the embedded runtime (`py:python_executable/0`), or `isolated_python` app env | Executable to run | +| `rlimits` | `#{}` | `#{as => Bytes, cpu => Seconds, nofile => N}`, applied with `setrlimit` before any user code | +| `cgroup` | none | Path of a cgroup v2 directory the child joins (limits written by you: `memory.max`, `cpu.max`, `pids.max`) | +| `env` | `#{}` | Extra environment variables for the child | +| `paths` | `[]` | Extra `sys.path` entries (registered `py_import` paths and imports are applied too) | +| `preload` | none | Code run once in the child before anything else | +| `kill_after` | `1000` | Milliseconds between a soft interrupt and `SIGKILL` | +| `restart` | `true` | Start a fresh child when the current one dies | +| `max_restarts`, `restart_period` | `5`, `10000` | Restart budget; past it the context process exits with the child's reason | +| `start_timeout` | `10000` | Milliseconds allowed for the child to connect | + +## Cancel work that ignores the embedded modes + +`py_context:interrupt/1` sends the child a signal that raises +`KeyboardInterrupt` in the running request, inside a blocking C call too. If +the request has not returned after `kill_after`, the child is killed: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 500}), +Self = self(), +spawn(fun() -> Self ! {done, py_context:eval(Ctx, <<"__import__('time').sleep(60)">>)} end), +timer:sleep(100), +ok = py_context:interrupt(Ctx), +receive {done, {error, interrupted}} -> ok end. +``` + +A timeout targets only the request that timed out: if the child is +executing it, the child is interrupted (and killed after `kill_after` if the +interrupt is not honoured); if it is still queued behind other callers' +requests, it is dropped from the queue and nobody else is interrupted. This +differs from the embedded modes, where an interrupt can only hit whatever +runs. `py_context:interrupt/1` remains context-wide: it interrupts the +request executing now. `py_context:kill/1` skips the soft step: + +```erlang +ok = py_context:kill(Ctx), +%% A fresh child is already serving; the Python state is gone +{ok, 4} = py_context:eval(Ctx, <<"2+2">>). +``` + +In-flight calls return `{error, interrupted}` (soft) or `{error, killed}` +(hard). + +## Bound memory and CPU + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, + rlimits => #{as => 512 * 1024 * 1024, + cpu => 30, + nofile => 256}}). +``` + +Past `as`, allocations fail with `MemoryError` in the child (or the child +dies if it cannot cope); past `cpu`, the child dies with `SIGXCPU` and the +caller gets `{error, {child_exited, {signal, 24}}}`. rlimits are POSIX and +are the portable bound: Linux and FreeBSD enforce all three in the kernel. +macOS ignores `RLIMIT_AS`, so there the child enforces `as` itself: a +watchdog thread polls its resident set every 50 ms and exits when it passes +the limit, and the caller gets `{error, {child_exited, {memory_limit, Bytes}}}`. +`cpu` and `nofile` are kernel-enforced on all three. A free-threaded +CPython build reserves a large virtual range at startup, so its `as` limit +must be well above what a regular build needs (several GB). + +With cgroups v2 (Linux only), create the group and write the limits, then +hand the directory to the context. On any other platform the option is +refused before a child is spawned, with `{error, {cgroup_unsupported, Os}}`: + +```sh +mkdir /sys/fs/cgroup/py_sandbox +echo 268435456 > /sys/fs/cgroup/py_sandbox/memory.max +echo "50000 100000" > /sys/fs/cgroup/py_sandbox/cpu.max +echo 64 > /sys/fs/cgroup/py_sandbox/pids.max +``` + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, cgroup => "/sys/fs/cgroup/py_sandbox"}). +``` + +The child joins the group before running any user code; if it cannot, the +start fails with `{error, {startup_error, [{cgroup, Reason}]}}`. + +## Survive a crash + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated}), +{error, {child_exited, {signal, 11}}} = + py_context:eval(Ctx, <<"__import__('ctypes').memset(0, 0, 1)">>), +{ok, 4} = py_context:eval(Ctx, <<"2+2">>). +``` + +The node, every other context and the context process itself are unaffected. +With `restart => false` the context process exits with +`{child_exited, Reason}` instead, so a supervisor of yours decides. + +## Call Erlang from the child + +`erlang.call`, `erlang.send`, `erlang.whereis`, `erlang.Pid`, `erlang.Atom` +work as in the embedded modes, from any Python thread, and callbacks nest: an +Erlang function called from Python may call back into the same context, from +the process running the callback. The context serves one caller at a time, +in order; a nested call from the callback process goes through at once, +while a call from any other process waits for the current request to finish +(so a callback that hands the nested call to another process and waits for +it would wait until its own timeout). + +```erlang +py:register_function(double, fun([X]) -> X * 2 end), +{ok, 84} = py_context:eval(Ctx, <<"__import__('erlang').call('double', 42)">>). +``` + +```python +import erlang + +def notify(pid): + erlang.send(pid, ('progress', 50)) # raises erlang.ProcessError if pid is dead +``` + +Round trips cross a Unix socket as external term format; the type mapping is +the one in [Type Conversion](type-conversion.md). One difference: integers +beyond 64 bits arrive intact (the NIF converter has no bignum path). + +## asyncio and worker loops + +The child runs a plain `asyncio` loop. A call that returns a coroutine is +awaited and its value returned: + +```python +async def fetch(n): + await asyncio.sleep(0.1) + return n * 2 +``` + +```erlang +{ok, 4} = py_context:call(Ctx, myapp, fetch, [2]). +``` + +The worker loop API of [Worker Loops](workers.md) works unchanged +(`start_loop`, `submit`, `submit_await`, `stop_loop`), and a wedged loop is +killed by `stop_loop/2` after its grace period. To serve on a socket Erlang +owns, hand the fd over with `py_context:pass_fd/2`; it crosses the control +socket with `SCM_RIGHTS`: + +```erlang +{ok, LSock} = gen_tcp:listen(8080, [binary, {active, false}]), +{ok, Fd} = inet:getfd(LSock), +[begin + {ok, Ctx} = py_context:new(#{mode => isolated}), + ok = py_context:start_loop(Ctx), + {ok, ChildFd} = py_context:pass_fd(Ctx, Fd), + {ok, _} = py_context:submit_await(Ctx, myapp, serve, [ChildFd]) + end || _ <- lists:seq(1, 4)]. +``` + +Inside a coroutine, `await erlang.async_call(name, *args)` keeps the loop +running while Erlang answers. + +## Process model + +- One child per context, started with `open_port` so the VM reaps it and + reports its exit status: no zombies, and a child that dies before + connecting is reported with its output (`{error, {child_exited_at_start, Reason, Output}}`). +- The context process is a `gen_statem` (`py_isolated`) with states + `idle`, `{busy, RequestId}`, `looping`, `stopping_loop` and + `{restarting, Reason}`; `sys:get_state(Ctx)` shows what it is doing and + `sys:trace(Ctx, true)` prints its events. It serves one caller at a time, + in order; a request that arrives while the child restarts waits for the + new child instead of failing. It outlives the process that created it, + and stops if that process crashes. +- The child and the context process talk over a Unix socket in a private + directory, framed exactly like the embedded callback pipe + (`<>`, body `<>`). +- A reader thread in the child owns the socket. It delivers requests to the + main thread, routes replies to whichever thread is waiting, handles + `interrupt` by signalling the main thread, and exits the process on EOF. + So when the BEAM dies, every child exits, even one stuck in a C call; on + Linux (`prctl(PR_SET_PDEATHSIG)`) and FreeBSD (`procctl(PROC_PDEATHSIG_CTL)`) + the kernel delivers `SIGKILL` for the same case. +- When the socket breaks or the child exits, pending calls fail with + `{error, {child_exited, Reason}}`, new calls fail the same way until the + restart has happened (which takes about 100 ms), and nothing hangs. +- Child stdout and stderr are forwarded to the Erlang logger, one line per + message, tagged with the context id and OS pid. `py_context:child_info/1` + returns `os_pid`, `python_version`, `executable` and `platform`. + +## Limits + +- Python object references cannot cross a process boundary: + `py_context:call_method/4` returns `{error, not_supported_in_isolated}`, + results are always converted to terms, and process-local environments + (`py:call(Ctx, ...)` per Erlang process) map to the child's single + namespace. +- `erlang.schedule*`, channels (`py_channel`, `py_byte_channel`), + `py_buffer`, shared dicts and the reactor need the embedded interpreter and + raise `RuntimeError("... not available in isolated mode")`. +- `py_context:loop_ref/1` returns `{error, not_supported_in_isolated}`; + `submit/4` without a running loop returns `{error, no_loop}` (there is no + event worker to step an idle loop). +- `erlang.call` from inside a coroutine blocks the loop, as in the embedded + modes; use `erlang.async_call`. +- The child decodes terms with the same rules as the NIF, so atoms sent from + Python are created in the VM's atom table. Do not let untrusted code mint + unbounded distinct atoms. +- No syscall filtering: process isolation plus rlimits is the boundary. A + seccomp (Linux) or Capsicum (FreeBSD) sandbox is a separate hardening step. +- Each call copies its arguments and result through the socket: a 1 MB + binary round-trips in about 1.3 ms, 16 MB in about 27 ms (worker mode: + 0.2 ms and 3 ms). For bulk data prefer a file, a socket the child reads + itself, or a shared mapping: `erlang-iommap` opens a file `MAP_SHARED` and + `region_binary/3` gives a binary over it, while the child maps the same + file with `mmap`; a 64 MB region costs 3 us on the Erlang side and 5 ms to + map in Python, against 12 ms to copy it through ETF. `py_buffer` is not + ported to that yet. + +## See also + +- [Interrupts](interrupts.md) for the embedded-mode interrupt semantics +- [Worker Loops](workers.md) for the loop API and serving on Erlang sockets +- [Memory](memory.md) for the owngil memory caps +- [Security](security.md) for the audit-hook sandbox of the embedded modes diff --git a/docs/security.md b/docs/security.md index 5ba726a..db402eb 100644 --- a/docs/security.md +++ b/docs/security.md @@ -144,6 +144,23 @@ if is_sandboxed(): print("Running inside Erlang VM - subprocess operations blocked") ``` +## Process Isolation + +The audit hook keeps Python from forking the VM; it does not protect the VM +from Python. A C extension can still segfault the node, and nothing can cap +the memory or CPU of embedded code. For that boundary run the context in a +child process: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, + rlimits => #{as => 256 * 1024 * 1024, cpu => 10}, + kill_after => 1000}). +``` + +A crash kills only the child, `py_context:kill/1` is total, and rlimits or +cgroups bound resources. See [Isolated Contexts](isolated.md). The child is +not sandboxed at the syscall level; that is a separate hardening step. + ## Signal Handling Note Signal handling is also not supported in the Erlang event loop. The `ErlangEventLoop` raises `NotImplementedError` for `add_signal_handler()` and `remove_signal_handler()`. Signal handling should be done at the Erlang VM level using Erlang's signal handling facilities. diff --git a/docs/workers.md b/docs/workers.md index 85fea63..74b83bf 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -28,6 +28,11 @@ Worker contexts get the same API on the shared main interpreter loop, which allows one running `ErlangEventLoop` per interpreter: use owngil (Python 3.14+) for several workers. +Isolated contexts (`mode => isolated`) get the same API on a plain asyncio +loop in their child process; hand sockets over with `py_context:pass_fd/2` +instead of `py:dup_fd/1`, and a wedged loop is killed by `stop_loop/2`. See +[Isolated Contexts](isolated.md). + ## Serve TCP on a socket Erlang owns Bind once in Erlang, duplicate the listen fd for each worker with diff --git a/priv/_erlang_impl/_etf.py b/priv/_erlang_impl/_etf.py new file mode 100644 index 0000000..47fceec --- /dev/null +++ b/priv/_erlang_impl/_etf.py @@ -0,0 +1,477 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Erlang external term format (ETF) codec, pure Python. + +Used by the isolated child process, where no NIF is available. The type +mapping mirrors c_src/py_convert.c so code behaves the same in `worker`, +`owngil` and `isolated` mode: + + Erlang -> Python Python -> Erlang + true / false -> True / False None -> none + none/nil/undefined-> None bool -> true / false + other atom -> str int -> integer (any size) + integer -> int float -> float (nan/inf -> atoms) + float -> float str, bytes -> binary + binary (utf-8) -> str list -> list + binary (other) -> bytes tuple -> tuple + {bytes, Bin} -> bytes dict -> map + list / string -> list Atom -> atom + tuple -> tuple Pid / Ref / Port-> pid / ref / port + map -> dict numpy ndarray -> list (tolist) + pid / ref / port -> Pid / Ref / Port other object -> binary(str(obj)) + +Pids, refs and ports are opaque: the decoder keeps their raw ETF bytes and +the encoder emits them unchanged, so they round-trip exactly. +""" + +import math +import struct + +__all__ = [ + 'Atom', 'Pid', 'Ref', 'Port', + 'encode', 'decode', 'DecodeError', +] + +VERSION = 131 + +# Tags +NEW_FLOAT_EXT = 70 +BIT_BINARY_EXT = 77 +NEW_PID_EXT = 88 +NEW_PORT_EXT = 89 +NEWER_REFERENCE_EXT = 90 +SMALL_INTEGER_EXT = 97 +INTEGER_EXT = 98 +FLOAT_EXT = 99 +ATOM_EXT = 100 +REFERENCE_EXT = 101 +PORT_EXT = 102 +PID_EXT = 103 +SMALL_TUPLE_EXT = 104 +LARGE_TUPLE_EXT = 105 +NIL_EXT = 106 +STRING_EXT = 107 +LIST_EXT = 108 +BINARY_EXT = 109 +SMALL_BIG_EXT = 110 +LARGE_BIG_EXT = 111 +NEW_REFERENCE_EXT = 114 +SMALL_ATOM_EXT = 115 +MAP_EXT = 116 +ATOM_UTF8_EXT = 118 +SMALL_ATOM_UTF8_EXT = 119 +V4_PORT_EXT = 120 + + +class DecodeError(ValueError): + """Raised on malformed external term format data.""" + + +class Atom(str): + """An Erlang atom. A str subclass so `Atom('ok') == 'ok'`, and so code + written for the embedded erlang.Atom keeps working.""" + + __slots__ = () + + def __new__(cls, name): + if isinstance(name, bytes): + name = name.decode('utf-8') + if not isinstance(name, str): + raise TypeError('atom name must be str') + return str.__new__(cls, name) + + def __repr__(self): + return 'erlang.Atom(%r)' % str.__str__(self) + + +class _Opaque: + """Base for pid/ref/port: value semantics over the raw ETF bytes.""" + + __slots__ = ('_raw',) + + def __init__(self, raw): + if not isinstance(raw, (bytes, bytearray)): + raise TypeError('%s wants raw ETF bytes' % type(self).__name__) + self._raw = bytes(raw) + + def __eq__(self, other): + return type(other) is type(self) and other._raw == self._raw + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((type(self).__name__, self._raw)) + + def __repr__(self): + return '' % (type(self).__name__, self._raw.hex()) + + @property + def raw(self): + return self._raw + + +class Pid(_Opaque): + __slots__ = () + + def __repr__(self): + try: + node, ident, serial, _ = _decode_pid_fields(self._raw) + return '@%s>' % (ident, serial, node) + except Exception: + return _Opaque.__repr__(self) + + +class Ref(_Opaque): + __slots__ = () + + +class Port(_Opaque): + __slots__ = () + + +# --------------------------------------------------------------------------- +# Encoding +# --------------------------------------------------------------------------- + +_pack_u8 = struct.Struct('>B').pack +_pack_u16 = struct.Struct('>H').pack +_pack_u32 = struct.Struct('>I').pack +_pack_i32 = struct.Struct('>i').pack +_pack_f64 = struct.Struct('>d').pack + + +def encode(obj): + """Encode a Python object as a complete ETF binary (with version byte).""" + out = bytearray([VERSION]) + _encode(obj, out) + return bytes(out) + + +def _encode_atom(name, out): + data = name.encode('utf-8') + n = len(data) + if n > 255: + raise ValueError('atom too long: %d bytes' % n) + if n < 256: + out += _pack_u8(SMALL_ATOM_UTF8_EXT) + out += _pack_u8(n) + out += data + + +def _encode(obj, out): + if obj is None: + _encode_atom('none', out) + elif obj is True: + _encode_atom('true', out) + elif obj is False: + _encode_atom('false', out) + elif isinstance(obj, Atom): + _encode_atom(str.__str__(obj), out) + elif isinstance(obj, int): + _encode_int(obj, out) + elif isinstance(obj, float): + if math.isnan(obj): + _encode_atom('nan', out) + elif math.isinf(obj): + _encode_atom('infinity' if obj > 0 else 'neg_infinity', out) + else: + out += _pack_u8(NEW_FLOAT_EXT) + out += _pack_f64(obj) + elif isinstance(obj, str): + data = obj.encode('utf-8', 'surrogatepass') + out += _pack_u8(BINARY_EXT) + out += _pack_u32(len(data)) + out += data + elif isinstance(obj, (bytes, bytearray, memoryview)): + data = bytes(obj) + out += _pack_u8(BINARY_EXT) + out += _pack_u32(len(data)) + out += data + elif isinstance(obj, _Opaque): + out += obj._raw + elif isinstance(obj, tuple): + n = len(obj) + if n < 256: + out += _pack_u8(SMALL_TUPLE_EXT) + out += _pack_u8(n) + else: + out += _pack_u8(LARGE_TUPLE_EXT) + out += _pack_u32(n) + for item in obj: + _encode(item, out) + elif isinstance(obj, list): + n = len(obj) + if n == 0: + out += _pack_u8(NIL_EXT) + else: + out += _pack_u8(LIST_EXT) + out += _pack_u32(n) + for item in obj: + _encode(item, out) + out += _pack_u8(NIL_EXT) + elif isinstance(obj, dict): + out += _pack_u8(MAP_EXT) + out += _pack_u32(len(obj)) + for k, v in obj.items(): + _encode(k, out) + _encode(v, out) + elif _is_ndarray(obj): + _encode(obj.tolist(), out) + elif isinstance(obj, (set, frozenset)): + _encode(list(obj), out) + else: + # Same fallback as py_to_term: the string representation as a binary + _encode(str(obj), out) + + +def _encode_int(value, out): + if 0 <= value <= 255: + out += _pack_u8(SMALL_INTEGER_EXT) + out += _pack_u8(value) + elif -2147483648 <= value <= 2147483647: + out += _pack_u8(INTEGER_EXT) + out += _pack_i32(value) + else: + sign = 1 if value < 0 else 0 + mag = -value if sign else value + n = (mag.bit_length() + 7) // 8 + digits = mag.to_bytes(n, 'little') + if n < 256: + out += _pack_u8(SMALL_BIG_EXT) + out += _pack_u8(n) + else: + out += _pack_u8(LARGE_BIG_EXT) + out += _pack_u32(n) + out += _pack_u8(sign) + out += digits + + +def _is_ndarray(obj): + t = type(obj) + if t.__module__ == 'numpy' and t.__name__ == 'ndarray': + return True + return hasattr(obj, 'tolist') and hasattr(obj, 'ndim') + + +# --------------------------------------------------------------------------- +# Decoding +# --------------------------------------------------------------------------- + +_unpack_u16 = struct.Struct('>H').unpack_from +_unpack_u32 = struct.Struct('>I').unpack_from +_unpack_i32 = struct.Struct('>i').unpack_from +_unpack_f64 = struct.Struct('>d').unpack_from + +_ATOM_TRUE = 'true' +_ATOM_FALSE = 'false' +_NONE_ATOMS = frozenset(('none', 'nil', 'undefined')) + + +def decode(data): + """Decode a complete ETF binary (with version byte) to a Python object.""" + if not data or data[0] != VERSION: + raise DecodeError('bad ETF version byte') + value, pos = _decode(data, 1) + if pos != len(data): + raise DecodeError('trailing bytes after term') + return value + + +def _atom_value(name): + """Map an atom to its Python value the way term_to_py does.""" + if name == _ATOM_TRUE: + return True + if name == _ATOM_FALSE: + return False + if name in _NONE_ATOMS: + return None + return name + + +def _decode(data, pos): + try: + tag = data[pos] + except IndexError: + raise DecodeError('truncated term') from None + pos += 1 + + if tag == SMALL_INTEGER_EXT: + return data[pos], pos + 1 + if tag == INTEGER_EXT: + return _unpack_i32(data, pos)[0], pos + 4 + if tag == BINARY_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + raw = bytes(data[pos:pos + n]) + if len(raw) != n: + raise DecodeError('truncated binary') + try: + return raw.decode('utf-8'), pos + n + except UnicodeDecodeError: + return raw, pos + n + if tag == SMALL_ATOM_UTF8_EXT: + n = data[pos] + pos += 1 + return _atom_value(bytes(data[pos:pos + n]).decode('utf-8')), pos + n + if tag == ATOM_UTF8_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return _atom_value(bytes(data[pos:pos + n]).decode('utf-8')), pos + n + if tag == ATOM_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return _atom_value(bytes(data[pos:pos + n]).decode('latin-1')), pos + n + if tag == SMALL_ATOM_EXT: + n = data[pos] + pos += 1 + return _atom_value(bytes(data[pos:pos + n]).decode('latin-1')), pos + n + if tag == SMALL_TUPLE_EXT or tag == LARGE_TUPLE_EXT: + if tag == SMALL_TUPLE_EXT: + n = data[pos] + pos += 1 + else: + (n,) = _unpack_u32(data, pos) + pos += 4 + items = [] + for _ in range(n): + item, pos = _decode(data, pos) + items.append(item) + # {bytes, Bin}: explicit bytes, as in term_to_py + if n == 2 and items[0] == 'bytes' and isinstance(items[1], (str, bytes)): + b = items[1] + if isinstance(b, str): + b = b.encode('utf-8', 'surrogatepass') + return b, pos + return tuple(items), pos + if tag == NIL_EXT: + return [], pos + if tag == STRING_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return list(data[pos:pos + n]), pos + n + if tag == LIST_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + items = [] + for _ in range(n): + item, pos = _decode(data, pos) + items.append(item) + # Tail: NIL for a proper list. An improper tail is kept as a final + # element, the same as enif_get_list_length failing is not an option + # here; this never happens for term_to_binary of proper lists. + if data[pos] == NIL_EXT: + pos += 1 + else: + tail, pos = _decode(data, pos) + items.append(tail) + return items, pos + if tag == MAP_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + result = {} + for _ in range(n): + k, pos = _decode(data, pos) + v, pos = _decode(data, pos) + result[_hashable(k)] = v + return result, pos + if tag == NEW_FLOAT_EXT: + return _unpack_f64(data, pos)[0], pos + 8 + if tag == FLOAT_EXT: + text = bytes(data[pos:pos + 31]).split(b'\x00', 1)[0] + return float(text), pos + 31 + if tag == SMALL_BIG_EXT or tag == LARGE_BIG_EXT: + if tag == SMALL_BIG_EXT: + n = data[pos] + pos += 1 + else: + (n,) = _unpack_u32(data, pos) + pos += 4 + sign = data[pos] + pos += 1 + value = int.from_bytes(bytes(data[pos:pos + n]), 'little') + return (-value if sign else value), pos + n + if tag in (NEW_PID_EXT, PID_EXT): + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) # node + pos += 8 # id, serial + pos += 4 if tag == NEW_PID_EXT else 1 # creation + return Pid(data[start:pos]), pos + if tag in (NEWER_REFERENCE_EXT, NEW_REFERENCE_EXT): + start = pos - 1 + (n,) = _unpack_u16(data, pos) + pos += 2 + _, pos = _decode_atom_raw(data, pos) + pos += 4 if tag == NEWER_REFERENCE_EXT else 1 + pos += 4 * n + return Ref(data[start:pos]), pos + if tag == REFERENCE_EXT: + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) + pos += 5 + return Ref(data[start:pos]), pos + if tag in (NEW_PORT_EXT, PORT_EXT, V4_PORT_EXT): + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) + pos += 8 if tag == V4_PORT_EXT else 4 + pos += 1 if tag == PORT_EXT else 4 + return Port(data[start:pos]), pos + if tag == BIT_BINARY_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + bits = data[pos] + pos += 1 + raw = bytes(data[pos:pos + n]) + return (raw, bits), pos + n + raise DecodeError('unsupported ETF tag %d' % tag) + + +def _decode_atom_raw(data, pos): + """Decode an atom (any encoding) returning its name; used inside + pid/ref/port where the value is not mapped.""" + tag = data[pos] + pos += 1 + if tag == SMALL_ATOM_UTF8_EXT or tag == SMALL_ATOM_EXT: + n = data[pos] + pos += 1 + elif tag == ATOM_UTF8_EXT or tag == ATOM_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + else: + raise DecodeError('expected atom, got tag %d' % tag) + return bytes(data[pos:pos + n]).decode('utf-8', 'replace'), pos + n + + +def _decode_pid_fields(raw): + tag = raw[0] + node, pos = _decode_atom_raw(raw, 1) + (ident,) = _unpack_u32(raw, pos) + (serial,) = _unpack_u32(raw, pos + 4) + pos += 8 + if tag == NEW_PID_EXT: + (creation,) = _unpack_u32(raw, pos) + else: + creation = raw[pos] + return node, ident, serial, creation + + +def _hashable(key): + """Map keys must be hashable; lists (Erlang lists/strings) become tuples.""" + if isinstance(key, list): + return tuple(_hashable(k) for k in key) + if isinstance(key, dict): + return tuple(sorted((_hashable(k), _hashable(v)) for k, v in key.items())) + return key diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py new file mode 100644 index 0000000..a276c36 --- /dev/null +++ b/priv/_erlang_impl/_isolated.py @@ -0,0 +1,824 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Runtime of an `isolated` context: the far end of the socket that +src/py_isolated.erl talks to. + +Wire format, shared with the blocking callback pipe of the embedded modes: + + <> + Body = <> Payload is ETF unless noted + + Status 0 request, Erlang -> child {call,M,F,A,K} | {eval,Code,Locals} + | {exec,Code} | ping | shutdown + | start_loop | stop_loop + | {submit,Ref,M,F,A,K} | pass_fd + Status 1 error reply (either way) reason term + Status 2 ok reply (either way) value term + Status 3 request, child -> Erlang {call,Name,Args} | {send,Pid,Msg} + | {whereis,Name} + Status 4 event, child -> Erlang {ready,Info} | {startup_error,R} + | {async_result,Ref,R} + | {loop_exit,R} | {log,Level,Msg} + Status 5 control, Erlang -> child interrupt (handled on the reader + thread, never queued) + +Threads: + * the main thread executes requests, one at a time, and owns the asyncio + loop; while it waits for the reply to an erlang.call it keeps serving + requests, so a callback may call back into this context (nesting); + * the reader thread owns socket reads. It routes replies to whoever is + waiting (any Python thread may use erlang.call/send/whereis), queues + requests for the main thread, handles `interrupt` by signalling the main + thread, and exits the process on EOF so a BEAM death never leaves an + orphan, even if the main thread is stuck in a C call. +""" + +import asyncio +import inspect +import os +import queue +import signal +import socket +import struct +import sys +import threading +import traceback + +from . import _etf +from ._etf import Atom, Pid, Ref, Port, DecodeError + +__all__ = ['Runtime', 'install_erlang_module'] + +STATUS_REQUEST = 0 +STATUS_ERROR = 1 +STATUS_OK = 2 +STATUS_CALLBACK = 3 +STATUS_EVENT = 4 +STATUS_CONTROL = 5 + +_HEADER = struct.Struct('=QI') # native byte order, no padding +_HEADER_LEN = _HEADER.size + +_INTERRUPT_SIGNAL = signal.SIGUSR1 + + +class ProcessError(Exception): + """Raised by erlang.send when the target process does not exist.""" + + +class SuspensionRequired(BaseException): + """Kept for source compatibility with the embedded erlang module. It is + never raised in isolated mode: callbacks are real socket round-trips.""" + + +class PipeBroken(RuntimeError): + """The control socket to Erlang is gone.""" + + +class _Interrupted(KeyboardInterrupt): + """KeyboardInterrupt raised by the interrupt signal handler. A subclass so + user code catching KeyboardInterrupt keeps working, while the dispatcher + can tell an Erlang interrupt from a stray Ctrl-C.""" + + +class _Waiter: + """Reply slot for a request this process sent to Erlang.""" + + __slots__ = ('event', 'result', 'inbox', 'future', 'loop') + + def __init__(self, inbox=None, future=None, loop=None): + self.event = None if (inbox is not None or future is not None) else threading.Event() + self.result = None + self.inbox = inbox + self.future = future + self.loop = loop + + def deliver(self, result): + self.result = result + if self.inbox is not None: + self.inbox.put(('reply', self)) + elif self.future is not None: + try: + self.loop.call_soon_threadsafe(_resolve_future, self.future, result) + except RuntimeError: + pass # the user's loop is closed: nobody is waiting + else: + self.event.set() + + +def _resolve_future(future, result): + if future.cancelled(): + return + status, value = result + if status == STATUS_OK: + future.set_result(value) + else: + future.set_exception(_callback_error(value)) + + +def _callback_error(reason): + if isinstance(reason, str): + return RuntimeError(reason) + if isinstance(reason, tuple) and len(reason) == 2 and reason[0] == 'noproc': + return ProcessError('process %r does not exist' % (reason[1],)) + return RuntimeError('erlang call failed: %r' % (reason,)) + + +class Runtime: + """One per child process.""" + + def __init__(self, sock, context_pid=None): + self.sock = sock + self.context_pid = context_pid + self._wlock = threading.Lock() + self._idlock = threading.Lock() + self._next_id = 1 + self._pending = {} + self._plock = threading.Lock() + self.inbox = queue.SimpleQueue() + self.broken = False + self.broken_reason = None + self.globals = {'__name__': '__main__', '__builtins__': __builtins__} + self.running = False # a request is executing on main thread + self.loop = None + self.loop_running = False + self._received_fds = [] + self._fds_lock = threading.Lock() + self._cancelled = set() # request ids Erlang gave up on + self._cancel_lock = threading.Lock() + self._exec_stack = [] # main thread: ids being executed (nesting) + self.main_thread = threading.main_thread() + self._reader = None + + # -- lifecycle --------------------------------------------------------- + + def start(self): + signal.signal(_INTERRUPT_SIGNAL, self._on_interrupt_signal) + self._reader = threading.Thread(target=self._reader_main, + name='erlang-reader', daemon=True) + self._reader.start() + + def get_loop(self): + if self.loop is None: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + return self.loop + + # -- signals ----------------------------------------------------------- + + def _on_interrupt_signal(self, signum, frame): + # Only a request in flight can be interrupted: an interrupt that + # lands between two requests is dropped, so it can never leak into + # the next one. + if self.running: + raise _Interrupted() + + def _signal_main(self): + try: + signal.pthread_kill(self.main_thread.ident, _INTERRUPT_SIGNAL) + except Exception: + pass + + # -- writing ----------------------------------------------------------- + + def _write_frame(self, frame_id, status, payload): + body = bytes([status]) + payload + data = _HEADER.pack(frame_id, len(body)) + body + # A signal landing inside sendall would tear the frame and + # desynchronise the stream: hold it until the write is complete. + on_main = threading.current_thread() is self.main_thread + if on_main: + signal.pthread_sigmask(signal.SIG_BLOCK, {_INTERRUPT_SIGNAL}) + try: + with self._wlock: + if self.broken: + raise PipeBroken(self.broken_reason or 'socket to Erlang is closed') + try: + self.sock.sendall(data) + except OSError as exc: + self._mark_broken('write failed: %s' % exc) + raise PipeBroken(self.broken_reason) from None + except BaseException: + # Anything else escaping mid-write leaves a torn frame + self._mark_broken('write interrupted') + raise + finally: + if on_main: + signal.pthread_sigmask(signal.SIG_UNBLOCK, {_INTERRUPT_SIGNAL}) + + def reply(self, frame_id, status, value): + self._write_frame(frame_id, status, _etf.encode(value)) + + def event(self, term): + self._write_frame(0, STATUS_EVENT, _etf.encode(term)) + + def _alloc_id(self): + with self._idlock: + n = self._next_id + self._next_id = n + 1 + return n + + # -- child -> Erlang requests ------------------------------------------- + + def request(self, term, timeout=None): + """Send a status-3 request and wait for its reply. + + On the main thread the wait also serves requests coming from Erlang, + so nested calls work. Returns (status, value).""" + if self.broken: + raise PipeBroken(self.broken_reason) + on_main = threading.current_thread() is self.main_thread + waiter = _Waiter(inbox=self.inbox if on_main else None) + frame_id = self._alloc_id() + with self._plock: + self._pending[frame_id] = waiter + try: + self._write_frame(frame_id, STATUS_CALLBACK, _etf.encode(term)) + except PipeBroken: + with self._plock: + self._pending.pop(frame_id, None) + raise + if on_main: + return self._wait_on_main(waiter) + if not waiter.event.wait(timeout): + with self._plock: + self._pending.pop(frame_id, None) + raise TimeoutError('no reply from Erlang') + return waiter.result + + def request_async(self, term): + """Send a status-3 request; returns an asyncio Future for the reply.""" + if self.broken: + raise PipeBroken(self.broken_reason) + loop = asyncio.get_running_loop() + future = loop.create_future() + waiter = _Waiter(future=future, loop=loop) + frame_id = self._alloc_id() + with self._plock: + self._pending[frame_id] = waiter + try: + self._write_frame(frame_id, STATUS_CALLBACK, _etf.encode(term)) + except PipeBroken: + with self._plock: + self._pending.pop(frame_id, None) + raise + return future + + def _wait_on_main(self, waiter): + while True: + kind, item = self.inbox.get() + if kind == 'reply': + if item is waiter: + return waiter.result + # stale reply for an interrupted wait: drop + continue + if kind == 'request': + self._serve(*item) + elif kind == 'broken': + raise PipeBroken(item) + elif kind == 'interrupt': + if self._exec_stack and self._exec_stack[-1] == item: + raise _Interrupted() + # stale: for a request that already finished + + # -- reader thread ----------------------------------------------------- + + def _reader_main(self): + try: + self._read_loop() + except Exception as exc: # never leave silently + self._mark_broken('reader failed: %r' % (exc,)) + # EOF or error: Erlang is gone (or closed us on purpose). Nothing + # useful can happen in this process any more. _exit so a main thread + # stuck in a C call cannot keep the process alive. + os._exit(0) + + def _read_loop(self): + sock = self.sock + buf = bytearray() + need_hdr = _HEADER_LEN + while True: + try: + data, fds, _flags, _addr = socket.recv_fds(sock, 1024 * 1024, 16) + except InterruptedError: + continue + except OSError as exc: + self._mark_broken('read failed: %s' % exc) + return + if fds: + with self._fds_lock: + self._received_fds.extend(fds) + if not data: + self._mark_broken('Erlang closed the socket') + return + buf += data + # Resumable parse: header, then body, buffering partial frames + while True: + if len(buf) < need_hdr: + break + frame_id, body_len = _HEADER.unpack_from(buf, 0) + total = _HEADER_LEN + body_len + if len(buf) < total: + break + body = bytes(buf[_HEADER_LEN:total]) + del buf[:total] + self._on_frame(frame_id, body) + + def _on_frame(self, frame_id, body): + if not body: + self._mark_broken('empty frame') + return + status = body[0] + try: + term = _etf.decode(body[1:]) if len(body) > 1 else None + except (DecodeError, struct.error, IndexError) as exc: + if status == STATUS_REQUEST: + self.reply(frame_id, STATUS_ERROR, + (Atom('bad_request'), 'malformed frame: %s' % exc)) + return + self._mark_broken('malformed frame from Erlang: %s' % exc) + return + if status in (STATUS_OK, STATUS_ERROR): + with self._plock: + waiter = self._pending.pop(frame_id, None) + if waiter is not None: + waiter.deliver((status, term)) + elif status == STATUS_CONTROL: + self._on_control(term) + elif status == STATUS_REQUEST: + self._on_request(frame_id, term) + else: + self._mark_broken('unexpected status %d from Erlang' % status) + + def _on_control(self, term): + if term == 'interrupt': + self._signal_main() + elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'interrupt': + target = term[1] + stack = list(self._exec_stack) + if target == 'loop': + if self.loop_running: + self._signal_main() + elif stack and stack[-1] == target: + self._signal_main() + elif target in stack: + # An outer request blocked in a callback wait: its wait + # raises when the nested request finishes + self.inbox.put(('interrupt', target)) + # else: already finished (or still queued: cancel handles that) + elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'cancel': + with self._cancel_lock: + self._cancelled.add(term[1]) + # A cancel that arrives after its request ran stays behind; + # keep the set bounded (ids only grow, oldest are stalest) + if len(self._cancelled) > 1024: + for old in sorted(self._cancelled)[:512]: + self._cancelled.discard(old) + elif term == 'stop_loop': + loop = self.loop + if loop is not None and self.loop_running: + loop.call_soon_threadsafe(loop.stop) + + def _on_request(self, frame_id, term): + """Requests that must not wait for the main thread are handled here; + everything else is queued for it.""" + tag = term[0] if isinstance(term, tuple) else term + if tag == 'ping': + self.reply(frame_id, STATUS_OK, Atom('pong')) + elif tag == 'submit': + self._on_submit(frame_id, term) + elif tag == 'stop_loop': + loop = self.loop + if loop is not None and self.loop_running: + loop.call_soon_threadsafe(loop.stop) + self.reply(frame_id, STATUS_OK, Atom('ok')) + else: + self.reply(frame_id, STATUS_ERROR, Atom('no_loop')) + elif self.loop_running and tag != 'shutdown': + # The main thread is inside run_forever: run the request as a + # loop callback so it does not wait for the loop to end. + self.loop.call_soon_threadsafe(self._serve, frame_id, term) + else: + self.inbox.put(('request', (frame_id, term))) + + def _on_submit(self, frame_id, term): + _, task_ref, module, func, args, kwargs = term + loop = self.loop + if loop is None or not self.loop_running: + self.reply(frame_id, STATUS_ERROR, Atom('no_loop')) + return + self.reply(frame_id, STATUS_OK, Atom('ok')) + + def schedule(): + try: + fn = _resolve(module, func) + result = fn(*_as_list(args), **_as_dict(kwargs)) + if inspect.isawaitable(result): + task = asyncio.ensure_future(result) + task.add_done_callback( + lambda t: self._report_task(task_ref, t)) + return + self.event((Atom('async_result'), task_ref, (Atom('ok'), result))) + except BaseException as exc: + self.event((Atom('async_result'), task_ref, + (Atom('error'), _exc_term(exc)))) + loop.call_soon_threadsafe(schedule) + + def _report_task(self, task_ref, task): + try: + if task.cancelled(): + result = (Atom('error'), Atom('cancelled')) + elif task.exception() is not None: + result = (Atom('error'), _exc_term(task.exception())) + else: + result = (Atom('ok'), task.result()) + self.event((Atom('async_result'), task_ref, result)) + except PipeBroken: + pass + + def _mark_broken(self, reason): + if self.broken: + return + self.broken = True + self.broken_reason = reason + with self._plock: + pending = list(self._pending.values()) + self._pending.clear() + for waiter in pending: + waiter.deliver((STATUS_ERROR, reason)) + self.inbox.put(('broken', reason)) + + # -- main thread ------------------------------------------------------- + + def serve_forever(self): + """Main-thread request loop. Returns when Erlang asks for shutdown.""" + while True: + try: + kind, item = self.inbox.get() + except _Interrupted: + continue # interrupt raced with the end of a request + if kind == 'request': + if self._serve(*item) == 'shutdown': + return + elif kind == 'broken': + return + # stale replies are dropped + + def _serve(self, frame_id, term): + """Execute one Erlang request and reply. Returns 'shutdown' when the + child should exit.""" + with self._cancel_lock: + if frame_id in self._cancelled: + self._cancelled.discard(frame_id) + return None # the caller timed out while this was queued + tag = term[0] if isinstance(term, tuple) else term + if tag == 'shutdown': + try: + self.reply(frame_id, STATUS_OK, Atom('ok')) + except PipeBroken: + pass + return 'shutdown' + if tag == 'start_loop': + return self._run_loop(frame_id) + if tag == 'pass_fd': + with self._fds_lock: + fd = self._received_fds.pop(0) if self._received_fds else None + if fd is None: + self.reply(frame_id, STATUS_ERROR, Atom('no_fd_received')) + else: + self.reply(frame_id, STATUS_OK, fd) + return None + + self._exec_stack.append(frame_id) + try: + status, value = self._execute(tag, term) + finally: + self._exec_stack.pop() + try: + self.reply(frame_id, status, value) + except PipeBroken: + pass + except KeyboardInterrupt: + # running is False here so the handler does not raise; a + # stray Ctrl-C style interrupt must still not lose the reply + try: + self.reply(frame_id, status, value) + except (PipeBroken, KeyboardInterrupt): + pass + return None + + def _execute(self, tag, term): + """Run call/eval/exec with interrupt handling. The handler raises only + while `running` is set, and a late signal is absorbed by the retry so + the reply is always sent.""" + prev = self.running + result = None + while result is None: + self.running = True + try: + result = STATUS_OK, self._dispatch(tag, term) + except KeyboardInterrupt: # includes _Interrupted + result = STATUS_ERROR, Atom('interrupted') + except PipeBroken as exc: + result = STATUS_ERROR, (Atom('pipe_broken'), str(exc)) + except StopIteration: + result = STATUS_ERROR, (Atom('StopIteration'), None) + except (SystemExit, GeneratorExit): + self.running = prev + raise + except BaseException as exc: + result = STATUS_ERROR, _exc_term(exc) + finally: + # Cleared first thing so a second signal landing in the + # bookkeeping above is dropped by the handler, not raised + self.running = False + self.running = prev + return result + + def _dispatch(self, tag, term): + if tag == 'init': + return self._init(term) + if tag == 'call': + _, module, func, args, kwargs = term + fn = _resolve(module, func, self.globals) + result = fn(*_as_list(args), **_as_dict(kwargs)) + elif tag == 'eval': + _, code, locals_ = term + loc = dict(self.globals) + loc.update(_as_dict(locals_)) + result = eval(compile(_as_text(code), '', 'eval'), self.globals, loc) + elif tag == 'exec': + _, code = term + exec(compile(_as_text(code), '', 'exec'), self.globals) + return Atom('ok') + else: + raise RuntimeError('unknown request %r' % (tag,)) + if inspect.isawaitable(result): + result = self.get_loop().run_until_complete(result) + return result + + def _init(self, term): + _, context_pid, paths, imports = term + self.context_pid = context_pid + for path in reversed(_as_list(paths)): + path = _as_text(path) + if path not in sys.path: + sys.path.insert(0, path) + import importlib + for name in _as_list(imports): + importlib.import_module(_as_text(name)) + return Atom('ok') + + def _run_loop(self, frame_id): + loop = self.get_loop() + self.loop_running = True + self.reply(frame_id, STATUS_OK, Atom('ok')) + self.running = True + result = Atom('ok') + self._exec_stack.append('loop') + try: + loop.run_forever() + except KeyboardInterrupt: + self.running = False + result = (Atom('error'), Atom('interrupted')) + except BaseException as exc: + self.running = False + result = (Atom('error'), _exc_term(exc)) + finally: + self._exec_stack.pop() + self.running = False + self.loop_running = False + # 10: submits scheduled between loop.stop() and here sit in the + # ready queue; run them into tasks, then cancel everything so + # each reports {error, cancelled} instead of vanishing + _drain_and_cancel(loop) + try: + self.event((Atom('loop_exit'), result)) + except PipeBroken: + pass + return None + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +def _as_list(v): + if v is None: + return [] + if isinstance(v, (list, tuple)): + return list(v) + return [v] + + +def _as_dict(v): + if v is None: + return {} + if isinstance(v, dict): + return {(k if isinstance(k, str) else str(k)): val for k, val in v.items()} + return {} + + +def _as_text(code): + if isinstance(code, bytes): + return code.decode('utf-8') + if isinstance(code, list): # Erlang charlist + return ''.join(chr(c) for c in code) + return code + + +def _resolve(module, func, globals_=None): + module = _as_text(module) + func = _as_text(func) + if module in ('__main__', '') and globals_ is not None and func in globals_: + return globals_[func] + import importlib + if module == '__main__' and globals_ is not None: + raise AttributeError("name '%s' is not defined in the context" % func) + mod = importlib.import_module(module) + try: + return getattr(mod, func) + except AttributeError: + raise AttributeError("module '%s' has no attribute '%s'" % (module, func)) from None + + +def _exc_term(exc): + if isinstance(exc, KeyboardInterrupt): + return Atom('interrupted') + try: + msg = str(exc) + except Exception: + msg = 'unknown' + return (Atom(type(exc).__name__), msg) + + +def _drain_and_cancel(loop): + for _ in range(3): + try: + loop.run_until_complete(asyncio.sleep(0)) + except BaseException: + break + _cancel_all_tasks(loop) + + +def _cancel_all_tasks(loop): + try: + tasks = [t for t in asyncio.all_tasks(loop) if not t.done()] + except RuntimeError: + return + for t in tasks: + t.cancel() + if tasks: + try: + loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True)) + except BaseException: + pass + + +# --------------------------------------------------------------------------- +# The `erlang` module seen by user code +# --------------------------------------------------------------------------- + +def install_erlang_module(runtime): + """Build the `erlang` module for this child and register it in + sys.modules. Mirrors the embedded module's public surface; anything that + cannot cross a process boundary raises a clear RuntimeError.""" + import types + mod = types.ModuleType('erlang', __doc__) + rt = runtime + + def _not_supported(name): + def fn(*args, **kwargs): + raise RuntimeError( + '%s is not available in isolated mode (it needs the embedded ' + 'interpreter); use erlang.call/erlang.send instead' % name) + fn.__name__ = name + return fn + + def call(name, *args, **kwargs): + if kwargs: + raise TypeError('erlang.call takes positional arguments only') + status, value = rt.request((Atom('call'), _as_text(name), list(args))) + if status == STATUS_OK: + return value + raise _callback_error(value) + + async def async_call(name, *args): + return await rt.request_async((Atom('call'), _as_text(name), list(args))) + + def send(pid, message): + if not isinstance(pid, Pid): + raise TypeError('erlang.send: pid must be an erlang.Pid, got %s' + % type(pid).__name__) + status, value = rt.request((Atom('send'), pid, message)) + if status != STATUS_OK: + raise _callback_error(value) + return None + + def whereis(name): + status, value = rt.request((Atom('whereis'), Atom(_as_text(name)))) + if status != STATUS_OK: + raise _callback_error(value) + return value + + def self_(): + return rt.context_pid + + def atom(name): + return Atom(name) + + def is_isolated(): + return True + + def run(main, *, debug=None): + loop = rt.get_loop() + if debug is not None: + loop.set_debug(debug) + return loop.run_until_complete(main) + + def new_event_loop(): + return asyncio.new_event_loop() + + def get_event_loop_policy(): + return asyncio.get_event_loop_policy() + + def install(*, silent=False): + return None + + def spawn_task(coro, *, name=None): + loop = rt.get_loop() + return loop.create_task(coro, name=name) + + def sleep(seconds): + try: + asyncio.get_running_loop() + except RuntimeError: + import time + time.sleep(seconds) + return None + return asyncio.sleep(seconds) + + def log(level, message): + rt.event((Atom('log'), Atom(_as_text(level)), str(message))) + + class Function: + __slots__ = ('name',) + + def __init__(self, name): + self.name = name + + def __call__(self, *args): + return call(self.name, *args) + + def __repr__(self): + return '' % self.name + + def __getattr__(name): + if name.startswith('_'): + raise AttributeError(name) + return Function(name) + + from . import _server as server + + ns = dict( + call=call, async_call=async_call, send=send, whereis=whereis, + self=self_, atom=atom, Atom=Atom, Pid=Pid, Ref=Ref, Port=Port, + ProcessError=ProcessError, SuspensionRequired=SuspensionRequired, + Function=Function, is_isolated=is_isolated, run=run, + new_event_loop=new_event_loop, get_event_loop_policy=get_event_loop_policy, + install=install, spawn_task=spawn_task, sleep=sleep, log=log, + server=server, __getattr__=__getattr__, + schedule=_not_supported('erlang.schedule'), + schedule_py=_not_supported('erlang.schedule_py'), + schedule_inline=_not_supported('erlang.schedule_inline'), + consume_time_slice=lambda *_a, **_k: False, + channel=_not_supported('erlang.channel'), + byte_channel=_not_supported('erlang.byte_channel'), + reactor=_not_supported('erlang.reactor'), + shared_dict=_not_supported('erlang.shared_dict'), + Channel=_not_supported('erlang.Channel'), + ByteChannel=_not_supported('erlang.ByteChannel'), + __all__=['call', 'async_call', 'send', 'whereis', 'self', 'atom', + 'Atom', 'Pid', 'Ref', 'ProcessError', 'SuspensionRequired', + 'run', 'sleep', 'spawn_task', 'server', 'is_isolated'], + ) + mod.__dict__.update(ns) + sys.modules['erlang'] = mod + return mod + + +def format_exception(exc): + return ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__)) diff --git a/priv/py_isolated_child.py b/priv/py_isolated_child.py new file mode 100644 index 0000000..006e074 --- /dev/null +++ b/priv/py_isolated_child.py @@ -0,0 +1,222 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Entry point of an `isolated` context child process. + +Started by src/py_isolated.erl as + + python3 py_isolated_child.py SOCKET_PATH [--rlimit-as BYTES] + [--rlimit-cpu SECONDS] [--rlimit-nofile N] [--cgroup DIR] + +Order of operations matters: limits are applied before anything else is +imported, the parent-death signal is armed before the socket connects, and +the ready/startup_error event is the first frame Erlang sees. +""" + +import os +import sys + + +def _die(reason): + sys.stderr.write('py_isolated_child: %s\n' % reason) + sys.stderr.flush() + os._exit(3) + + +def _parse_args(argv): + if len(argv) < 2: + _die('usage: py_isolated_child.py SOCKET_PATH [options]') + opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None} + i = 2 + while i < len(argv): + flag = argv[i] + if flag in ('--rlimit-as', '--rlimit-cpu', '--rlimit-nofile'): + opts['rlimits'][flag[len('--rlimit-'):]] = int(argv[i + 1]) + i += 2 + elif flag == '--cgroup': + opts['cgroup'] = argv[i + 1] + i += 2 + else: + _die('unknown option %s' % flag) + return opts + + +def _arm_parent_death(): + """Get SIGKILL when the parent (the BEAM) dies: prctl on Linux, procctl + on FreeBSD. Elsewhere the reader thread's EOF handling covers it.""" + SIGKILL = 9 + try: + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + if sys.platform.startswith('linux'): + PR_SET_PDEATHSIG = 1 + libc.prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0) + elif sys.platform.startswith('freebsd'): + # procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &sig) (FreeBSD 11.2+) + P_PID = 0 + PROC_PDEATHSIG_CTL = 11 + sig = ctypes.c_int(SIGKILL) + libc.procctl.argtypes = [ctypes.c_int, ctypes.c_int64, ctypes.c_int, ctypes.c_void_p] + libc.procctl(P_PID, 0, PROC_PDEATHSIG_CTL, ctypes.byref(sig)) + else: + return + # The parent may already be gone between fork and the call + if os.getppid() == 1: + os._exit(0) + except Exception: + pass + + +# macOS does not enforce RLIMIT_AS: `as` is enforced by a watchdog thread +# there (see _start_memory_watchdog) instead of setrlimit. +_AS_VIA_WATCHDOG = sys.platform == 'darwin' + + +def _apply_rlimits(limits): + if not limits: + return [] + import resource + if _AS_VIA_WATCHDOG: + limits = {k: v for k, v in limits.items() if k != 'as'} + names = { + 'as': getattr(resource, 'RLIMIT_AS', None), + 'cpu': getattr(resource, 'RLIMIT_CPU', None), + 'nofile': getattr(resource, 'RLIMIT_NOFILE', None), + } + errors = [] + for key, value in limits.items(): + res = names.get(key) + if res is None: + errors.append((key, 'not supported on this platform')) + continue + try: + _soft, hard = resource.getrlimit(res) + if hard != resource.RLIM_INFINITY and hard < value: + value = hard # cannot raise a hard limit, clamp to it + resource.setrlimit(res, (value, hard)) + except (ValueError, OSError) as exc: + errors.append((key, str(exc))) + return errors + + +def _start_memory_watchdog(limit, runtime): + """Portable memory bound: poll this process's resident set every 50 ms + and exit when it passes `limit`. Erlang reports the in-flight call as + {error, {child_exited, {memory_limit, Bytes}}}.""" + import resource + import threading + from _erlang_impl._etf import Atom + + def rss(): + r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return r if sys.platform == 'darwin' else r * 1024 + + def watch(): + while True: + used = rss() + if used > limit: + try: + runtime.event((Atom('memory_limit'), used)) + except Exception: + pass + os._exit(3) + threading.Event().wait(0.05) + + t = threading.Thread(target=watch, name='erlang-memory-watchdog', daemon=True) + t.start() + + +def _join_cgroup(path): + """cgroup v2, best effort: the directory is created by the operator (or + by Erlang) with the limits already written; we only join it.""" + if not path: + return None + if not sys.platform.startswith('linux'): + return 'cgroups are Linux only (platform %s); use rlimits' % sys.platform + try: + with open(os.path.join(path, 'cgroup.procs'), 'w') as f: + f.write(str(os.getpid())) + return None + except OSError as exc: + return str(exc) + + +def _connect(path): + import socket + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + # Default Unix socket buffers are small (8 KB on macOS); the kernel + # clamps to its maximum, so this is best effort. + for opt in (socket.SO_SNDBUF, socket.SO_RCVBUF): + try: + sock.setsockopt(socket.SOL_SOCKET, opt, 1024 * 1024) + except OSError: + pass + return sock + + +def main(argv): + opts = _parse_args(argv) + _arm_parent_death() + rlimit_errors = _apply_rlimits(opts['rlimits']) + cgroup_error = _join_cgroup(opts['cgroup']) + + priv = os.path.dirname(os.path.abspath(__file__)) + if priv not in sys.path: + sys.path.insert(0, priv) + + try: + sock = _connect(opts['socket']) + except OSError as exc: + _die('cannot connect to %s: %s' % (opts['socket'], exc)) + + from _erlang_impl import _isolated + from _erlang_impl._etf import Atom + + runtime = _isolated.Runtime(sock) + _isolated.install_erlang_module(runtime) + runtime.start() + if _AS_VIA_WATCHDOG and 'as' in opts['rlimits']: + _start_memory_watchdog(opts['rlimits']['as'], runtime) + + if rlimit_errors or cgroup_error: + problems = [(Atom('rlimit'), Atom(k), msg) for k, msg in rlimit_errors] + if cgroup_error: + problems.append((Atom('cgroup'), cgroup_error)) + try: + runtime.event((Atom('startup_error'), problems)) + finally: + os._exit(2) + + info = { + Atom('os_pid'): os.getpid(), + Atom('python_version'): '%d.%d.%d' % sys.version_info[:3], + Atom('executable'): sys.executable, + Atom('platform'): sys.platform, + } + runtime.event((Atom('ready'), info)) + + try: + runtime.serve_forever() + finally: + try: + sock.close() + except OSError: + pass + os._exit(0) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/rebar.config b/rebar.config index c36b4fc..07d555b 100644 --- a/rebar.config +++ b/rebar.config @@ -63,6 +63,7 @@ <<"docs/threading.md">>, <<"docs/asyncio.md">>, <<"docs/workers.md">>, + <<"docs/isolated.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -95,6 +96,7 @@ <<"docs/threading.md">>, <<"docs/asyncio.md">>, <<"docs/workers.md">>, + <<"docs/isolated.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 251cf8f..70a95fa 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "4.1.0"}, + {vsn, "4.2.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py.erl b/src/py.erl index 0d16c9a..8b9218a 100644 --- a/src/py.erl +++ b/src/py.erl @@ -112,6 +112,8 @@ context/0, context/1, interrupt/1, + kill/1, + python_executable/0, start_contexts/0, start_contexts/1, stop_contexts/0, @@ -928,6 +930,19 @@ create_venv(Path, Opts) -> %% @private Get the Python executable path %% When embedded, sys.executable returns the embedding app (beam.smp) %% so we reconstruct the path from sys.prefix and version info +%% @doc Path of the Python interpreter matching the embedded runtime. +%% +%% Reconstructed from `sys.prefix' (when embedded, `sys.executable' is the +%% VM). Used as the default interpreter of isolated contexts and for venvs. +-spec python_executable() -> string(). +python_executable() -> + get_python_executable(). + +%% @doc Kill the child of an isolated context. See py_context:kill/1. +-spec kill(pid()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + py_context:kill(Ctx). + -spec get_python_executable() -> string(). get_python_executable() -> %% Use a single expression to find the Python executable diff --git a/src/py_context.erl b/src/py_context.erl index fa1a72f..d857b87 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -69,6 +69,8 @@ ]). %% Internal exports +-export([kill/1, pass_fd/2, child_info/1]). + -export([init/3, init/4, init_ref_tab/0]). %% Exported for py_reactor_context @@ -83,7 +85,7 @@ %% reply is drained instead of being left in the caller's mailbox. -define(INTERRUPT_GRACE_MS, 1000). --type context_mode() :: worker | owngil. +-type context_mode() :: worker | owngil | isolated. -type context() :: pid(). -export_type([context_mode/0, context/0]). @@ -115,9 +117,12 @@ %% The process creates a Python context based on the mode: %% - `worker' - Create a thread-state worker (main interpreter namespace) %% - `owngil' - Create a sub-interpreter with its own GIL (Python 3.14+) +%% - `isolated' - Run CPython in a child OS process (see py_isolated) %% %% The `owngil' mode creates a dedicated pthread for each context, allowing -%% true parallel Python execution. Requires Python 3.14+. +%% true parallel Python execution. Requires Python 3.14+. The `isolated' +%% mode gives failure isolation: the child can be killed, capped with +%% rlimits, and a crash in a C extension only takes the child down. %% %% @param Id Unique identifier for this context %% @param Mode Context mode @@ -138,17 +143,25 @@ start_link(Id, Mode) -> {ok, pid()} | {error, term()}. start_link(Id, Mode, Opts) when is_map(Opts) -> Parent = self(), - Pid = spawn_link(fun() -> init(Parent, Id, Mode, Opts) end), + Pid = proc_lib:spawn_link(fun() -> init(Parent, Id, Mode, Opts) end), receive {Pid, started} -> {ok, Pid}; {Pid, {error, Reason}} -> {error, Reason} - after 5000 -> + after start_timeout(Mode, Opts) -> exit(Pid, kill), + _ = ets:member(?REF_TAB, Pid) andalso ets:delete(?REF_TAB, Pid), {error, timeout} end. +%% @private An isolated child has to spawn and connect; give it its +%% start_timeout plus a margin. Embedded contexts start in well under 5 s. +start_timeout(isolated, Opts) -> + maps:get(start_timeout, Opts, 10000) + 2000; +start_timeout(_Mode, _Opts) -> + 5000. + %% @doc Stop a py_context process. -spec stop(context()) -> ok. stop(Ctx) when is_pid(Ctx) -> @@ -169,7 +182,7 @@ stop(Ctx) when is_pid(Ctx) -> %% @doc Create a new context with options map. %% %% Options: -%% - `mode' - Context mode (worker | owngil), default: worker +%% - `mode' - Context mode (worker | owngil | isolated), default: worker %% - `memory_limit' - Cap in bytes on memory allocated by this context. %% Requires `mode => owngil' and the runtime started with %% `enable_memory_limits'; see py_nif:context_set_memory_limit/2 for what @@ -417,6 +430,15 @@ get_nif_ref(Ctx) when is_pid(Ctx) -> -spec interrupt(context()) -> ok | not_running. interrupt(Ctx) when is_pid(Ctx) -> case lookup_nif_ref(Ctx) of + {ok, isolated} -> + %% The context process is never blocked in a NIF: ask it. It + %% signals the child and arms the SIGKILL backstop. + MRef = erlang:monitor(process, Ctx), + Ctx ! {interrupt, self(), MRef}, + case await_ctrl_reply(Ctx, MRef, 5000) of + ok -> ok; + _ -> not_running + end; {ok, Ref} -> try py_nif:context_interrupt(Ref) of ok -> ok; @@ -428,6 +450,73 @@ interrupt(Ctx) when is_pid(Ctx) -> not_running end. +%% @doc Kill the child process of an isolated context with SIGKILL. +%% +%% Total and immediate, whatever the child is doing (a C call, a numpy +%% kernel, a blocked read). In-flight calls return `{error, killed}'. With +%% `restart => true' (the default) a fresh child is started and the context +%% stays usable, with its Python state gone. Embedded contexts (`worker', +%% `owngil') cannot be killed: they return `{error, not_isolated}'. +%% +%% @param Ctx Context process +%% @returns ok | {error, not_isolated} +-spec kill(context()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {kill, self(), MRef}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @doc Hand a file descriptor to the child of an isolated context. +%% +%% The fd is sent over the control socket (`SCM_RIGHTS') and the number it +%% got in the child is returned; use it with `erlang.server.serve' from a +%% submitted coroutine. Get the fd with `py:dup_fd/1' on a listening socket. +%% +%% @param Ctx Context process +%% @param Fd File descriptor in this VM +%% @returns {ok, ChildFd} | {error, Reason} +-spec pass_fd(context(), non_neg_integer()) -> {ok, non_neg_integer()} | {error, term()}. +pass_fd(Ctx, Fd) when is_pid(Ctx), is_integer(Fd) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {pass_fd, self(), MRef, Fd}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @doc Information about the child of an isolated context: `os_pid', +%% `python_version', `executable', `platform'. +-spec child_info(context()) -> {ok, map()} | {error, term()}. +child_info(Ctx) when is_pid(Ctx) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {child_info, self(), MRef}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @private Interrupt on behalf of a timed-out request. An isolated context +%% cancels that request only (interrupting the child if it is the one +%% executing, dropping it from the queue otherwise); embedded contexts can +%% only interrupt whatever runs. +interrupt_request(Ctx, ReqMRef) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + Ctx ! {interrupt_request, ReqMRef}, + ok; + _ -> + interrupt(Ctx) + end. + %% @private Create the pid -> NIF reference table. Called by the supervisor %% before any context starts. -spec init_ref_tab() -> ok. @@ -513,6 +602,18 @@ submit(Ctx, Module, Func, Args) -> -spec submit(context(), atom() | binary(), atom() | binary(), list(), map()) -> {ok, reference()} | {error, term()}. submit(Ctx, Module, Func, Args, Kwargs) when is_pid(Ctx), is_list(Args), is_map(Kwargs) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + TaskRef = make_ref(), + MRef = erlang:monitor(process, Ctx), + Ctx ! {submit, self(), MRef, TaskRef, to_binary(Module), to_binary(Func), Args, Kwargs}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + submit_embedded(Ctx, Module, Func, Args, Kwargs) + end. + +%% @private +submit_embedded(Ctx, Module, Func, Args, Kwargs) -> case loop_ref(Ctx) of {ok, LoopRef} -> TaskRef = make_ref(), @@ -563,7 +664,7 @@ await_reply(Ctx, MRef, Timeout) -> {'DOWN', MRef, process, Ctx, Reason} -> {error, {context_died, Reason}} after Timeout -> - _ = interrupt(Ctx), + _ = interrupt_request(Ctx, MRef), receive {MRef, _Late} -> erlang:demonitor(MRef, [flush]); @@ -586,6 +687,9 @@ await_ctrl_reply(Ctx, MRef, Timeout) -> {'DOWN', MRef, process, Ctx, Reason} -> {error, {context_died, Reason}} after Timeout -> + %% An isolated context drops the pending entry so a late reply is + %% not delivered to a caller that stopped waiting + Ctx ! {cancel_ctrl, MRef}, erlang:demonitor(MRef, [flush]), {error, timeout} end. @@ -622,6 +726,8 @@ init(Parent, Id, Mode) -> init(Parent, Id, Mode, #{}). %% @private +init(Parent, Id, isolated, Opts) -> + py_isolated:init(Parent, Id, isolated, Opts); init(Parent, Id, Mode, Opts) -> process_flag(trap_exit, true), case create_context(Mode) of diff --git a/src/py_isolated.erl b/src/py_isolated.erl new file mode 100644 index 0000000..4de2b7a --- /dev/null +++ b/src/py_isolated.erl @@ -0,0 +1,1193 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Context process for `isolated' mode: CPython in a child OS process. +%%% +%%% A py_context started with `mode => isolated' runs this state machine +%%% instead of the embedded loop. The child is spawned through a port (so +%%% the VM reaps it and reports its exit status) and talks over a Unix +%%% socket using the frame format of the blocking callback pipe: +%%% +%%% ``` +%%% <> +%%% Body = <> +%%% ''' +%%% +%%% Status 0 request to the child, 1/2 error/ok reply (either direction), +%%% 3 request from the child (`erlang.call', `erlang.send', `erlang.whereis'), +%%% 4 event from the child, 5 control to the child (`interrupt', `cancel', +%%% `stop_loop'). +%%% +%%% == States == +%%% +%%%
    +%%%
  • `idle' - child up, nothing on its main thread.
    • +%%%
  • `{busy, Id}' - top-level request `Id' executing. Requests from +%%% other callers are postponed (served in order once the child is +%%% free); requests from a process running a callback for this +%%% context are nested and dispatched at once.
    • +%%%
  • `looping' - `start_loop' accepted, `run_forever' on the main +%%% thread; `call/eval/exec' answer `{error, loop_running}'.
    • +%%%
  • `stopping_loop' - `stop_loop' sent, waiting for the loop to exit; +%%% interrupt after the grace period, SIGKILL after `kill_after'.
    • +%%%
  • `{restarting, Reason}' - the child is gone or being killed; +%%% requests are postponed until the new child is up. In-flight +%%% requests fail with `{error, Reason}'.
    • +%%%
+%%% +%%% The message protocol with py_context is unchanged: requests are plain +%%% messages `{call, From, MRef, ...}' answered with `From ! {MRef, Reply}'. +%%% Use `sys:get_state/1' to see the state and `sys:trace/2' for events. +%%% +%%% @private +-module(py_isolated). + +-behaviour(gen_statem). + +-export([init/4, python_executable/1]). + +%% gen_statem callbacks +-export([callback_mode/0, init/1, handle_event/4, terminate/3, code_change/4, + format_status/1]). + +-define(REF_TAB, py_context_refs). +-define(STATUS_REQUEST, 0). +-define(STATUS_ERROR, 1). +-define(STATUS_OK, 2). +-define(STATUS_CALLBACK, 3). +-define(STATUS_EVENT, 4). +-define(STATUS_CONTROL, 5). + +-define(DEFAULT_KILL_AFTER_MS, 1000). +-define(DEFAULT_START_TIMEOUT_MS, 10000). +-define(DEFAULT_MAX_RESTARTS, 5). +-define(DEFAULT_RESTART_PERIOD_MS, 10000). +-define(SHUTDOWN_GRACE_MS, 1000). +-define(EXIT_STATUS_WAIT_MS, 5000). +-define(SOCKET_BUF, 1024 * 1024). + +-record(child, { + port :: port(), + os_pid :: pos_integer(), + listener :: socket:socket() | undefined, + sock :: socket:socket(), + sock_path :: string(), + buf = <<>> :: binary(), + info = #{} :: map() +}). + +%% A running worker loop (state `looping' / `stopping_loop') +-record(loop, { + owner :: pid() | undefined, + owner_mon :: reference() | undefined, + stop_waiters = [] :: [{pid(), reference()}] +}). + +-record(data, { + id :: term(), + parent :: pid() | undefined, + opts :: map(), + child :: #child{} | undefined, + next_id = 1 :: pos_integer(), + %% Id => {From, MRef, Kind} for requests the child has + pending = #{} :: map(), + %% TaskRef => SubmitterPid for submit/5 results + tasks = #{} :: map(), + %% MonitorRef => FrameId of callbacks running in their own process, + %% and Pid => MonitorRef of those processes (their requests are nested) + callbacks = #{} :: map(), + cb_pids = #{} :: map(), + restarts = [] :: [integer()], + loop :: #loop{} | undefined, + %% Request id (or `loop') the armed kill backstop is bound to + kill_target :: pos_integer() | loop | undefined, + %% Callers of kill/1 answered once the new child is up + kill_waiters = [] :: [{pid(), reference()}] +}). + +-define(IS_MAIN(K), (K =:= call orelse K =:= eval orelse K =:= exec orelse + (is_tuple(K) andalso element(1, K) =:= start_loop))). + +-type state() :: idle | {busy, pos_integer()} | looping | stopping_loop + | {restarting, term()}. +-export_type([state/0]). + +%% ============================================================================ +%% Entry point (called from py_context:init/4 in the context process) +%% ============================================================================ + +%% @private Runs in the process py_context:start_link/3 spawned with +%% proc_lib. The child start is bounded by `start_timeout'; the parent +%% is answered before the state machine takes over. +init(Parent, Id, _Mode, Opts) -> + process_flag(trap_exit, true), + %% gen_statem stops when its OTP parent (head of '$ancestors') exits, + %% for any reason. A context outlives the process that created it, as + %% the embedded contexts do; only a crash of that process (non-normal + %% EXIT, handled below) or a shutdown stops it. Point the parent slot at + %% ourselves so gen_statem leaves the creator's exit to us. + put('$ancestors', [self() | case get('$ancestors') of + L when is_list(L) -> L; + _ -> [] + end]), + ets:insert(?REF_TAB, {self(), isolated}), + Data0 = #data{id = Id, parent = Parent, opts = Opts}, + case start_child(Data0) of + {ok, Data} -> + Parent ! {self(), started}, + %% Arm the socket select before handling events + {State, Data1, Actions} = drain_socket(idle, Data, []), + gen_statem:enter_loop(?MODULE, [], State, Data1, Actions); + {error, Reason} -> + ets:delete(?REF_TAB, self()), + Parent ! {self(), {error, Reason}} + end. + +%% ============================================================================ +%% gen_statem callbacks +%% ============================================================================ + +callback_mode() -> + [handle_event_function, state_enter]. + +%% @private Not used: the process is started through init/4 and +%% gen_statem:enter_loop/5. +init(_Args) -> + {stop, use_init_4}. + +%% ---- state enter ----------------------------------------------------------- + +handle_event(enter, _Old, idle, #data{kill_waiters = Waiters} = Data) -> + [W ! {M, ok} || {W, M} <- Waiters], + {keep_state, Data#data{kill_waiters = []}}; +handle_event(enter, _Old, {restarting, _}, _Data) -> + %% SIGKILL was sent (or the child is exiting): the port reports it + %% within milliseconds; this is the safety net + {keep_state_and_data, [{state_timeout, ?EXIT_STATUS_WAIT_MS, exit_status}]}; +handle_event(enter, _Old, stopping_loop, _Data) -> + keep_state_and_data; +handle_event(enter, _Old, _State, _Data) -> + keep_state_and_data; + +%% ---- socket and port ------------------------------------------------------- + +handle_event(info, {'$socket', S, select, _}, State, #data{child = #child{sock = S}} = Data) -> + result(drain_socket(State, Data, [])); +handle_event(info, {'$socket', S, abort, {_, Reason}}, State, #data{child = #child{sock = S}} = Data) -> + result(socket_broken(Reason, State, Data, [])); +handle_event(info, {'$socket', _, _, _}, _State, _Data) -> + keep_state_and_data; +handle_event(info, {Port, {data, Out}}, _State, #data{child = #child{port = Port}} = Data) -> + log_output(Data, Out), + keep_state_and_data; +handle_event(info, {Port, {exit_status, Status}}, State, #data{child = #child{port = Port}} = Data) -> + child_exited(exit_reason(Status), State, Data); +handle_event(info, {Port, _}, _State, _Data) when is_port(Port) -> + keep_state_and_data; +handle_event(state_timeout, exit_status, {restarting, _} = State, #data{child = Child} = Data) -> + logger:error("py_context ~p (isolated): child ~p did not exit after SIGKILL", + [Data#data.id, Child#child.os_pid]), + child_exited({signal, 9}, State, Data); + +%% ---- requests -------------------------------------------------------------- + +handle_event(info, {call, From, MRef, Module, Func, Args, Kwargs}, State, Data) -> + request(From, MRef, call, {call, Module, Func, Args, Kwargs}, State, Data); +handle_event(info, {call, From, MRef, Module, Func, Args, Kwargs, _EnvRef}, State, Data) -> + request(From, MRef, call, {call, Module, Func, Args, Kwargs}, State, Data); +handle_event(info, {eval, From, MRef, Code, Locals}, State, Data) -> + request(From, MRef, eval, {eval, iolist_to_binary(Code), Locals}, State, Data); +handle_event(info, {eval, From, MRef, Code, Locals, _EnvRef}, State, Data) -> + request(From, MRef, eval, {eval, iolist_to_binary(Code), Locals}, State, Data); +handle_event(info, {exec, From, MRef, Code}, State, Data) -> + request(From, MRef, exec, {exec, iolist_to_binary(Code)}, State, Data); +handle_event(info, {exec, From, MRef, Code, _EnvRef}, State, Data) -> + request(From, MRef, exec, {exec, iolist_to_binary(Code)}, State, Data); +handle_event(info, {start_loop, From, MRef, Owner}, State, Data) -> + request(From, MRef, {start_loop, Owner}, start_loop, State, Data); +handle_event(info, {submit, From, MRef, TaskRef, Module, Func, Args, Kwargs}, State, Data) -> + request(From, MRef, {submit, TaskRef}, {submit, TaskRef, Module, Func, Args, Kwargs}, + State, Data); +handle_event(info, {pass_fd, From, MRef, Fd}, State, Data) -> + pass_fd(From, MRef, Fd, State, Data); +handle_event(info, {call_method, From, MRef, _ObjRef, _Method, _Args}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; + +%% ---- introspection --------------------------------------------------------- + +handle_event(info, {get_interp_id, From, MRef}, _State, #data{child = Child}) -> + Id = case Child of + #child{os_pid = P} -> P; + _ -> 0 + end, + From ! {MRef, {ok, Id}}, + keep_state_and_data; +handle_event(info, {is_subinterp, From, MRef}, _State, _Data) -> + From ! {MRef, true}, + keep_state_and_data; +handle_event(info, {create_local_env, From, MRef}, _State, _Data) -> + %% Process-local environments are a NIF feature; the child has one + %% namespace per context. A fresh ref keeps py:call(Ctx, ...) working. + From ! {MRef, {ok, make_ref()}}, + keep_state_and_data; +handle_event(info, {get_nif_ref, From, MRef}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; +handle_event(info, {loop_ref, From, MRef}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; +handle_event(info, {child_info, From, MRef}, _State, #data{child = Child}) -> + Info = case Child of + #child{os_pid = P, info = I} -> I#{os_pid => P}; + undefined -> #{} + end, + From ! {MRef, {ok, Info}}, + keep_state_and_data; +handle_event(info, {cancel_ctrl, MRef}, _State, #data{pending = Pending} = Data) -> + %% The caller of a control request stopped waiting: drop the entry so + %% the late reply is not delivered + Drop = [Id || {Id, {_, M, _}} <- maps:to_list(Pending), M =:= MRef], + {keep_state, Data#data{pending = maps:without(Drop, Pending)}}; + +%% ---- interrupt, kill, stop ------------------------------------------------- + +handle_event(info, {interrupt, From, MRef}, State, Data) -> + case executing_request(State, Data) of + undefined -> + From ! {MRef, not_running}, + keep_state_and_data; + Target -> + From ! {MRef, ok}, + result(send_interrupt(Target, State, Data, [])) + end; +handle_event(info, {interrupt_request, ReqMRef}, State, #data{pending = Pending} = Data) -> + %% A timed-out request: if the child has it, interrupt that request + %% only. If it is still postponed here nothing happens: py_context has + %% already stopped waiting, and the reply goes nowhere. + case [Id || {Id, {_, M, _}} <- maps:to_list(Pending), M =:= ReqMRef] of + [Id] -> result(send_interrupt(Id, State, Data, [])); + [] -> keep_state_and_data + end; +handle_event({timeout, kill}, Target, State, #data{pending = Pending} = Data) -> + Still = case Target of + loop -> State =:= looping orelse State =:= stopping_loop; + Id -> maps:is_key(Id, Pending) + end, + case Still andalso Data#data.child =/= undefined of + true -> + logger:warning("py_context ~p (isolated): interrupt not honoured, killing child", + [Data#data.id]), + kill(killed, State, Data#data{kill_target = undefined}); + false -> + {keep_state, Data#data{kill_target = undefined}} + end; +handle_event(info, {kill, From, MRef}, State, Data) -> + case State of + {restarting, _} -> + %% Already on its way: answer with the others when it is back + {keep_state, Data#data{kill_waiters = [{From, MRef} | Data#data.kill_waiters]}}; + _ -> + kill(killed, State, Data#data{kill_waiters = [{From, MRef} | Data#data.kill_waiters]}) + end; +handle_event(info, {stop, From, MRef}, _State, Data) -> + Data1 = stop_child(Data, graceful), + From ! {MRef, ok}, + {stop, normal, Data1}; + +%% ---- worker loop ----------------------------------------------------------- + +handle_event(info, {stop_loop, From, MRef, GraceMs}, looping, #data{loop = Loop} = Data) -> + _ = send_frame(Data#data.child, 0, ?STATUS_CONTROL, stop_loop), + Loop1 = Loop#loop{stop_waiters = [{From, MRef} | Loop#loop.stop_waiters]}, + {next_state, stopping_loop, Data#data{loop = Loop1}, + [{state_timeout, GraceMs, interrupt}]}; +handle_event(info, {stop_loop, From, MRef, _GraceMs}, stopping_loop, #data{loop = Loop} = Data) -> + Loop1 = Loop#loop{stop_waiters = [{From, MRef} | Loop#loop.stop_waiters]}, + {keep_state, Data#data{loop = Loop1}}; +handle_event(info, {stop_loop, From, MRef, _GraceMs}, _State, _Data) -> + From ! {MRef, {error, no_loop}}, + keep_state_and_data; +handle_event(state_timeout, interrupt, stopping_loop, Data) -> + %% Cooperative stop did not land: interrupt, then the kill backstop + result(send_interrupt(loop, stopping_loop, Data, [])); +handle_event(info, {'DOWN', Mon, process, _Owner, _Reason}, looping, + #data{loop = #loop{owner_mon = Mon} = Loop} = Data) -> + %% Owner is gone: nobody will hear the exit, stop the loop + _ = send_frame(Data#data.child, 0, ?STATUS_CONTROL, stop_loop), + Loop1 = Loop#loop{owner = undefined, owner_mon = undefined}, + {next_state, stopping_loop, Data#data{loop = Loop1}, + [{state_timeout, 5000, interrupt}]}; + +%% ---- callbacks (child -> Erlang) ------------------------------------------- + +handle_event(info, {callback_reply, FrameId, Status, Term}, State, + #data{child = Child} = Data) when Child =/= undefined -> + case State of + {restarting, _} -> + keep_state_and_data; + _ -> + case send_frame(Child, FrameId, Status, Term) of + ok -> keep_state_and_data; + {error, Reason} -> result(socket_broken(Reason, State, Data, [])) + end + end; +handle_event(info, {callback_reply, _, _, _}, _State, _Data) -> + keep_state_and_data; +handle_event(info, {'DOWN', Mon, process, Pid, Reason}, State, #data{callbacks = Cbs} = Data) -> + case maps:take(Mon, Cbs) of + {Id, Rest} -> + Data1 = Data#data{callbacks = Rest, cb_pids = maps:remove(Pid, Data#data.cb_pids)}, + case Reason of + normal -> + {keep_state, Data1}; + _ -> + %% The callback process died without replying: the + %% child must not wait for it + Msg = iolist_to_binary(io_lib:format("callback crashed: ~p", [Reason])), + handle_event(info, {callback_reply, Id, ?STATUS_ERROR, Msg}, State, Data1) + end; + error -> + keep_state_and_data + end; + +%% ---- exits ----------------------------------------------------------------- + +handle_event(info, {'EXIT', Parent, Reason}, _State, #data{parent = Parent}) when Reason =/= normal -> + %% Whoever started us is gone: do not leave a child without an owner + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, Reason}, _State, _Data) + when Reason =:= shutdown; Reason =:= kill -> + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, {shutdown, _} = Reason}, _State, _Data) -> + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, _Reason}, _State, _Data) -> + keep_state_and_data; +handle_event(info, _Other, _State, _Data) -> + keep_state_and_data. + +terminate(Reason, _State, #data{child = Child} = Data) -> + _ = case Child of + undefined -> Data; + _ when Reason =:= normal; Reason =:= shutdown -> stop_child(Data, graceful); + _ -> stop_child(Data, kill) + end, + ets:delete(?REF_TAB, self()), + ok. + +code_change(_OldVsn, State, Data, _Extra) -> + {ok, State, Data}. + +%% @private Keep sys:get_status readable: the socket buffer is noise +format_status(#{data := #data{child = #child{} = Child} = Data} = Status) -> + Status#{data => Data#data{child = Child#child{buf = <<>>}}}; +format_status(Status) -> + Status. + +%% @doc Python executable used for isolated children: the `python' option, +%% then the `isolated_python' application env, then the interpreter matching +%% the embedded runtime, then `python3' from PATH. +-spec python_executable(map()) -> string() | {error, term()}. +python_executable(Opts) -> + Candidate = case maps:get(python, Opts, undefined) of + undefined -> + case application:get_env(erlang_python, isolated_python) of + {ok, P} -> P; + undefined -> default_python() + end; + P -> P + end, + resolve_exe(to_list(Candidate)). + +default_python() -> + case persistent_term:get({?MODULE, python}, undefined) of + undefined -> + Exe = try py:python_executable() catch _:_ -> "python3" end, + persistent_term:put({?MODULE, python}, Exe), + Exe; + Exe -> + Exe + end. + +resolve_exe(Exe) -> + case filename:pathtype(Exe) of + absolute -> + case filelib:is_file(Exe) of + true -> Exe; + false -> {error, {python_not_found, Exe}} + end; + _ -> + case os:find_executable(Exe) of + false -> {error, {python_not_found, Exe}}; + Found -> Found + end + end. + +%% ============================================================================ +%% Child startup +%% ============================================================================ + +start_child(#data{opts = Opts} = St) -> + case check_platform_opts(Opts) of + ok -> start_child_1(St); + {error, _} = Err -> Err + end. + +%% cgroups exist only on Linux; rlimits are POSIX and apply everywhere. +%% RLIMIT_AS is enforced by the kernel on Linux and FreeBSD; on macOS the +%% child enforces `as' with a watchdog thread on its resident set. +check_platform_opts(Opts) -> + case {maps:get(cgroup, Opts, undefined), os:type()} of + {undefined, _} -> ok; + {_, {unix, linux}} -> ok; + {_, {unix, Os}} -> {error, {cgroup_unsupported, Os}} + end. + +start_child_1(#data{opts = Opts} = St) -> + case python_executable(Opts) of + {error, _} = Err -> + Err; + Python -> + case spawn_child(Python, Opts) of + {ok, Child} -> + handshake(St#data{child = Child}); + {error, _} = Err -> + Err + end + end. + +spawn_child(Python, Opts) -> + Dir = sock_dir(), + Path = filename:join(Dir, "ctx_" ++ integer_to_list(erlang:unique_integer([positive])) ++ ".sock"), + _ = file:delete(Path), + case socket:open(local, stream, default) of + {ok, L} -> + try + ok = socket:bind(L, #{family => local, path => Path}), + ok = socket:listen(L), + Script = filename:join(priv_dir(), "py_isolated_child.py"), + Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts)], + PortOpts = [exit_status, stderr_to_stdout, binary, use_stdio, + {args, Args}, {env, env_opt(Opts)}], + Port = open_port({spawn_executable, Python}, PortOpts), + OsPid = case erlang:port_info(Port, os_pid) of + {os_pid, Pid} -> Pid; + _ -> 0 + end, + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + case accept_child(L, Port, Timeout) of + {ok, S} -> + _ = file:delete(Path), + tune_socket(S), + {ok, #child{port = Port, os_pid = OsPid, listener = L, + sock = S, sock_path = Path}}; + {error, Reason} -> + _ = file:delete(Path), + socket:close(L), + kill_port(Port, OsPid), + {error, Reason} + end + catch + Class:Err:Stack -> + _ = file:delete(Path), + socket:close(L), + {error, {spawn_failed, {Class, Err, Stack}}} + end; + {error, Reason} -> + {error, {socket_open_failed, Reason}} + end. + +%% Accept while also watching the port: a child that dies before connecting +%% (bad interpreter, missing script) is reported with its output. +accept_child(L, Port, Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + accept_child(L, Port, Deadline, []). + +accept_child(L, Port, Deadline, Out) -> + case socket:accept(L, nowait) of + {ok, S} -> + %% Output printed before connecting is still worth logging + [self() ! {Port, {data, D}} || D <- lists:reverse(Out)], + {ok, S}; + {select, {select_info, _, Handle}} -> + Left = max(0, Deadline - erlang:monotonic_time(millisecond)), + receive + {'$socket', L, select, Handle} -> + accept_child(L, Port, Deadline, Out); + {Port, {exit_status, Status}} -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {child_exited_at_start, exit_reason(Status), + drain_port_output(Port, Out)}}; + {Port, {data, D}} -> + %% Keep it here, not in the mailbox: re-sending it would + %% make this receive return at once and never time out + accept_child(L, Port, Deadline, [D | Out]) + after Left -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {start_timeout, drain_port_output(Port, Out)}} + end; + {error, Reason} -> + {error, {accept_failed, Reason}} + end. + +%% Default Unix socket buffers are small (8 KB on macOS); large payloads +%% would cross in hundreds of wakeups. Best effort: the kernel clamps. +tune_socket(S) -> + _ = socket:setopt(S, {otp, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, sndbuf}, ?SOCKET_BUF), + ok. + +drain_port_output(Port, Acc) -> + receive + {Port, {data, D}} -> drain_port_output(Port, [D | Acc]) + after 50 -> + iolist_to_binary(lists:reverse(Acc)) + end. + +%% Blocking handshake: ready event, init request, then the preload exec. +handshake(#data{child = Child, opts = Opts} = St0) -> + St = St0#data{}, + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + case recv_frame_sync(Child, Timeout) of + {ok, {0, ?STATUS_EVENT, {ready, Info}}, Child1} -> + St1 = St#data{child = Child1#child{info = Info}}, + Paths = [to_bin(P) || P <- py_import:all_paths()] ++ extra_paths(Opts), + %% Registered imports are pre-cached in sys.modules, as + %% interp_apply_imports does for the embedded modes + Imports = lists:usort([to_bin(M) || {M, _} <- py_import:all_imports()]), + case sync_request(St1, {init, self(), Paths, Imports}, Timeout) of + {{ok, _}, St2} -> + run_preload(St2, Timeout); + {{error, Reason}, St2} -> + stop_child(St2, kill), + {error, {init_failed, Reason}} + end; + {ok, {0, ?STATUS_EVENT, {startup_error, Problems}}, Child1} -> + stop_child(St#data{child = Child1}, kill), + {error, {startup_error, Problems}}; + {ok, {0, ?STATUS_EVENT, {memory_limit, Rss}}, Child1} -> + %% The memory watchdog fired before the child was ready + stop_child(St#data{child = Child1}, kill), + {error, {startup_error, [{memory_limit, Rss}]}}; + {ok, Other, Child1} -> + stop_child(St#data{child = Child1}, kill), + {error, {unexpected_handshake, Other}}; + {error, Reason} -> + stop_child(St, kill), + {error, {handshake_failed, Reason}} + end. + +run_preload(#data{opts = Opts} = St, Timeout) -> + Code = case maps:get(preload, Opts, undefined) of + undefined -> py_preload_code(); + C -> [py_preload_code(), <<"\n">>, iolist_to_binary(C)] + end, + case iolist_to_binary(Code) of + <<>> -> + {ok, St}; + Bin -> + case sync_request(St, {exec, Bin}, Timeout) of + {{ok, _}, St1} -> + {ok, St1}; + {{error, Reason}, St1} -> + logger:warning("py_context ~p (isolated): preload failed: ~p", + [St#data.id, Reason]), + {ok, St1} + end + end. + +%% Global preload registered with py_preload (applied to every context) +py_preload_code() -> + try py_preload:get_code() of + Code when is_binary(Code) -> Code; + _ -> <<>> + catch + _:_ -> <<>> + end. + +extra_paths(Opts) -> + [to_bin(P) || P <- maps:get(paths, Opts, [])]. + +%% Send a request and wait for its reply, ignoring nothing: callbacks made +%% by the child during startup are served too. +sync_request(#data{child = Child, next_id = Id} = St, Term, Timeout) -> + case send_frame(Child, Id, ?STATUS_REQUEST, Term) of + ok -> sync_wait(St#data{next_id = Id + 1}, Id, Timeout); + {error, Reason} -> {{error, Reason}, St} + end. + +run_callback_bounded(Term, Timeout) -> + {Pid, Mon} = spawn_monitor(fun() -> exit({callback_done, run_callback(Term)}) end), + receive + {'DOWN', Mon, process, Pid, {callback_done, Result}} -> + Result; + {'DOWN', Mon, process, Pid, Reason} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("callback crashed: ~p", [Reason]))} + after Timeout -> + erlang:demonitor(Mon, [flush]), + exit(Pid, kill), + {?STATUS_ERROR, <<"callback timed out during context start">>} + end. + +sync_wait(#data{child = Child} = St, Id, Timeout) -> + case recv_frame_sync(Child, Timeout) of + {ok, {FrameId, ?STATUS_CALLBACK, Term}, Child1} -> + %% Callbacks made while the child starts (preload, imports) run + %% in a separate process so a crash cannot take this one down. + %% They cannot re-enter this context yet: the loop is not + %% running, so such a call would wait for the handshake timeout. + St1 = St#data{child = Child1}, + {Status, Reply} = run_callback_bounded(Term, Timeout), + case send_frame(Child1, FrameId, Status, Reply) of + ok -> sync_wait(St1, Id, Timeout); + {error, Reason} -> {{error, Reason}, St1} + end; + {ok, {Id, Status, Term}, Child1} + when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> + {reply_term(Status, Term), St#data{child = Child1}}; + {ok, {0, ?STATUS_EVENT, {log, Level, Msg}}, Child1} -> + log_event(St, Level, Msg), + sync_wait(St#data{child = Child1}, Id, Timeout); + {ok, {_, _, _}, Child1} -> + sync_wait(St#data{child = Child1}, Id, Timeout); + {error, Reason} -> + {{error, Reason}, St} + end. + +recv_frame_sync(#child{buf = Buf} = Child, Timeout) -> + case parse_frame(Buf) of + {ok, Frame, Rest} -> + {ok, Frame, Child#child{buf = Rest}}; + more -> + case socket:recv(Child#child.sock, 0, Timeout) of + {ok, <<>>} -> + {error, closed}; + {ok, Data} -> + recv_frame_sync(Child#child{buf = <>}, Timeout); + {error, {Reason, _Data}} -> + {error, Reason}; + {error, Reason} -> + {error, Reason} + end + end. + +%% ============================================================================ +%% Requests to the child +%% ============================================================================ + +%% Every path returns a gen_statem result. Main-thread requests (call, eval, +%% exec, start_loop) are served one caller at a time; the rest is handled by +%% the child's reader thread and can go in any state that has a child. +request(From, MRef, {start_loop, _}, _Term, State, _Data) + when State =:= looping; State =:= stopping_loop -> + From ! {MRef, {error, already_running}}, + keep_state_and_data; +request(From, MRef, Kind, _Term, looping, _Data) when ?IS_MAIN(Kind) -> + From ! {MRef, {error, loop_running}}, + keep_state_and_data; +request(_From, _MRef, _Kind, _Term, {restarting, _}, _Data) -> + {keep_state_and_data, [postpone]}; +request(_From, _MRef, Kind, _Term, stopping_loop, _Data) when ?IS_MAIN(Kind) -> + {keep_state_and_data, [postpone]}; +request(From, MRef, Kind, Term, {busy, _} = State, #data{cb_pids = CbPids} = Data) + when ?IS_MAIN(Kind) -> + case maps:is_key(From, CbPids) of + true -> + %% Nested: a callback of the executing request calls back in + result(dispatch(From, MRef, Kind, Term, State, Data)); + false -> + {keep_state_and_data, [postpone]} + end; +request(From, MRef, Kind, Term, idle, Data) when ?IS_MAIN(Kind) -> + case dispatch(From, MRef, Kind, Term, idle, Data) of + {idle, Data1, Actions} -> + {next_state, {busy, Data1#data.next_id - 1}, Data1, Actions}; + Other -> + result(Other) + end; +request(From, MRef, Kind, Term, State, Data) -> + result(dispatch(From, MRef, Kind, Term, State, Data)). + +dispatch(From, MRef, Kind, Term, State, #data{child = Child, next_id = Id, pending = Pending} = Data) -> + case send_frame(Child, Id, ?STATUS_REQUEST, Term) of + ok -> + {State, Data#data{next_id = Id + 1, pending = Pending#{Id => {From, MRef, Kind}}}, []}; + {error, Reason} -> + From ! {MRef, {error, {child_exited, Reason}}}, + socket_broken(Reason, State, Data, []) + end. + +pass_fd(_From, _MRef, _Fd, {restarting, _}, _Data) -> + {keep_state_and_data, [postpone]}; +pass_fd(From, MRef, Fd, _State, #data{child = Child, next_id = Id, pending = Pending} = Data) + when is_integer(Fd), Fd >= 0 -> + Frame = frame(Id, ?STATUS_REQUEST, term_to_binary(pass_fd)), + Msg = #{iov => [Frame], + ctrl => [#{level => socket, type => rights, data => <>}]}, + case socket:sendmsg(Child#child.sock, Msg) of + ok -> + {keep_state, Data#data{next_id = Id + 1, pending = Pending#{Id => {From, MRef, pass_fd}}}}; + {error, Reason} -> + From ! {MRef, {error, {pass_fd_failed, Reason}}}, + keep_state_and_data + end; +pass_fd(From, MRef, Fd, _State, _Data) -> + From ! {MRef, {error, {invalid_fd, Fd}}}, + keep_state_and_data. + +%% Turn a {State, Data, Actions} triple into a gen_statem result +result({stop, Reason, Data}) -> + {stop, Reason, Data}; +result({State, Data, Actions}) -> + {next_state, State, Data, Actions}. + +%% ============================================================================ +%% Frames from the child +%% ============================================================================ + +%% Read until the socket would block, processing complete frames. +drain_socket({restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +drain_socket(State, #data{child = #child{sock = S, buf = Buf} = Child} = Data, Actions) -> + case socket:recv(S, 0, nowait) of + {ok, <<>>} -> + socket_broken(closed, State, Data, Actions); + {ok, Bytes} -> + Data1 = Data#data{child = Child#child{buf = <>}}, + case process_frames(State, Data1, Actions) of + {{restarting, _}, _, _} = Broken -> Broken; + {State1, Data2, Actions1} -> drain_socket(State1, Data2, Actions1) + end; + {select, _SelectInfo} -> + process_frames(State, Data, Actions); + {error, {Reason, Bytes}} when is_binary(Bytes) -> + Data1 = Data#data{child = Child#child{buf = <>}}, + {State1, Data2, Actions1} = process_frames(State, Data1, Actions), + socket_broken(Reason, State1, Data2, Actions1); + {error, Reason} -> + socket_broken(Reason, State, Data, Actions) + end. + +process_frames({restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +process_frames(State, #data{child = #child{buf = Buf} = Child} = Data, Actions) -> + case parse_frame(Buf) of + {ok, Frame, Rest} -> + Data1 = Data#data{child = Child#child{buf = Rest}}, + {State1, Data2, Actions1} = handle_frame(Frame, State, Data1, Actions), + process_frames(State1, Data2, Actions1); + more -> + {State, Data, Actions}; + {error, Reason} -> + socket_broken({malformed_frame, Reason}, State, Data, Actions) + end. + +parse_frame(<>) -> + case Body of + <> -> + try + Term = case Payload of + <<>> -> undefined; + _ -> binary_to_term(Payload) + end, + {ok, {Id, Status, Term}, Rest} + catch + error:badarg -> {error, bad_etf} + end; + <<>> -> + {error, empty_body} + end; +parse_frame(_) -> + more. + + +handle_frame({Id, Status, Term}, State, #data{pending = Pending} = Data, Actions) + when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> + case maps:take(Id, Pending) of + {{From, MRef, Kind}, Rest} -> + {Data1, Actions1} = cancel_kill_timer(Id, Data#data{pending = Rest}, Actions), + {Next, Data2} = deliver(Kind, From, MRef, reply_term(Status, Term), Data1), + %% Only the reply of the request holding the main thread frees + %% it; nested replies (callbacks calling back in) do not + State1 = case {Next, State} of + {looping, _} -> looping; + {done, {busy, Id}} -> idle; + _ -> State + end, + {State1, Data2, Actions1}; + error -> + {State, Data, Actions} + end; +handle_frame({Id, ?STATUS_CALLBACK, Term}, State, #data{callbacks = Cbs, cb_pids = CbPids} = Data, Actions) -> + Owner = self(), + {Pid, Mon} = spawn_monitor(fun() -> + {Status, Reply} = run_callback(Term), + Owner ! {callback_reply, Id, Status, Reply} + end), + {State, Data#data{callbacks = Cbs#{Mon => Id}, cb_pids = CbPids#{Pid => Mon}}, Actions}; +handle_frame({_, ?STATUS_EVENT, Event}, State, Data, Actions) -> + on_child_event(Event, State, Data, Actions); +handle_frame({_, _, _}, State, Data, Actions) -> + {State, Data, Actions}. + +%% Deliver a reply. Returns `done' for a main-thread request, `looping' +%% when a loop just started, `keep' otherwise. +deliver(exec, From, MRef, {ok, _}, Data) -> + From ! {MRef, ok}, + {done, Data}; +deliver({start_loop, Owner}, From, MRef, {ok, _}, Data) -> + Mon = case is_pid(Owner) of + true -> erlang:monitor(process, Owner); + false -> undefined + end, + From ! {MRef, ok}, + {looping, Data#data{loop = #loop{owner = Owner, owner_mon = Mon}}}; +deliver({submit, TaskRef}, From, MRef, {ok, _}, #data{tasks = Tasks} = Data) -> + From ! {MRef, {ok, TaskRef}}, + {keep, Data#data{tasks = Tasks#{TaskRef => From}}}; +deliver(Kind, From, MRef, Reply, Data) when ?IS_MAIN(Kind) -> + From ! {MRef, Reply}, + {done, Data}; +deliver(_Kind, From, MRef, Reply, Data) -> + From ! {MRef, Reply}, + {keep, Data}. + +on_child_event({async_result, TaskRef, Result}, State, #data{tasks = Tasks} = Data, Actions) -> + case maps:take(TaskRef, Tasks) of + {Pid, Rest} -> + Pid ! {async_result, TaskRef, Result}, + {State, Data#data{tasks = Rest}, Actions}; + error -> + {State, Data, Actions} + end; +on_child_event({loop_exit, Result}, State, Data, Actions) + when State =:= looping; State =:= stopping_loop -> + {Data1, Actions1} = cancel_kill_timer(loop, Data, Actions), + {idle, loop_exited(Result, Data1), Actions1}; +on_child_event({memory_limit, Rss}, State, Data, Actions) -> + %% The child's memory watchdog is exiting; the exit_status follows + socket_broken({memory_limit, Rss}, State, Data, Actions); +on_child_event({log, Level, Msg}, State, Data, Actions) -> + log_event(Data, Level, Msg), + {State, Data, Actions}; +on_child_event(_, State, Data, Actions) -> + {State, Data, Actions}. + +reply_term(?STATUS_OK, Term) -> {ok, Term}; +reply_term(?STATUS_ERROR, Term) -> {error, Term}. + +%% --------------------------------------------------------------------------- +%% Callbacks (child -> Erlang) +%% --------------------------------------------------------------------------- + +run_callback({call, Name, Args}) -> + ArgsList = case Args of + L when is_list(L) -> L; + T when is_tuple(T) -> tuple_to_list(T); + _ -> [Args] + end, + try py_callback:execute(to_bin(Name), ArgsList) of + {ok, Result} -> + {?STATUS_OK, Result}; + {error, {not_found, N}} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("Function '~s' not registered", [N]))}; + {error, {Class, Reason, _Stack}} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("~p: ~p", [Class, Reason]))} + catch + Class:Reason -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("~p:~p", [Class, Reason]))} + end; +run_callback({send, Pid, Msg}) when is_pid(Pid) -> + case node(Pid) =:= node() andalso not is_process_alive(Pid) of + true -> {?STATUS_ERROR, {noproc, Pid}}; + false -> Pid ! Msg, {?STATUS_OK, ok} + end; +run_callback({send, Other, _}) -> + {?STATUS_ERROR, {badarg, Other}}; +run_callback({whereis, Name}) -> + try + Atom = if is_atom(Name) -> Name; + is_binary(Name) -> binary_to_existing_atom(Name, utf8); + is_list(Name) -> list_to_existing_atom(Name) + end, + case erlang:whereis(Atom) of + undefined -> {?STATUS_OK, none}; + Pid -> {?STATUS_OK, Pid} + end + catch + _:_ -> {?STATUS_OK, none} + end; +run_callback(Other) -> + {?STATUS_ERROR, {unknown_request, Other}}. + + +%% ============================================================================ +%% Interrupt / kill +%% ============================================================================ + +%% What an interrupt targets: the innermost main-thread request the child +%% is executing (nested requests are dispatched while the outer waits), or +%% the loop. +executing_request(looping, _Data) -> loop; +executing_request(stopping_loop, _Data) -> loop; +executing_request({busy, _}, #data{pending = Pending}) -> + lists:max([Id || {Id, {_, _, Kind}} <- maps:to_list(Pending), ?IS_MAIN(Kind)]); +executing_request(_State, _Data) -> undefined. + +%% The child signals only if Target is what it is executing, so an +%% interrupt for a request that just completed cannot hit its successor. +%% The kill backstop is bound to Target. +send_interrupt(Target, State, #data{opts = Opts} = Data, Actions) -> + case send_frame(Data#data.child, 0, ?STATUS_CONTROL, {interrupt, Target}) of + ok -> + After = maps:get(kill_after, Opts, ?DEFAULT_KILL_AFTER_MS), + {State, Data#data{kill_target = Target}, + [{{timeout, kill}, After, Target} | Actions]}; + {error, Reason} -> + socket_broken(Reason, State, Data, Actions) + end. + +%% A reply for the interrupted request means the interrupt landed +cancel_kill_timer(Target, #data{kill_target = Target} = Data, Actions) -> + {Data#data{kill_target = undefined}, [{{timeout, kill}, cancel} | Actions]}; +cancel_kill_timer(_Target, Data, Actions) -> + {Data, Actions}. + +%% SIGKILL the child; the port's exit_status drives the restart. Callers +%% of kill/1 are answered when the new child is idle. +kill(Reason, State, #data{child = #child{port = Port, os_pid = OsPid}} = Data) -> + kill_port(Port, OsPid), + result(enter_restarting(Reason, State, Data, [])); +kill(_Reason, _State, _Data) -> + keep_state_and_data. + +kill_port(Port, OsPid) -> + case OsPid > 0 andalso erlang:port_info(Port) =/= undefined of + true -> _ = py_nif:os_kill(OsPid, 9), ok; + false -> ok + end. + + +%% ============================================================================ +%% Failure handling and restart +%% ============================================================================ + +%% Nothing can reach the child any more. Make sure it exits; the port's +%% exit_status (which follows within milliseconds) fails the pending +%% requests with the real cause and runs the restart policy. +socket_broken(_Reason, {restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +socket_broken(Reason, State, #data{child = #child{port = Port, os_pid = OsPid}} = Data, Actions) -> + logger:debug("py_context ~p (isolated): socket to child ~p broken: ~p", + [Data#data.id, OsPid, Reason]), + kill_port(Port, OsPid), + enter_restarting({child_exited, {socket, Reason}}, State, Data, Actions). + +enter_restarting(_Reason, {restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +enter_restarting(Reason, _State, Data, Actions) -> + %% Timers of the old child are meaningless now + {{restarting, Reason}, Data#data{kill_target = undefined}, + [{{timeout, kill}, cancel} | Actions]}. + +exit_reason(Status) when Status > 128 -> {signal, Status - 128}; +exit_reason(Status) -> {exit_status, Status}. + +%% The port reported the child's exit: fail what was in flight, then +%% restart within the budget or stop. +child_exited(Reason, State, #data{child = Child, opts = Opts} = Data0) -> + close_child(Child), + FailReason = case {State, Reason} of + {{restarting, killed}, _} -> killed; + %% The memory watchdog announced the exit; our SIGKILL may win the race + {{restarting, {child_exited, {socket, {memory_limit, _} = Mem}}}, _} -> {child_exited, Mem}; + %% We killed it because the socket broke: report the socket, unless + %% the child was already dying of something more telling + {{restarting, {child_exited, {socket, _}} = SockReason}, {signal, 9}} -> SockReason; + {{restarting, {child_exited, {socket, _}}}, _} -> {child_exited, Reason}; + {{restarting, Other}, _} -> Other; + {_, _} -> {child_exited, Reason} + end, + Data1 = fail_pending(FailReason, Data0#data{child = undefined, kill_target = undefined}), + case FailReason of + killed -> + logger:info("py_context ~p (isolated): child killed", [Data0#data.id]); + _ -> + logger:warning("py_context ~p (isolated): child exited: ~p", + [Data0#data.id, FailReason]) + end, + case maps:get(restart, Opts, true) andalso restart_allowed(Data1) of + true -> + Now = erlang:monotonic_time(millisecond), + Data2 = Data1#data{restarts = [Now | Data1#data.restarts]}, + case start_child(Data2) of + {ok, Data3} -> + logger:info("py_context ~p (isolated): child restarted (pid ~p)", + [Data0#data.id, (Data3#data.child)#child.os_pid]), + result(drain_socket(idle, Data3, [{{timeout, kill}, cancel}])); + {error, RestartError} -> + logger:error("py_context ~p (isolated): restart failed: ~p", + [Data0#data.id, RestartError]), + {stop, {child_restart_failed, RestartError}, Data1} + end; + false -> + {stop, {child_exited, Reason}, Data1} + end. + +%% In-flight requests, submitted tasks and a running loop fail with Reason. +%% Postponed requests are untouched: they are served by the next child, or +%% their callers get a DOWN if the process stops. +fail_pending(Reason, #data{pending = Pending, tasks = Tasks} = Data) -> + maps:foreach(fun(_, {From, MRef, _Kind}) -> + From ! {MRef, {error, Reason}} + end, Pending), + maps:foreach(fun(TaskRef, Pid) -> + Pid ! {async_result, TaskRef, {error, Reason}} + end, Tasks), + Data1 = case Data#data.loop of + undefined -> Data; + _ -> loop_exited({error, Reason}, Data) + end, + Data1#data{pending = #{}, tasks = #{}}. + +restart_allowed(#data{restarts = Restarts, opts = Opts}) -> + Max = maps:get(max_restarts, Opts, ?DEFAULT_MAX_RESTARTS), + Period = maps:get(restart_period, Opts, ?DEFAULT_RESTART_PERIOD_MS), + Now = erlang:monotonic_time(millisecond), + Recent = [T || T <- Restarts, Now - T =< Period], + length(Recent) < Max. + + +%% Graceful: ask the child to exit, wait briefly, then SIGKILL. +stop_child(#data{child = undefined} = Data, _How) -> + Data; +stop_child(#data{child = #child{port = Port, os_pid = OsPid} = Child} = Data, How) -> + case How of + graceful -> + _ = send_frame(Child, 0, ?STATUS_REQUEST, shutdown), + receive + {Port, {exit_status, _}} -> ok + after ?SHUTDOWN_GRACE_MS -> + kill_port(Port, OsPid), + wait_exit(Port) + end; + _ -> + kill_port(Port, OsPid), + wait_exit(Port) + end, + close_child(Child), + Data1 = fail_pending({child_exited, stopped}, Data), + Data1#data{child = undefined}. + +wait_exit(Port) -> + receive + {Port, {exit_status, _}} -> ok + after 2000 -> + ok + end. + +close_child(#child{port = Port, sock = S, listener = L}) -> + _ = socket:close(S), + _ = socket:close(L), + try port_close(Port) catch error:badarg -> ok end, + ok. + + +%% --------------------------------------------------------------------------- +%% Worker loop helpers +%% --------------------------------------------------------------------------- + +loop_exited(Result, #data{loop = #loop{owner = Owner, owner_mon = Mon, + stop_waiters = Waiters}} = Data) -> + case Mon of + undefined -> ok; + _ -> erlang:demonitor(Mon, [flush]) + end, + case is_pid(Owner) of + true -> Owner ! {py_loop_exit, self(), Result}; + false -> ok + end, + [W ! {M, ok} || {W, M} <- Waiters], + Data#data{loop = undefined}; +loop_exited(_Result, Data) -> + Data. + +%% --------------------------------------------------------------------------- +%% Wire helpers +%% --------------------------------------------------------------------------- + +frame(Id, Status, Payload) -> + Body = <>, + <>. + +send_frame(#child{sock = S}, Id, Status, Term) -> + case socket:send(S, frame(Id, Status, term_to_binary(Term))) of + ok -> ok; + {error, {Reason, _Rest}} -> {error, Reason}; + {error, Reason} -> {error, Reason} + end. + +log_output(#data{id = Id, child = #child{os_pid = OsPid}}, Data) -> + Lines = binary:split(Data, <<"\n">>, [global, trim_all]), + [logger:info("py_context ~p (isolated pid ~p): ~s", [Id, OsPid, L]) || L <- Lines], + ok. + +log_event(#data{id = Id}, Level, Msg) -> + Lvl = case Level of + error -> error; warning -> warning; debug -> debug; _ -> info + end, + logger:log(Lvl, "py_context ~p (isolated): ~s", [Id, Msg]). + +sock_dir() -> + Base = case os:getenv("TMPDIR") of + false -> "/tmp"; + T -> T + end, + Dir = filename:join(Base, "erlang_python_" ++ os:getpid()), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + _ = file:change_mode(Dir, 8#700), + Dir. + +priv_dir() -> + case code:priv_dir(erlang_python) of + {error, bad_name} -> + filename:join(filename:dirname(filename:dirname(code:which(?MODULE))), "priv"); + Dir -> + Dir + end. + +rlimit_args(Opts) -> + Limits = maps:get(rlimits, Opts, #{}), + lists:append([case maps:get(K, Limits, undefined) of + undefined -> []; + V when is_integer(V), V >= 0 -> ["--rlimit-" ++ atom_to_list(K), integer_to_list(V)] + end || K <- [as, cpu, nofile]]). + +cgroup_args(Opts) -> + case maps:get(cgroup, Opts, undefined) of + undefined -> []; + Dir -> ["--cgroup", to_list(Dir)] + end. + +env_opt(Opts) -> + [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))]. + +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); +to_bin(B) when is_binary(B) -> B. + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(B) when is_binary(B) -> unicode:characters_to_list(B); +to_list(L) when is_list(L) -> L. diff --git a/src/py_nif.erl b/src/py_nif.erl index 0357bc6..e868dc8 100644 --- a/src/py_nif.erl +++ b/src/py_nif.erl @@ -131,6 +131,7 @@ close_fd/1, %% File descriptor utilities dup_fd/1, + os_kill/2, %% Test helpers for fd monitoring (using pipes) create_test_pipe/0, close_test_fd/1, @@ -992,6 +993,11 @@ create_test_pipe() -> dup_fd(_Fd) -> ?NIF_STUB. +%% @doc Send a signal to an OS process (kill(2)). Used by isolated contexts. +-spec os_kill(pos_integer(), non_neg_integer()) -> ok | {error, esrch | eperm | einval}. +os_kill(_Pid, _Signal) -> + ?NIF_STUB. + %% @doc Close a test file descriptor. -spec close_test_fd(integer()) -> ok | {error, term()}. close_test_fd(_Fd) -> diff --git a/test/py_isolated_SUITE.erl b/test/py_isolated_SUITE.erl new file mode 100644 index 0000000..937bd9d --- /dev/null +++ b/test/py_isolated_SUITE.erl @@ -0,0 +1,874 @@ +%%% @doc Common Test suite for `isolated' context mode. +%%% +%%% The interpreter runs in a child OS process. Round-trip cases run in a +%%% worker group too, so the two modes are held to the same results. The +%%% isolation cases (kill, segfault, rlimits, reaping, socket break) are what +%%% the embedded modes cannot do; the ones marked "contrast" assert the +%%% embedded behaviour as well, to document the difference. +-module(py_isolated_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + test_call_eval_exec/1, + test_state_persists/1, + test_kwargs/1, + test_type_round_trip/1, + test_python_error/1, + test_missing_module_and_function/1, + test_large_payloads/1, + test_callback_round_trip/1, + test_nested_callback/1, + test_callback_error/1, + test_send_to_pid/1, + test_concurrent_callers/1, + test_timeout_interrupts_sleep/1, + test_queued_timeout_does_not_interrupt_others/1, + test_sys_state_reflects_activity/1, + test_requests_during_restart_are_served/1, + test_kill_reply_after_restart/1, + test_stop_while_busy_and_looping/1, + test_sys_get_status/1, + test_context_outlives_creator/1, + test_pool_of_isolated_contexts/1, + test_child_info/1, + test_sleep_is_interrupted/1, + test_sleep_not_interrupted_in_worker/1, + test_kill_backstop/1, + test_kill_restarts_child/1, + test_segfault_kills_only_child/1, + test_rlimit_as/1, + test_rlimit_cpu/1, + test_rlimit_nofile/1, + test_reaped_on_stop/1, + test_reaped_on_crash/1, + test_reaped_on_kill/1, + test_no_orphan_when_vm_dies/1, + test_socket_break_mid_call/1, + test_restart_false_exits_context/1, + test_restart_budget/1, + test_numpy_imports/1, + test_not_supported_fail_loud/1, + test_bad_python_fails_at_start/1, + test_startup_error_reported/1, + test_cgroup_option_platform/1, + test_env_option/1, + test_preload_option/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> + [{group, worker}, {group, isolated}, {group, isolation}]. + +groups() -> + RoundTrip = [ + test_call_eval_exec, + test_state_persists, + test_kwargs, + test_type_round_trip, + test_python_error, + test_missing_module_and_function, + test_large_payloads, + test_callback_round_trip, + test_nested_callback, + test_callback_error, + test_send_to_pid, + test_concurrent_callers, + test_timeout_interrupts_sleep + ], + Isolation = [ + test_pool_of_isolated_contexts, + test_child_info, + test_sleep_is_interrupted, + test_sleep_not_interrupted_in_worker, + test_kill_backstop, + test_kill_restarts_child, + test_segfault_kills_only_child, + test_rlimit_as, + test_rlimit_cpu, + test_rlimit_nofile, + test_reaped_on_stop, + test_reaped_on_crash, + test_reaped_on_kill, + test_no_orphan_when_vm_dies, + test_socket_break_mid_call, + test_restart_false_exits_context, + test_restart_budget, + test_numpy_imports, + test_not_supported_fail_loud, + test_queued_timeout_does_not_interrupt_others, + test_sys_state_reflects_activity, + test_requests_during_restart_are_served, + test_kill_reply_after_restart, + test_stop_while_busy_and_looping, + test_sys_get_status, + test_context_outlives_creator, + test_bad_python_fails_at_start, + test_startup_error_reported, + test_cgroup_option_platform, + test_env_option, + test_preload_option + ], + [{worker, [], RoundTrip}, + {isolated, [], RoundTrip}, + {isolation, [], Isolation}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, test_dir()} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(isolation, Config) -> + [{mode, isolated} | Config]; +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Round-trip cases (both modes) +%%% ============================================================================ + +test_call_eval_exec(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, add, [1, 2]), + {ok, 4.0} = py_context:call(C, math, sqrt, [16]), + {ok, 6} = py_context:eval(C, <<"2*3">>), + {ok, 10} = py_context:eval(C, <<"a + b">>, #{a => 4, b => 6}), + ok = py_context:exec(C, <<"def twice(x):\n return 2 * x\n">>), + {ok, 14} = py_context:call(C, '__main__', twice, [7]), + stop(C). + +test_state_persists(Config) -> + C = new_ctx(Config), + ok = py_context:exec(C, <<"counter = 0">>), + lists:foreach(fun(_) -> ok = py_context:exec(C, <<"counter += 1">>) end, + lists:seq(1, 10)), + {ok, 10} = py_context:eval(C, <<"counter">>), + stop(C). + +test_kwargs(Config) -> + C = new_ctx(Config), + {ok, {[1, 2], [{<<"a">>, 3}, {<<"b">>, <<"x">>}]}} = + py_context:call(C, ?TEST_MOD, kwargs_probe, [1, 2], #{a => 3, b => <<"x">>}), + stop(C). + +test_type_round_trip(Config) -> + C = new_ctx(Config), + Probe = fun(V) -> + {ok, Got} = py_context:call(C, ?TEST_MOD, identity, [V]), + Got + end, + TypeOf = fun(V) -> + {ok, T} = py_context:call(C, ?TEST_MOD, type_name, [V]), + T + end, + true = Probe(true), + false = Probe(false), + none = Probe(none), + none = Probe(undefined), + none = Probe(nil), + <<"str">> = TypeOf(<<"héllo"/utf8>>), + <<"héllo"/utf8>> = Probe(<<"héllo"/utf8>>), + <<"bytes">> = TypeOf(<<255, 0, 1>>), + <<255, 0, 1>> = Probe(<<255, 0, 1>>), + <<"bytes">> = TypeOf({bytes, <<"abc">>}), + <<"abc">> = Probe({bytes, <<"abc">>}), + 42 = Probe(42), + -1 = Probe(-1), + %% Integers beyond 64 bits: the NIF converter has no bignum path + %% (worker mode returns none); the ETF codec carries them exactly. + case ?config(mode, Config) of + isolated -> + Big = 1 bsl 100, + Big = Probe(Big), + NegBig = -(1 bsl 100), + NegBig = Probe(NegBig); + _ -> + ok + end, + 3.5 = Probe(3.5), + [] = Probe([]), + [1, [2, 3], {4}] = Probe([1, [2, 3], {4}]), + "abc" = Probe("abc"), + {1, 2, 3} = Probe({1, 2, 3}), + #{<<"k">> := [1, 2], 3 := {<<"a">>}} = Probe(#{<<"k">> => [1, 2], 3 => {a}}), + <<"some_atom">> = Probe(some_atom), + <<"str">> = TypeOf(some_atom), + Self = self(), + Self = Probe(Self), + <<"Pid">> = TypeOf(Self), + Ref = make_ref(), + Ref = Probe(Ref), + <<"Ref">> = TypeOf(Ref), + {ok, nan} = py_context:eval(C, <<"float('nan')">>), + {ok, infinity} = py_context:eval(C, <<"float('inf')">>), + {ok, neg_infinity} = py_context:eval(C, <<"float('-inf')">>), + {ok, {1, 2, 3}} = py_context:eval(C, <<"(1, 2, 3)">>), + stop(C). + +test_python_error(Config) -> + C = new_ctx(Config), + {error, {'ValueError', Msg}} = py_context:call(C, ?TEST_MOD, raise_value_error, [<<"boom">>]), + true = lists:prefix("boom", to_list(Msg)), + {error, {'ZeroDivisionError', _}} = py_context:eval(C, <<"1/0">>), + {error, {'SyntaxError', _}} = py_context:exec(C, <<"def (:">>), + %% Still usable + {ok, 2} = py_context:eval(C, <<"1+1">>), + stop(C). + +test_missing_module_and_function(Config) -> + C = new_ctx(Config), + {error, {'ModuleNotFoundError', _}} = py_context:call(C, no_such_module_xyz, f, []), + {error, {'AttributeError', _}} = py_context:call(C, math, no_such_function, []), + stop(C). + +test_large_payloads(Config) -> + C = new_ctx(Config), + lists:foreach(fun(Size) -> + Bin = crypto:strong_rand_bytes(Size), + {ok, Bin} = py_context:call(C, ?TEST_MOD, identity, [Bin]), + {ok, Out} = py_context:call(C, ?TEST_MOD, big_payload, [Size]), + Size = byte_size(Out) + end, [1024 * 1024, 16 * 1024 * 1024]), + stop(C). + +test_callback_round_trip(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_double">>, fun([X]) -> X * 2 end), + {ok, 84} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_double">>, 42]), + {ok, 84} = py_context:eval(C, <<"__import__('erlang').call('iso_double', 42)">>), + %% Attribute sugar: erlang.iso_double(...) + {ok, 84} = py_context:eval(C, <<"__import__('erlang').iso_double(42)">>), + py_callback:unregister(<<"iso_double">>), + stop(C). + +%% @doc A callback that calls back into the same context (nesting), two +%% levels deep. The socket protocol nests arbitrarily; worker mode's +%% suspension protocol does not, so that group only checks it is loud. +test_nested_callback(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_nested">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, add, [X, 1]), + R + end), + py_callback:register(<<"iso_nested2">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, X]), + R + 100 + end), + case ?config(mode, Config) of + isolated -> + {ok, 11} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, 10]), + {ok, 111} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested2">>, 10]); + worker -> + case py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, 10]) of + {ok, 11} -> ok; + {error, _} -> ok + end + end, + py_callback:unregister(<<"iso_nested">>), + py_callback:unregister(<<"iso_nested2">>), + stop(C). + +test_callback_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_crash">>, fun(_) -> error(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, callback_error_type, [<<"iso_crash">>]), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, callback_error_type, [<<"iso_not_registered">>]), + py_callback:unregister(<<"iso_crash">>), + stop(C). + +test_send_to_pid(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), {hello, 1}]), + receive {<<"hello">>, 1} -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_concurrent_callers(Config) -> + C = new_ctx(Config), + Self = self(), + N = 20, + Pids = [spawn_link(fun() -> + Results = [py_context:call(C, ?TEST_MOD, add, [I, J]) || J <- lists:seq(1, 25)], + Self ! {done, I, Results} + end) || I <- lists:seq(1, N)], + lists:foreach(fun(I) -> + receive + {done, I, Results} -> + Expected = [{ok, I + J} || J <- lists:seq(1, 25)], + Expected = Results + after 30000 -> + ct:fail({timeout, I}) + end + end, lists:seq(1, N)), + _ = Pids, + stop(C). + +test_timeout_interrupts_sleep(Config) -> + C = new_ctx(Config), + T0 = erlang:monotonic_time(millisecond), + {error, timeout} = py_context:eval(C, <<"__import__('time').sleep(0.3)">>, #{}, 100), + Elapsed = erlang:monotonic_time(millisecond) - T0, + %% Both modes return promptly; the sleep itself ends by the time we + %% call again in worker mode (0.3 s), immediately in isolated mode. + true = Elapsed < 1500, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop(C). + +%%% ============================================================================ +%%% Isolation cases +%%% ============================================================================ + +test_pool_of_isolated_contexts(_Config) -> + {ok, Ctxs} = py_context_router:start_pool(iso_pool, 4, isolated), + 4 = length(Ctxs), + Pids = lists:usort([begin + {ok, #{os_pid := P}} = py_context:child_info(Cx), P + end || Cx <- Ctxs]), + 4 = length(Pids), + [{ok, 4} = py_context:eval(Cx, <<"2+2">>) || Cx <- Ctxs], + {ok, 4.0} = py:call(iso_pool, math, sqrt, [16]), + ok = py_context_router:stop_pool(iso_pool), + timer:sleep(200), + [false = os_pid_alive(P) || P <- Pids], + ok. + +test_child_info(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid, python_version := V, executable := Exe}} = py_context:child_info(C), + true = is_integer(Pid) andalso Pid > 0, + true = is_binary(V), + true = is_binary(Exe), + true = os_pid_alive(Pid), + {error, not_isolated} = py_context:child_info(self()), + stop(C). + +%% @doc The case the embedded modes cannot pass: a blocking C call is +%% interrupted at once. +test_sleep_is_interrupted(Config) -> + C = new_ctx(Config), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:call(C, ?TEST_MOD, sleep_then, [60, ok])} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:interrupt(C), + receive + {result, R} -> + {error, interrupted} = R, + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("interrupted after ~p ms", [Elapsed]), + true = Elapsed < 1000 + after 5000 -> + ct:fail(sleep_not_interrupted) + end, + %% Same child, still usable, state intact + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc Contrast: in worker mode the interrupt only lands when the C call +%% returns, so a 1.5 s sleep takes its full time. +test_sleep_not_interrupted_in_worker(_Config) -> + {ok, C} = py_context:new(#{mode => worker}), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:eval(C, <<"__import__('time').sleep(1.5)">>)} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + _ = py_context:interrupt(C), + receive + {result, _} -> + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("worker returned after ~p ms", [Elapsed]), + true = Elapsed >= 1000 + after 10000 -> + ct:fail(worker_never_returned) + end, + py_context:stop(C), + ok. + +%% @doc Signals blocked in the child: the soft interrupt cannot land and the +%% kill backstop fires after kill_after. +test_kill_backstop(Config) -> + C = new_ctx(Config, #{kill_after => 300}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:call(C, ?TEST_MOD, blocked_sleep, [60])} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:interrupt(C), + receive + {result, {error, killed}} -> + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("killed after ~p ms", [Elapsed]), + true = Elapsed < 3000 + after 10000 -> + ct:fail(backstop_did_not_fire) + end, + false = os_pid_alive(Pid1), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_kill_restarts_child(Config) -> + C = new_ctx(Config), + ok = py_context:exec(C, <<"state = 'before'">>), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:kill(C), + false = os_pid_alive(Pid1), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + %% State is gone, the context is not + {error, {'NameError', _}} = py_context:eval(C, <<"state">>), + {ok, 4} = py_context:eval(C, <<"2+2">>), + {error, not_isolated} = py_context:kill(self()), + stop(C). + +%% @doc The headline case: a segfault kills one child, the node and every +%% other context survive. +test_segfault_kills_only_child(Config) -> + C = new_ctx(Config), + Other = new_ctx(Config), + {ok, W} = py_context:new(#{mode => worker}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + {error, {child_exited, {signal, Sig}}} = py_context:call(C, ?TEST_MOD, segfault, []), + true = is_segfault_signal(Sig), + false = os_pid_alive(Pid1), + true = is_process_alive(C), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid2 =/= Pid1, + {ok, 4} = py_context:eval(C, <<"2+2">>), + {ok, 4} = py_context:eval(Other, <<"2+2">>), + {ok, 4} = py_context:eval(W, <<"2+2">>), + py_context:stop(W), + stop(Other), + stop(C). + +%% @doc `as' is enforced by the kernel (Linux, FreeBSD) or by the child's +%% RSS watchdog (macOS); either way the allocation fails or the child dies, +%% the node is unaffected and the context recovers. +test_rlimit_as(Config) -> + case sanitized_child() of + true -> {skip, "sanitizer runtime in the child needs unbounded address space"}; + false -> test_rlimit_as_1(Config) + end. + +test_rlimit_as_1(Config) -> + %% Free-threaded CPython reserves a large address range at startup, so + %% a limit that a regular build fits in keeps it from even importing + %% the socket module. + Probe = new_ctx(Config), + FreeThreaded = py_context:eval(Probe, + <<"hasattr(__import__('sys'), '_is_gil_enabled') and not __import__('sys')._is_gil_enabled()">>), + stop(Probe), + case FreeThreaded of + {ok, true} -> {skip, "free-threaded CPython needs an as limit far above test sizes"}; + _ -> test_rlimit_as_2(Config) + end. + +test_rlimit_as_2(Config) -> + C = new_ctx(Config, #{rlimits => #{as => 1024 * 1024 * 1024}}), + Result = py_context:call(C, ?TEST_MOD, allocate_and_touch, [2 * 1024 * 1024 * 1024], #{}, 120000), + ct:log("allocate past as limit: ~p", [Result]), + case {Result, rlimit_as_enforced()} of + {{error, {'MemoryError', _}}, _} -> ok; + {{error, {child_exited, {memory_limit, _}}}, false} -> ok; + {{error, {child_exited, _}}, true} -> ok; + Other -> ct:fail({unexpected, Other}) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_rlimit_cpu(Config) -> + C = new_ctx(Config, #{rlimits => #{cpu => 1}}), + Result = py_context:call(C, ?TEST_MOD, spin, [30], #{}, 20000), + ct:log("spin past RLIMIT_CPU: ~p", [Result]), + %% SIGXCPU (24 on Linux and BSD/macOS) + {error, {child_exited, {signal, Sig}}} = Result, + true = Sig =:= 24 orelse Sig =:= 30, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_rlimit_nofile(Config) -> + C = new_ctx(Config, #{rlimits => #{nofile => 32}}), + {ok, 32} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_NOFILE)[0]">>), + stop(C). + +test_reaped_on_stop(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:stop(C), + wait_gone(Pid), + ok. + +test_reaped_on_crash(Config) -> + C = new_ctx(Config, #{restart => false}), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + unlink(C), + Mon = erlang:monitor(process, C), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + receive {'DOWN', Mon, process, C, _} -> ok after 5000 -> ct:fail(context_survived) end, + wait_gone(Pid), + ok. + +test_reaped_on_kill(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:kill(C), + wait_gone(Pid), + stop(C). + +%% @doc A child of another VM must not outlive that VM. +test_no_orphan_when_vm_dies(_Config) -> + case peer_available() of + false -> + {skip, "peer module not available"}; + true -> + %% standard_io works without distribution + {ok, Peer, _Node} = peer:start_link(#{ + connection => standard_io, + args => lists:append([["-pa", P] || P <- code:get_path()]) + }), + {ok, _} = peer:call(Peer, application, ensure_all_started, [erlang_python]), + {ok, C} = peer:call(Peer, py_context, new, [#{mode => isolated}]), + {ok, #{os_pid := Pid}} = peer:call(Peer, py_context, child_info, [C]), + true = os_pid_alive(Pid), + %% Park the child in a blocking C call so only the EOF watchdog + %% (or PDEATHSIG) can end it + ok = peer:cast(Peer, py_context, eval, [C, <<"__import__('time').sleep(60)">>]), + timer:sleep(300), + %% Hard stop: the peer VM is killed, nothing in it runs cleanup + peer:stop(Peer), + wait_gone(Pid), + ok + end. + +%% @doc The socket breaks under a pending call: it fails with a clear error, +%% the next call does not hang, and the restart recovers. +test_socket_break_mid_call(Config) -> + C = new_ctx(Config), + Result = py_context:call(C, ?TEST_MOD, close_control_socket, [], #{}, 10000), + ct:log("call under socket break: ~p", [Result]), + {error, {child_exited, _}} = Result, + T0 = erlang:monotonic_time(millisecond), + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + true = erlang:monotonic_time(millisecond) - T0 < 3000, + stop(C). + +test_restart_false_exits_context(Config) -> + C = new_ctx(Config, #{restart => false}), + unlink(C), + Mon = erlang:monitor(process, C), + {error, {child_exited, {signal, Sig}}} = py_context:call(C, ?TEST_MOD, segfault, []), + true = is_segfault_signal(Sig), + receive + {'DOWN', Mon, process, C, {child_exited, {signal, Sig}}} -> ok + after 5000 -> + ct:fail(context_did_not_exit) + end, + {error, {context_died, _}} = py_context:eval(C, <<"1">>), + ok. + +test_restart_budget(Config) -> + C = new_ctx(Config, #{max_restarts => 2, restart_period => 60000}), + unlink(C), + Mon = erlang:monitor(process, C), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + {ok, 4} = py_context:eval(C, <<"2+2">>), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + {ok, 4} = py_context:eval(C, <<"2+2">>), + %% Third crash exceeds the budget + _ = py_context:call(C, ?TEST_MOD, segfault, []), + receive {'DOWN', Mon, process, C, _} -> ok after 5000 -> ct:fail(budget_not_enforced) end, + ok. + +test_numpy_imports(Config) -> + C = new_ctx(Config), + case py_context:eval(C, <<"__import__('importlib.util').util.find_spec('numpy') is not None">>) of + {ok, true} -> + {ok, 45} = py_context:call(C, ?TEST_MOD, numpy_sum, [10]), + {ok, [[1, 2], [3, 4]]} = py_context:eval(C, <<"__import__('numpy').array([[1,2],[3,4]])">>), + stop(C); + _ -> + stop(C), + {skip, "numpy not installed for the child interpreter"} + end. + +%% @doc A caller whose request is still queued times out: its request is +%% dropped, the request that is executing is not interrupted, and the kill +%% backstop does not fire because the context is busy. +test_queued_timeout_does_not_interrupt_others(Config) -> + C = new_ctx(Config, #{kill_after => 200}), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + Self = self(), + %% Occupies the child for 1.5 s + spawn_link(fun() -> + Self ! {long, py_context:call(C, ?TEST_MOD, sleep_then, [1.5, done], #{}, 10000)} + end), + timer:sleep(100), + %% Queued behind it, gives up after 200 ms + {error, timeout} = py_context:eval(C, <<"'never'">>, #{}, 200), + receive + {long, R} -> {ok, <<"done">>} = R + after 5000 -> + ct:fail(long_call_lost) + end, + %% Same child, no kill happened, and the cancelled eval never ran + {ok, #{os_pid := Pid}} = py_context:child_info(C), + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop(C). + +%% @doc The gen_statem state names what the context is doing. +test_sys_state_reflects_activity(Config) -> + C = new_ctx(Config), + {idle, _} = sys:get_state(C), + Self = self(), + spawn_link(fun() -> Self ! {done, py_context:call(C, ?TEST_MOD, sleep_then, [0.5, x])} end), + timer:sleep(100), + {{busy, Id}, _} = sys:get_state(C), + true = is_integer(Id), + receive {done, {ok, <<"x">>}} -> ok after 5000 -> ct:fail(no_reply) end, + {idle, _} = sys:get_state(C), + ok = py_context:start_loop(C), + {looping, _} = sys:get_state(C), + ok = py_context:stop_loop(C), + {idle, _} = sys:get_state(C), + stop(C). + +%% @doc A request arriving while the child restarts waits for the new child +%% instead of failing. +test_requests_during_restart_are_served(Config) -> + C = new_ctx(Config), + Self = self(), + Crasher = spawn_link(fun() -> Self ! {crash, py_context:call(C, ?TEST_MOD, segfault, [])} end), + %% Sent right behind the segfault: postponed through {restarting, _} + spawn_link(fun() -> Self ! {next, py_context:eval(C, <<"2+2">>, #{}, 10000)} end), + receive + {crash, {error, {child_exited, {signal, Sig}}}} -> true = is_segfault_signal(Sig) + after 10000 -> ct:fail(no_crash_report) + end, + receive {next, {ok, 4}} -> ok after 10000 -> ct:fail(request_not_served_after_restart) end, + _ = Crasher, + stop(C). + +%% @doc kill/1 answers once the new child is up, so the next call cannot +%% race the restart. +test_kill_reply_after_restart(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:kill(C), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {idle, _} = sys:get_state(C), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc stop/1 from a third process while a call runs and while a loop +%% runs: nothing hangs, waiting callers get a reply or a DOWN. +test_stop_while_busy_and_looping(Config) -> + C1 = new_ctx(Config), + Self = self(), + spawn_link(fun() -> Self ! {busy, py_context:call(C1, ?TEST_MOD, sleep_then, [5, x], #{}, 10000)} end), + timer:sleep(100), + ok = py_context:stop(C1), + receive + {busy, {error, _}} -> ok + after 5000 -> ct:fail(busy_caller_hung) + end, + C2 = new_ctx(Config), + ok = py_context:start_loop(C2), + ok = py_context:stop(C2), + receive {py_loop_exit, C2, _} -> ok after 5000 -> ct:fail(no_loop_exit_on_stop) end, + false = is_process_alive(C2), + ok. + +test_sys_get_status(Config) -> + C = new_ctx(Config), + {status, C, {module, gen_statem}, _} = sys:get_status(C), + ok = sys:trace(C, true), + {ok, 4} = py_context:eval(C, <<"2+2">>), + ok = sys:trace(C, false), + stop(C). + +%% @doc The process that created the context may exit normally; the +%% context keeps serving (as embedded contexts do). +test_context_outlives_creator(Config) -> + Self = self(), + spawn(fun() -> Self ! {ctx, new_ctx(Config)} end), + C = receive {ctx, X} -> X after 15000 -> ct:fail(no_ctx) end, + timer:sleep(100), + true = is_process_alive(C), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_not_supported_fail_loud(Config) -> + C = new_ctx(Config), + {error, not_supported_in_isolated} = py_context:call_method(C, make_ref(), <<"x">>, []), + {error, {'RuntimeError', Msg}} = py_context:eval(C, <<"__import__('erlang').schedule('x')">>), + true = string:find(to_list(Msg), "isolated") =/= nomatch, + {error, {'RuntimeError', _}} = py_context:eval(C, <<"__import__('erlang').Channel()">>), + {error, not_supported_in_isolated} = py_context:loop_ref(C), + stop(C). + +test_bad_python_fails_at_start(_Config) -> + {error, {python_not_found, _}} = py_context:new(#{mode => isolated, python => "/no/such/python"}), + {error, {child_exited_at_start, _, _}} = py_context:new(#{mode => isolated, python => "/bin/sh"}), + ok. + +test_startup_error_reported(_Config) -> + case sanitized_child() of + true -> {skip, "sanitizer runtime in the child needs unbounded address space"}; + false -> test_startup_error_reported_1() + end. + +test_startup_error_reported_1() -> + %% An impossible rlimit is reported, not silently ignored. The kernel + %% may kill the child before it connects (child_exited_at_start), the + %% child may report the failed setrlimit (startup_error), or the macOS + %% watchdog may end it at once; all are loud. + case py_context:new(#{mode => isolated, rlimits => #{as => 1}}) of + {error, {startup_error, _}} -> ok; + {error, {child_exited_at_start, _, _}} -> ok; + {error, {handshake_failed, _}} -> ok; + {ok, C} -> py_context:stop(C), ct:fail(limit_ignored); + Other -> ct:fail({unexpected, Other}) + end. + +%% @doc cgroups exist only on Linux: elsewhere the option is refused before +%% a child is spawned, and rlimits remain the way to bound the child. +test_cgroup_option_platform(_Config) -> + case os:type() of + {unix, linux} -> + %% A non-writable path is reported by the child + {error, {startup_error, [{cgroup, _}]}} = + py_context:new(#{mode => isolated, cgroup => "/nonexistent/cgroup"}), + ok; + {unix, Os} -> + {error, {cgroup_unsupported, Os}} = + py_context:new(#{mode => isolated, cgroup => "/sys/fs/cgroup/x"}), + %% Limits still apply without cgroups + {ok, C} = py_context:new(#{mode => isolated, rlimits => #{nofile => 48, cpu => 5}}), + {ok, 48} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_NOFILE)[0]">>), + {ok, 5} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_CPU)[0]">>), + stop(C) + end. + +test_env_option(Config) -> + C = new_ctx(Config, #{env => #{"PY_ISOLATED_PROBE" => "yes"}}), + {ok, <<"yes">>} = py_context:eval(C, <<"__import__('os').environ.get('PY_ISOLATED_PROBE')">>), + stop(C). + +test_preload_option(Config) -> + C = new_ctx(Config, #{preload => <<"preloaded = 'yes'">>}), + {ok, <<"yes">>} = py_context:eval(C, <<"preloaded">>), + stop(C). + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + Opts = maps:merge(#{mode => Mode, paths => [TestDir]}, Extra), + {ok, C} = py_context:new(Opts), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +test_dir() -> + filename:join(code:lib_dir(erlang_python), "test"). + +os_pid_alive(Pid) -> + case py_nif:os_kill(Pid, 0) of + ok -> + %% Alive, or a zombie: a zombie shows as Z in ps + case string:trim(os:cmd("ps -o stat= -p " ++ integer_to_list(Pid))) of + "" -> false; + "Z" ++ _ -> zombie; + _ -> true + end; + {error, esrch} -> + false; + {error, eperm} -> + true + end. + +wait_gone(Pid) -> + wait_gone(Pid, 50). + +wait_gone(Pid, 0) -> + ct:fail({child_still_present, Pid, os_pid_alive(Pid)}); +wait_gone(Pid, N) -> + case os_pid_alive(Pid) of + false -> ok; + _ -> timer:sleep(100), wait_gone(Pid, N - 1) + end. + +%% A sanitizer runtime (LD_PRELOAD=libasan in the ASan job) is inherited +%% by the child: it aborts on a segfault and reserves terabytes of address +%% space, so rlimit cases cannot mean anything there. +sanitized_child() -> + Pre = case os:getenv("LD_PRELOAD") of false -> ""; P -> P end, + string:find(Pre, "asan") =/= nomatch orelse os:getenv("ASAN_OPTIONS") =/= false. + +is_segfault_signal(11) -> true; +is_segfault_signal(6) -> sanitized_child(); +is_segfault_signal(_) -> false. + +rlimit_as_enforced() -> + case os:type() of + {unix, linux} -> true; + {unix, freebsd} -> true; + _ -> false + end. + +peer_available() -> + code:ensure_loaded(peer) =:= {module, peer}. + +to_list(B) when is_binary(B) -> binary_to_list(B); +to_list(L) when is_list(L) -> L. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_isolated_async_SUITE.erl b/test/py_isolated_async_SUITE.erl new file mode 100644 index 0000000..02ad05c --- /dev/null +++ b/test/py_isolated_async_SUITE.erl @@ -0,0 +1,516 @@ +%%% @doc Common Test suite: asyncio in an isolated context. +%%% +%%% The child runs a plain asyncio loop. This suite mirrors +%%% py_worker_loop_SUITE (start_loop/submit/stop_loop, serving on fds Erlang +%%% owns) and the coroutine cases of py_async_task_SUITE. The headline case, +%%% a loop wedged in a blocking C call that stop_loop/2 kills, has a worker +%%% group counterpart documenting that the embedded loop cannot be stopped +%%% that way. +-module(py_isolated_async_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + end_per_testcase/2 +]). + +-export([ + test_start_stop_loop/1, + test_start_twice/1, + test_stop_idle/1, + test_submit_idle_and_running/1, + test_submit_ordering/1, + test_submit_errors_reported/1, + test_calls_rejected_while_running/1, + test_interrupt_ends_loop/1, + test_owner_death_stops_loop/1, + test_stop_context_while_looping/1, + test_long_submitted_call/1, + test_preload_before_loop/1, + test_tcp_serve_on_passed_fd/1, + test_udp_serve_on_passed_fd/1, + test_adopt_accepted_fd/1, + test_three_workers_one_listen_fd/1, + test_pass_fd_invalid/1, + test_call_awaits_coroutine/1, + test_gather_is_concurrent/1, + test_async_error/1, + test_concurrent_submitted_tasks/1, + test_large_async_result/1, + test_async_call_in_coroutine/1, + test_async_calls_concurrent/1, + test_async_call_error/1, + test_send_from_coroutine/1, + test_run_helper_compat/1, + test_stream_via_send/1, + test_blocked_loop_is_killed/1, + test_blocked_loop_survives_in_worker/1 +]). + +-define(TEST_MOD, py_test_isolated). +-define(HOST, {127, 0, 0, 1}). + +all() -> + [{group, isolated}, {group, worker_contrast}]. + +groups() -> + Cases = [ + test_start_stop_loop, + test_start_twice, + test_stop_idle, + test_submit_idle_and_running, + test_submit_ordering, + test_submit_errors_reported, + test_calls_rejected_while_running, + test_interrupt_ends_loop, + test_owner_death_stops_loop, + test_stop_context_while_looping, + test_long_submitted_call, + test_preload_before_loop, + test_tcp_serve_on_passed_fd, + test_udp_serve_on_passed_fd, + test_adopt_accepted_fd, + test_three_workers_one_listen_fd, + test_pass_fd_invalid, + test_call_awaits_coroutine, + test_gather_is_concurrent, + test_async_error, + test_concurrent_submitted_tasks, + test_large_async_result, + test_async_call_in_coroutine, + test_async_calls_concurrent, + test_async_call_error, + test_send_from_coroutine, + test_run_helper_compat, + test_stream_via_send, + test_blocked_loop_is_killed + ], + [{isolated, [], Cases}, + {worker_contrast, [], [test_blocked_loop_survives_in_worker]}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(worker_contrast, Config) -> + [{mode, worker} | Config]; +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Worker loop lifecycle +%%% ============================================================================ + +test_start_stop_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:stop_loop(C), + receive {py_loop_exit, C, ok} -> ok after 2000 -> ct:fail(no_loop_exit) end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_start_twice(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, already_running} = py_context:start_loop(C), + ok = py_context:stop_loop(C), + stop(C). + +test_stop_idle(Config) -> + C = new_ctx(Config), + {error, no_loop} = py_context:stop_loop(C), + stop(C). + +test_submit_idle_and_running(Config) -> + C = new_ctx(Config), + %% Without a running loop, submit reports no loop (the embedded modes + %% step the loop through the event worker; the child has no such thing) + {error, no_loop} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + {ok, 7} = py_context:submit_await(C, ?TEST_MOD, add, [3, 4]), + ok = py_context:stop_loop(C), + stop(C). + +test_submit_ordering(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + Refs = [begin + {ok, R} = py_context:submit(C, ?TEST_MOD, async_add, [I, 0]), + {I, R} + end || I <- lists:seq(1, 100)], + lists:foreach(fun({I, R}) -> + {ok, I} = py_event_loop:await(R, 5000) + end, Refs), + ok = py_context:stop_loop(C), + stop(C). + +test_submit_errors_reported(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, {'ModuleNotFoundError', _}} = py_context:submit_await(C, no_such_mod_xyz, f, []), + {error, {'AttributeError', _}} = py_context:submit_await(C, ?TEST_MOD, no_such_fn, []), + {error, {'KeyError', _}} = py_context:submit_await(C, ?TEST_MOD, async_raise, [<<"k">>]), + {error, {'ValueError', _}} = py_context:submit_await(C, ?TEST_MOD, raise_value_error, [<<"v">>]), + ok = py_context:stop_loop(C), + stop(C). + +test_calls_rejected_while_running(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, loop_running} = py_context:eval(C, <<"1">>), + {error, loop_running} = py_context:exec(C, <<"x = 1">>), + {error, loop_running} = py_context:call(C, math, sqrt, [4]), + ok = py_context:stop_loop(C), + {ok, 1} = py_context:eval(C, <<"1">>), + stop(C). + +test_interrupt_ends_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + ok = py_context:interrupt(C), + receive {py_loop_exit, C, {error, interrupted}} -> ok + after 3000 -> ct:fail(loop_not_interrupted) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_owner_death_stops_loop(Config) -> + C = new_ctx(Config), + Owner = spawn(fun() -> receive die -> ok end end), + ok = py_context:start_loop(C, #{owner => Owner}), + Owner ! die, + wait_until(fun() -> py_context:eval(C, <<"1">>) =:= {ok, 1} end, 5000), + stop(C). + +test_stop_context_while_looping(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:start_loop(C), + ok = py_context:stop(C), + wait_until(fun() -> py_nif:os_kill(Pid, 0) =:= {error, esrch} end, 5000), + ok. + +%% @doc No 30 s cap on a submitted call (the pipe write deadline of the +%% embedded modes does not apply): a 2 s task completes, and the loop keeps +%% answering meanwhile. +test_long_submitted_call(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, R} = py_context:submit(C, ?TEST_MOD, slow_task, [2]), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + {ok, <<"slow_done">>} = py_event_loop:await(R, 10000), + ok = py_context:stop_loop(C), + stop(C). + +test_preload_before_loop(Config) -> + C = new_ctx(Config, #{preload => <<"import py_test_isolated\npy_test_isolated.counter_increment(5)">>}), + ok = py_context:start_loop(C), + {ok, 5} = py_context:submit_await(C, ?TEST_MOD, counter_value, []), + ok = py_context:stop_loop(C), + stop(C). + +%%% ============================================================================ +%%% Serving on fds Erlang owns +%%% ============================================================================ + +test_tcp_serve_on_passed_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, ChildFd} = listen_pass(C), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [ChildFd]), + [<<"ok:x">> = roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 100)], + {ok, 100} = py_context:submit_await(C, py_test_workerloop, served_count, []), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [ChildFd]), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop(C). + +test_udp_serve_on_passed_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, USock} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(USock), + {ok, Fd} = inet:getfd(USock), + {ok, ChildFd} = py_context:pass_fd(C, Fd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_udp, [ChildFd]), + {ok, Client} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + ok = gen_udp:send(Client, ?HOST, Port, <<"ping">>), + {ok, {_, _, <<"udp:ping">>}} = gen_udp:recv(Client, 0, 2000), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [ChildFd]), + gen_udp:close(Client), + gen_udp:close(USock), + ok = py_context:stop_loop(C), + stop(C). + +test_adopt_accepted_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(LSock), + Self = self(), + spawn_link(fun() -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(S, <<"adopted?">>), + Self ! {client, gen_tcp:recv(S, 0, 3000)}, + gen_tcp:close(S) + end), + {ok, Conn} = gen_tcp:accept(LSock, 2000), + {ok, ConnFd} = inet:getfd(Conn), + {ok, ChildFd} = py_context:pass_fd(C, ConnFd), + {ok, <<"adopted">>} = py_context:submit_await(C, py_test_workerloop, adopt, [ChildFd]), + gen_tcp:close(Conn), + receive {client, {ok, <<"ok:adopted?">>}} -> ok + after 3000 -> ct:fail(no_reply_through_adopted_fd) + end, + gen_tcp:close(LSock), + ok = py_context:stop_loop(C), + stop(C). + +%% @doc gunicorn shape, out of process: one listen socket, three child +%% processes accepting on their copy of it. +test_three_workers_one_listen_fd(Config) -> + Ctxs = [new_ctx(Config) || _ <- lists:seq(1, 3)], + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 512}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + lists:foreach(fun({I, C}) -> + ok = py_context:start_loop(C), + {ok, ChildFd} = py_context:pass_fd(C, LFd), + Tag = list_to_binary("w" ++ integer_to_list(I) ++ ":"), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [ChildFd, Tag]) + end, lists:zip(lists:seq(1, 3), Ctxs)), + Replies = [roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 300)], + 300 = length([R || R <- Replies, binary:part(R, byte_size(R) - 4, 4) =:= <<"ok:x">>]), + Tags = lists:usort([binary:part(R, 0, 3) || R <- Replies]), + ct:log("workers that served: ~p", [Tags]), + %% Which child wins accept() is up to the kernel; a fast worker can + %% starve another over 300 connections. Two distinct workers prove + %% the socket is shared. + true = length(Tags) >= 2, + [ok = py_context:stop_loop(C) || C <- Ctxs], + [stop(C) || C <- Ctxs], + gen_tcp:close(LSock), + ok. + +test_pass_fd_invalid(Config) -> + C = new_ctx(Config), + {error, _} = py_context:pass_fd(C, 123456), + {error, {invalid_fd, -1}} = py_context:pass_fd(C, -1), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%%% ============================================================================ +%%% Coroutines +%%% ============================================================================ + +test_call_awaits_coroutine(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:exec(C, <<"import py_test_isolated">>), + {ok, 3} = py_context:eval(C, <<"py_test_isolated.async_add(a, b)">>, #{a => 1, b => 2}), + stop(C). + +test_gather_is_concurrent(Config) -> + C = new_ctx(Config), + T0 = erlang:monotonic_time(millisecond), + {ok, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} = py_context:call(C, ?TEST_MOD, async_sleep_gather, [10, 0.1]), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("10 x sleep(0.1) gathered in ~p ms", [Elapsed]), + true = Elapsed < 600, + stop(C). + +test_async_error(Config) -> + C = new_ctx(Config), + {error, {'KeyError', _}} = py_context:call(C, ?TEST_MOD, async_raise, [<<"nope">>]), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_concurrent_submitted_tasks(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + Refs = [begin {ok, R} = py_context:submit(C, ?TEST_MOD, task_value, [I]), {I, R} end + || I <- lists:seq(1, 100)], + lists:foreach(fun({I, R}) -> + Expected = I * I, + {ok, Expected} = py_event_loop:await(R, 10000) + end, Refs), + ok = py_context:stop_loop(C), + stop(C). + +test_large_async_result(Config) -> + C = new_ctx(Config), + Size = 16 * 1024 * 1024, + {ok, Bin} = py_context:call(C, ?TEST_MOD, async_big, [Size]), + Size = byte_size(Bin), + stop(C). + +test_async_call_in_coroutine(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_double">>, fun([X]) -> X * 2 end), + {ok, 84} = py_context:call(C, ?TEST_MOD, async_erlang_call, [<<"as_double">>, 42]), + py_callback:unregister(<<"as_double">>), + stop(C). + +test_async_calls_concurrent(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_double">>, fun([X]) -> X * 2 end), + Expected = [I * 2 || I <- lists:seq(0, 99)], + {ok, Expected} = py_context:call(C, ?TEST_MOD, async_erlang_calls, [<<"as_double">>, 100]), + py_callback:unregister(<<"as_double">>), + stop(C). + +test_async_call_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_fail">>, fun(_) -> error(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, async_erlang_call_error, [<<"as_fail">>]), + py_callback:unregister(<<"as_fail">>), + stop(C). + +test_send_from_coroutine(Config) -> + C = new_ctx(Config), + {ok, <<"sent">>} = py_context:call(C, ?TEST_MOD, async_send, [self(), coro_msg]), + receive <<"coro_msg">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_run_helper_compat(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, run_helper_compat, []), + stop(C). + +%% @doc Streaming out of an isolated context: a submitted coroutine pushes +%% items with erlang.send; order and the done marker are asserted. +test_stream_via_send(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, R} = py_context:submit(C, ?TEST_MOD, stream_to, [self(), 1000]), + Items = collect_items([]), + Expected = lists:seq(0, 999), + Expected = Items, + {ok, 1000} = py_event_loop:await(R, 5000), + ok = py_context:stop_loop(C), + stop(C). + +%%% ============================================================================ +%%% The headline async case +%%% ============================================================================ + +%% @doc A coroutine wedged in time.sleep inside the loop: stop_loop/2 asks, +%% interrupts, then kills. The context is usable right after. +test_blocked_loop_is_killed(Config) -> + C = new_ctx(Config, #{kill_after => 500}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:start_loop(C), + %% The interrupt signal reaches time.sleep; block it so only the + %% backstop can end the loop + {ok, _} = py_context:submit(C, ?TEST_MOD, blocked_sleep, [60]), + timer:sleep(300), + T0 = erlang:monotonic_time(millisecond), + Result = py_context:stop_loop(C, 300), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("stop_loop on a wedged loop: ~p after ~p ms", [Result, Elapsed]), + ok = Result, + true = Elapsed < 5000, + receive {py_loop_exit, C, _} -> ok after 2000 -> ct:fail(no_loop_exit) end, + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc Contrast: the embedded loop cannot be killed; a wedged loop makes +%% stop_loop/2 time out and the sleep runs to completion. +test_blocked_loop_survives_in_worker(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, _} = py_context:submit(C, py_test_workerloop, block_loop, [4]), + timer:sleep(300), + Result = py_context:stop_loop(C, 300), + ct:log("worker stop_loop on a wedged loop: ~p", [Result]), + {error, timeout} = Result, + %% Wait out the sleep so the context can be stopped cleanly + timer:sleep(4500), + _ = (try py_context:stop(C) catch _:_ -> ok end), + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +listen_pass(C) -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 128}]), + {ok, Port} = inet:port(LSock), + {ok, Fd} = inet:getfd(LSock), + {ok, ChildFd} = py_context:pass_fd(C, Fd), + {LSock, Port, ChildFd}. + +roundtrip(Port, Data) -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(S, Data), + {ok, Reply} = gen_tcp:recv(S, 0, 3000), + gen_tcp:close(S), + Reply. + +collect_items(Acc) -> + receive + {<<"item">>, I} -> collect_items([I | Acc]); + <<"done">> -> lists:reverse(Acc) + after 5000 -> + ct:fail({incomplete, length(Acc)}) + end. + +wait_until(Fun, TimeoutMs) -> + Deadline = erlang:monotonic_time(millisecond) + TimeoutMs, + wait_until_loop(Fun, Deadline). + +wait_until_loop(Fun, Deadline) -> + case Fun() of + true -> ok; + _ -> + case erlang:monotonic_time(millisecond) > Deadline of + true -> ct:fail(condition_not_met); + false -> timer:sleep(50), wait_until_loop(Fun, Deadline) + end + end. + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + Opts = maps:merge(#{mode => Mode, paths => [TestDir]}, Extra), + {ok, C} = py_context:new(Opts), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_isolated_soak_SUITE.erl b/test/py_isolated_soak_SUITE.erl new file mode 100644 index 0000000..76fab9d --- /dev/null +++ b/test/py_isolated_soak_SUITE.erl @@ -0,0 +1,277 @@ +%%% @doc Soak test for `isolated' mode: a random mix of everything the mode +%%% offers, run for a while, with resource counters checked before and +%%% after. The point is to show no deadlock (every operation returns), no +%%% runaway loop (the mix keeps making progress) and no leak (Erlang +%%% processes, ports, ETS entries, memory, VM file descriptors and OS +%%% children return to their baseline). +%%% +%%% Duration is 60 s by default; set `PY_ISOLATED_SOAK_SECONDS' to change it. +-module(py_isolated_soak_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([ + test_mixed_workload_no_leak/1, + test_callback_storm_no_deadlock/1, + test_interrupt_kill_storm/1, + test_loop_start_stop_churn/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> [ + test_callback_storm_no_deadlock, + test_interrupt_kill_storm, + test_loop_start_stop_churn, + test_mixed_workload_no_leak +]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + py_callback:register(<<"soak_echo">>, fun([X]) -> X end), + py_callback:register(<<"soak_incr">>, fun([X]) -> X + 1 end), + py_callback:register(<<"soak_fail">>, fun(_) -> error(deliberate) end), + py_callback:register(<<"soak_tid">>, fun([T, I]) -> T * 1000 + I end), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + py_callback:unregister(<<"soak_echo">>), + py_callback:unregister(<<"soak_incr">>), + py_callback:unregister(<<"soak_fail">>), + py_callback:unregister(<<"soak_tid">>), + ok = application:stop(erlang_python), + ok. + +%% @doc Many Erlang callers, each mixing plain calls, callbacks, nested +%% callbacks, thread-pool callbacks and callback errors on a shared context. +%% Every call must return within its timeout. +test_callback_storm_no_deadlock(Config) -> + C = new_ctx(Config), + py_callback:register(<<"soak_nested">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, add, [X, 1], #{}, 20000), R + end), + Self = self(), + Workers = 16, + Rounds = 40, + [spawn_link(fun() -> Self ! {done, I, storm(C, I, Rounds, [])} end) || I <- lists:seq(1, Workers)], + Failures = lists:append([receive {done, _, F} -> F after 120000 -> ct:fail(worker_hung) end + || _ <- lists:seq(1, Workers)]), + ct:log("failures: ~p", [Failures]), + [] = Failures, + py_callback:unregister(<<"soak_nested">>), + ok = py_context:stop(C), + ok. + +storm(_C, _I, 0, Acc) -> + Acc; +storm(C, I, N, Acc) -> + Op = (I + N) rem 6, + R = case Op of + 0 -> py_context:call(C, ?TEST_MOD, add, [I, N], #{}, 20000); + 1 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_echo">>, {I, N}], #{}, 20000); + 2 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_nested">>, N], #{}, 20000); + 3 -> py_context:call(C, ?TEST_MOD, pool_calls, [<<"soak_incr">>, 4, 20], #{}, 20000); + 4 -> py_context:call(C, ?TEST_MOD, callback_error_type, [<<"soak_fail">>], #{}, 20000); + 5 -> py_context:call(C, ?TEST_MOD, thread_calls, [<<"soak_tid">>, 4, 5], #{}, 20000) + end, + Expected = case Op of + 0 -> {ok, I + N}; + 1 -> {ok, {I, N}}; + 2 -> {ok, N + 1}; + 3 -> {ok, false}; %% incr, not double: the helper compares to i*2 + 4 -> {ok, <<"RuntimeError">>}; + 5 -> {ok, {<<"ok">>, true, 20}} + end, + Acc1 = case R of + Expected -> Acc; + {ok, _} when Op =:= 3 -> Acc; %% value checked by shape only + Other -> [{I, N, Op, Other} | Acc] + end, + storm(C, I, N - 1, Acc1). + +%% @doc Interrupts and kills racing with calls: nothing hangs, the context +%% always answers again, and no child is left behind. +test_interrupt_kill_storm(Config) -> + C = new_ctx(Config, #{kill_after => 200, max_restarts => 1000}), + Pids = lists:map(fun(N) -> + {ok, #{os_pid := P}} = py_context:child_info(C), + Self = self(), + spawn_link(fun() -> + Self ! {res, py_context:call(C, ?TEST_MOD, sleep_then, [5, N], #{}, 30000)} + end), + timer:sleep(20 + N rem 30), + case N rem 3 of + 0 -> py_context:interrupt(C); + 1 -> py_context:kill(C); + 2 -> py_context:interrupt(C), py_context:kill(C) + end, + receive {res, R} -> + case R of + {error, interrupted} -> ok; + {error, killed} -> ok; + {error, {child_exited, _}} -> ok; + {ok, N} -> ok; + Other -> ct:fail({unexpected, N, Other}) + end + after 15000 -> ct:fail({hung_after_interrupt, N}) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 30000), + P + end, lists:seq(1, 30)), + ok = py_context:stop(C), + timer:sleep(300), + Alive = [P || P <- lists:usort(Pids), py_nif:os_kill(P, 0) =:= ok], + [] = Alive, + ok. + +%% @doc start_loop / submit / stop_loop repeated, with a wedged loop every +%% few rounds so the kill backstop runs. +test_loop_start_stop_churn(Config) -> + C = new_ctx(Config, #{kill_after => 200, max_restarts => 1000}), + lists:foreach(fun(N) -> + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2], #{}, 30000), + case N rem 4 of + 0 -> + {ok, _} = py_context:submit(C, ?TEST_MOD, blocked_sleep, [30]), + timer:sleep(50), + ok = py_context:stop_loop(C, 100); + _ -> + ok = py_context:stop_loop(C, 2000) + end, + receive {py_loop_exit, C, _} -> ok after 5000 -> ct:fail({no_loop_exit, N}) end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 30000) + end, lists:seq(1, 24)), + ok = py_context:stop(C), + ok. + +%% @doc The long one: contexts started and stopped, calls with payloads, +%% callbacks, coroutines, interrupts, crashes, for a fixed duration. +%% Counters must return to baseline. +test_mixed_workload_no_leak(Config) -> + Seconds = list_to_integer(os:getenv("PY_ISOLATED_SOAK_SECONDS", "60")), + %% Warm up so lazily created resources are in the baseline + Warm = new_ctx(Config), + {ok, _} = py_context:call(Warm, ?TEST_MOD, callback, [<<"soak_echo">>, 1]), + ok = py_context:stop(Warm), + timer:sleep(500), + erlang:garbage_collect(), + Base = counters(), + ct:log("baseline: ~p", [Base]), + Deadline = erlang:monotonic_time(millisecond) + Seconds * 1000, + Self = self(), + Workers = [spawn_link(fun() -> Self ! {worker, I, mixed(Config, I, Deadline, 0, [])} end) + || I <- lists:seq(1, 6)], + Stats = [receive {worker, _, S} -> S after (Seconds + 120) * 1000 -> ct:fail(worker_hung) end + || _ <- Workers], + Ops = lists:sum([O || {O, _} <- Stats]), + Errs = lists:append([E || {_, E} <- Stats]), + ct:log("ops: ~p, unexpected errors: ~p", [Ops, lists:sublist(Errs, 20)]), + ct:print("soak: ~p ops in ~p s, ~p unexpected errors", [Ops, Seconds, length(Errs)]), + true = Ops > 0, + [] = Errs, + timer:sleep(1000), + erlang:garbage_collect(), + After = counters(), + ct:log("after: ~p", [After]), + check_no_growth(Base, After), + ok. + +mixed(Config, I, Deadline, Ops, Errs) -> + case erlang:monotonic_time(millisecond) > Deadline of + true -> {Ops, Errs}; + false -> + C = new_ctx(Config, #{kill_after => 300, max_restarts => 1000}), + E1 = mixed_ops(C, I, 25, Errs), + ok = py_context:stop(C), + mixed(Config, I, Deadline, Ops + 25, E1) + end. + +mixed_ops(_C, _I, 0, Errs) -> + Errs; +mixed_ops(C, I, N, Errs) -> + Op = (I * 7 + N) rem 9, + R = case Op of + 0 -> py_context:eval(C, <<"sum(range(1000))">>, #{}, 30000); + 1 -> py_context:call(C, ?TEST_MOD, identity, [crypto:strong_rand_bytes(256 * 1024)], #{}, 30000); + 2 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_echo">>, [I, N]], #{}, 30000); + 3 -> py_context:call(C, ?TEST_MOD, async_sleep_gather, [5, 0.001], #{}, 30000); + 4 -> py_context:eval(C, <<"__import__('time').sleep(5)">>, #{}, 50); + 5 -> py_context:call(C, ?TEST_MOD, pool_calls, [<<"soak_echo">>, 4, 10], #{}, 30000); + 6 -> py_context:call(C, ?TEST_MOD, segfault, [], #{}, 30000); + 7 -> py_context:call(C, ?TEST_MOD, send, [self(), {soak, N}], #{}, 30000); + 8 -> py_context:kill(C) + end, + Ok = case {Op, R} of + {0, {ok, 499500}} -> true; + {1, {ok, B}} when is_binary(B) -> true; + {2, {ok, [I, N]}} -> true; + {3, {ok, [0, 1, 2, 3, 4]}} -> true; + {4, {error, timeout}} -> true; + {5, {ok, _}} -> true; + {6, {error, {child_exited, {signal, _}}}} -> true; + {7, {ok, true}} -> receive {<<"soak">>, N} -> true after 5000 -> false end; + {8, ok} -> true; + _ -> false + end, + %% After any op the context must answer + Alive = py_context:eval(C, <<"1">>, #{}, 30000) =:= {ok, 1}, + Errs1 = case Ok andalso Alive of + true -> Errs; + false -> [{op, Op, R, alive, Alive} | Errs] + end, + mixed_ops(C, I, N - 1, Errs1). + +%%% ============================================================================ +%%% Counters +%%% ============================================================================ + +counters() -> + #{ + processes => erlang:system_info(process_count), + ports => erlang:system_info(port_count), + refs => ets:info(py_context_refs, size), + memory_mb => erlang:memory(total) div (1024 * 1024), + binary_mb => erlang:memory(binary) div (1024 * 1024), + fds => beam_fd_count(), + children => child_count() + }. + +check_no_growth(Base, After) -> + Same = [processes, ports, refs, children], + lists:foreach(fun(K) -> + B = maps:get(K, Base), A = maps:get(K, After), + A =< B + 2 orelse ct:fail({leak, K, B, A}) + end, Same), + %% fds: allow a few for CT's own logging + maps:get(fds, After) =< maps:get(fds, Base) + 8 orelse + ct:fail({fd_leak, maps:get(fds, Base), maps:get(fds, After)}), + %% memory: within 64 MB of baseline after GC + maps:get(memory_mb, After) =< maps:get(memory_mb, Base) + 64 orelse + ct:fail({memory_growth, maps:get(memory_mb, Base), maps:get(memory_mb, After)}), + ok. + +beam_fd_count() -> + case os:type() of + {unix, linux} -> + length(filelib:wildcard("/proc/" ++ os:getpid() ++ "/fd/*")); + _ -> + Out = os:cmd("lsof -p " ++ os:getpid() ++ " 2>/dev/null | wc -l"), + list_to_integer(string:trim(Out)) - 1 + end. + +child_count() -> + Out = os:cmd("ps -ax -o ppid=,command= 2>/dev/null | grep py_isolated_child | grep -v grep | grep -c ' " ++ os:getpid() ++ " ' "), + case string:trim(Out) of + "" -> 0; + N -> list_to_integer(N) + end. + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(maps:merge(#{mode => isolated, paths => [TestDir]}, Extra)), + C. diff --git a/test/py_isolated_stress_SUITE.erl b/test/py_isolated_stress_SUITE.erl new file mode 100644 index 0000000..92fbc2d --- /dev/null +++ b/test/py_isolated_stress_SUITE.erl @@ -0,0 +1,162 @@ +%%% @doc Stress and profiling for `isolated' mode. +%%% +%%% Numbers are logged, not asserted tightly: the point is to show the cost +%%% of the process boundary next to worker mode on the same machine, and +%%% that churn does not leak OS processes or memory. +-module(py_isolated_stress_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1]). + +-export([ + test_call_latency_vs_worker/1, + test_callback_round_trips/1, + test_context_churn_no_leak/1, + test_startup_time/1, + test_payload_throughput/1, + test_parallel_contexts_cpu_bound/1 +]). + +all() -> [ + test_call_latency_vs_worker, + test_callback_round_trips, + test_context_churn_no_leak, + test_startup_time, + test_payload_throughput, + test_parallel_contexts_cpu_bound +]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + Config. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +%% @doc 10k sequential evals per mode; p50/p99 per call. +test_call_latency_vs_worker(_Config) -> + N = 10000, + {ok, I} = py_context:new(#{mode => isolated}), + {ok, W} = py_context:new(#{mode => worker}), + {ok, 2} = py_context:eval(I, <<"1+1">>), + {ok, 2} = py_context:eval(W, <<"1+1">>), + IsoLat = latencies(fun() -> {ok, 2} = py_context:eval(I, <<"1+1">>) end, N), + WrkLat = latencies(fun() -> {ok, 2} = py_context:eval(W, <<"1+1">>) end, N), + IsoCall = latencies(fun() -> {ok, 4.0} = py_context:call(I, math, sqrt, [16]) end, N), + WrkCall = latencies(fun() -> {ok, 4.0} = py_context:call(W, math, sqrt, [16]) end, N), + ct:log("eval isolated: ~s~n worker: ~s", [stats(IsoLat), stats(WrkLat)]), + ct:log("call isolated: ~s~n worker: ~s", [stats(IsoCall), stats(WrkCall)]), + ct:print("eval p50 isolated ~p us vs worker ~p us", [pct(IsoLat, 50), pct(WrkLat, 50)]), + py_context:stop(I), + py_context:stop(W), + ok. + +test_callback_round_trips(_Config) -> + {ok, C} = py_context:new(#{mode => isolated}), + py_callback:register(<<"stress_echo">>, fun([X]) -> X end), + Code = <<"__import__('erlang').call('stress_echo', 1)">>, + Lat = latencies(fun() -> {ok, 1} = py_context:eval(C, Code) end, 1000), + ct:log("eval+callback isolated: ~s", [stats(Lat)]), + %% 1000 callbacks inside one request + ok = py_context:exec(C, <<"import erlang\ndef burst(n):\n return sum(erlang.call('stress_echo', i) for i in range(n))\n">>), + T0 = erlang:monotonic_time(microsecond), + {ok, 499500} = py_context:call(C, '__main__', burst, [1000]), + Per = (erlang:monotonic_time(microsecond) - T0) / 1000, + ct:log("1000 callbacks in one request: ~.1f us each", [Per]), + py_callback:unregister(<<"stress_echo">>), + py_context:stop(C), + ok. + +%% @doc 100 contexts started and stopped: no child left, RSS reported. +test_context_churn_no_leak(_Config) -> + Pids = lists:map(fun(_) -> + {ok, C} = py_context:new(#{mode => isolated}), + {ok, #{os_pid := P}} = py_context:child_info(C), + {ok, 2} = py_context:eval(C, <<"1+1">>), + ok = py_context:stop(C), + P + end, lists:seq(1, 100)), + timer:sleep(500), + Alive = [P || P <- Pids, py_nif:os_kill(P, 0) =:= ok], + ct:log("children still alive after churn: ~p", [Alive]), + [] = Alive, + %% Memory per child + {ok, C} = py_context:new(#{mode => isolated}), + {ok, #{os_pid := P}} = py_context:child_info(C), + Rss = string:trim(os:cmd("ps -o rss= -p " ++ integer_to_list(P))), + ct:log("child RSS after start: ~s KB", [Rss]), + ct:print("child RSS: ~s KB", [Rss]), + {ok, _} = py_context:eval(C, <<"__import__('json').dumps([1]*1000)">>), + Rss2 = string:trim(os:cmd("ps -o rss= -p " ++ integer_to_list(P))), + ct:log("child RSS after json import: ~s KB", [Rss2]), + py_context:stop(C), + ok. + +test_startup_time(_Config) -> + Times = lists:map(fun(_) -> + T0 = erlang:monotonic_time(microsecond), + {ok, C} = py_context:new(#{mode => isolated}), + T = erlang:monotonic_time(microsecond) - T0, + ok = py_context:stop(C), + T + end, lists:seq(1, 20)), + ct:log("isolated context start (spawn -> ready -> init): ~s", [stats(Times)]), + ct:print("startup p50 ~p ms", [pct(Times, 50) div 1000]), + ok. + +test_payload_throughput(_Config) -> + {ok, I} = py_context:new(#{mode => isolated}), + {ok, W} = py_context:new(#{mode => worker}), + ok = py_context:exec(I, <<"def ident(x): return x">>), + ok = py_context:exec(W, <<"def ident(x): return x">>), + lists:foreach(fun(Size) -> + Bin = crypto:strong_rand_bytes(Size), + TI = timed(fun() -> {ok, Bin} = py_context:call(I, '__main__', ident, [Bin]) end), + TW = timed(fun() -> {ok, Bin} = py_context:call(W, '__main__', ident, [Bin]) end), + ct:log("~p MB round trip: isolated ~.1f ms (~.1f MB/s), worker ~.1f ms", + [Size div (1024 * 1024), TI / 1000, 2 * Size / 1048576 / (TI / 1.0e6), TW / 1000]) + end, [1024 * 1024, 16 * 1024 * 1024, 64 * 1024 * 1024]), + py_context:stop(I), + py_context:stop(W), + ok. + +%% @doc Four isolated children run CPU-bound work in parallel: the GIL is +%% per process, so wall time is close to one child's time. +test_parallel_contexts_cpu_bound(_Config) -> + Ctxs = [begin {ok, C} = py_context:new(#{mode => isolated}), C end || _ <- lists:seq(1, 4)], + Code = <<"sum(i*i for i in range(2000000))">>, + T1 = timed(fun() -> {ok, _} = py_context:eval(hd(Ctxs), Code) end), + Self = self(), + T4 = timed(fun() -> + [spawn_link(fun() -> Self ! {done, py_context:eval(C, Code)} end) || C <- Ctxs], + [receive {done, {ok, _}} -> ok after 60000 -> ct:fail(timeout) end || _ <- Ctxs] + end), + ct:log("cpu-bound: 1 child ~.1f ms, 4 children in parallel ~.1f ms", [T1 / 1000, T4 / 1000]), + %% On a dedicated machine T4 is close to T1 (see the log). CI VMs are + %% overcommitted, so only assert the children did not serialise. + true = T4 < 4 * T1, + [py_context:stop(C) || C <- Ctxs], + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +latencies(Fun, N) -> + lists:sort([timed(Fun) || _ <- lists:seq(1, N)]). + +timed(Fun) -> + T0 = erlang:monotonic_time(microsecond), + Fun(), + erlang:monotonic_time(microsecond) - T0. + +pct(Sorted, P) -> + Idx = max(1, min(length(Sorted), round(length(Sorted) * P / 100))), + lists:nth(Idx, Sorted). + +stats(Sorted) -> + Mean = lists:sum(Sorted) / length(Sorted), + io_lib:format("p50 ~p us, p99 ~p us, max ~p us, mean ~.1f us", + [pct(Sorted, 50), pct(Sorted, 99), lists:last(Sorted), Mean]). diff --git a/test/py_isolated_vm_SUITE.erl b/test/py_isolated_vm_SUITE.erl new file mode 100644 index 0000000..7572403 --- /dev/null +++ b/test/py_isolated_vm_SUITE.erl @@ -0,0 +1,477 @@ +%%% @doc Common Test suite: an isolated context as a participant in the VM. +%%% +%%% Mirrors, case for case, what py_pid_send_SUITE, py_callback_encoding_SUITE, +%%% py_thread_callback_SUITE and py_actor_SUITE prove for the embedded modes: +%%% pids, erlang.send, whereis, callback result encoding, Python threads +%%% calling Erlang, and actor-style state. Every case runs in a worker group +%%% too, so a divergence between modes fails as a pair. +-module(py_isolated_vm_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + end_per_testcase/2 +]). + +-export([ + test_pid_is_pid/1, + test_pid_equality_and_hash/1, + test_pid_in_structure/1, + test_send_simple/1, + test_send_multiple_ordered/1, + test_send_complex_term/1, + test_send_is_nonblocking/1, + test_send_to_dead_process/1, + test_send_bad_pid/1, + test_send_from_coroutine/1, + test_whereis/1, + test_suspension_is_base_exception/1, + test_callback_inside_except_exception/1, + test_encoding_binary_with_escapes/1, + test_encoding_binary_non_utf8/1, + test_encoding_large_binary/1, + test_encoding_atom_becomes_str/1, + test_encoding_empty_list/1, + test_encoding_erlang_string/1, + test_encoding_nested_containers/1, + test_encoding_pid_and_ref/1, + test_encoding_floats/1, + test_encoding_booleans_and_none/1, + test_encoding_python_types/1, + test_threads_call_erlang/1, + test_threadpool_calls/1, + test_threadpool_error/1, + test_threadpool_nested/1, + test_threads_high_concurrency/1, + test_counter_actor/1, + test_state_reset_on_restart/1, + test_state_isolated_between_contexts/1, + test_ping_pong/1, + test_feed_through_callback/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> + [{group, worker}, {group, isolated}]. + +groups() -> + Cases = [ + test_pid_is_pid, + test_pid_equality_and_hash, + test_pid_in_structure, + test_send_simple, + test_send_multiple_ordered, + test_send_complex_term, + test_send_is_nonblocking, + test_send_to_dead_process, + test_send_bad_pid, + test_send_from_coroutine, + test_whereis, + test_suspension_is_base_exception, + test_callback_inside_except_exception, + test_encoding_binary_with_escapes, + test_encoding_binary_non_utf8, + test_encoding_large_binary, + test_encoding_atom_becomes_str, + test_encoding_empty_list, + test_encoding_erlang_string, + test_encoding_nested_containers, + test_encoding_pid_and_ref, + test_encoding_floats, + test_encoding_booleans_and_none, + test_encoding_python_types, + test_threads_call_erlang, + test_threadpool_calls, + test_threadpool_error, + test_threadpool_nested, + test_threads_high_concurrency, + test_counter_actor, + test_state_reset_on_restart, + test_state_isolated_between_contexts, + test_ping_pong, + test_feed_through_callback + ], + [{worker, [], Cases}, {isolated, [], Cases}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Pids, send, whereis +%%% ============================================================================ + +test_pid_is_pid(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, is_pid, [self()]), + {ok, <<"Pid">>} = py_context:call(C, ?TEST_MOD, type_name, [self()]), + Self = self(), + {ok, Self} = py_context:call(C, ?TEST_MOD, identity, [Self]), + stop(C). + +test_pid_equality_and_hash(Config) -> + C = new_ctx(Config), + Self = self(), + Other = spawn(fun() -> receive stop -> ok end end), + {ok, true} = py_context:call(C, ?TEST_MOD, pid_equal, [Self, Self]), + {ok, false} = py_context:call(C, ?TEST_MOD, pid_equal, [Self, Other]), + {ok, true} = py_context:call(C, ?TEST_MOD, pid_hash_equal, [Self, Self]), + Other ! stop, + stop(C). + +test_pid_in_structure(Config) -> + C = new_ctx(Config), + Self = self(), + {ok, #{<<"owner">> := Self, <<"list">> := [Self, {Self, 1}]}} = + py_context:call(C, ?TEST_MOD, pid_in_structure, [Self]), + stop(C). + +test_send_simple(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), <<"hello">>]), + receive <<"hello">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_send_multiple_ordered(Config) -> + C = new_ctx(Config), + N = 500, + {ok, N} = py_context:call(C, ?TEST_MOD, send_many, [self(), N]), + Items = collect_items([]), + Expected = lists:seq(0, N - 1), + Expected = Items, + stop(C). + +test_send_complex_term(Config) -> + C = new_ctx(Config), + Term = {hello, 42, [1, 2, 3], #{<<"key">> => <<"value">>}, true, none, 1.5}, + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), Term]), + receive + {<<"hello">>, 42, [1, 2, 3], #{<<"key">> := <<"value">>}, true, none, 1.5} -> ok + after 2000 -> + ct:fail(no_message) + end, + stop(C). + +test_send_is_nonblocking(Config) -> + C = new_ctx(Config), + Sink = spawn(fun() -> receive stop -> ok end end), + {ok, Ms} = py_context:call(C, ?TEST_MOD, send_timing, [Sink, 1000]), + ct:log("1000 erlang.send took ~.1f ms (~.1f us each)", [Ms, Ms]), + true = Ms < 5000, + Sink ! stop, + stop(C). + +test_send_to_dead_process(Config) -> + C = new_ctx(Config), + Dead = spawn(fun() -> ok end), + timer:sleep(50), + false = is_process_alive(Dead), + {ok, <<"process_error">>} = py_context:call(C, ?TEST_MOD, send_to_dead, [Dead]), + stop(C). + +test_send_bad_pid(Config) -> + C = new_ctx(Config), + {ok, <<"type_error">>} = py_context:call(C, ?TEST_MOD, send_bad_pid, []), + stop(C). + +test_send_from_coroutine(Config) -> + C = new_ctx(Config), + {ok, <<"sent">>} = py_context:call(C, ?TEST_MOD, send_from_coroutine, [self(), from_coro]), + receive <<"from_coro">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_whereis(Config) -> + C = new_ctx(Config), + Name = py_isolated_vm_probe, + true = register(Name, self()), + Self = self(), + {ok, Self} = py_context:call(C, ?TEST_MOD, whereis, [<<"py_isolated_vm_probe">>]), + {ok, none} = py_context:call(C, ?TEST_MOD, whereis, [<<"no_such_registered_name_xyz">>]), + unregister(Name), + stop(C). + +test_suspension_is_base_exception(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, suspension_is_base_exception, []), + stop(C). + +test_callback_inside_except_exception(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_echo">>, fun([X]) -> X end), + {ok, {<<"ok">>, 42}} = py_context:call(C, ?TEST_MOD, call_inside_except_exception, [<<"vm_echo">>, 42]), + py_callback:unregister(<<"vm_echo">>), + stop(C). + +%%% ============================================================================ +%%% Callback result encoding (py_callback_encoding_SUITE) +%%% ============================================================================ + +test_encoding_binary_with_escapes(Config) -> + C = new_ctx(Config), + Value = <<"back\\slash \"dq\" 'sq'\nnewline\ttab\r">>, + Value = probe(C, Value), + <<"str">> = probe_type(C, Value), + stop(C). + +test_encoding_binary_non_utf8(Config) -> + C = new_ctx(Config), + Value = <<0, 1, 255, 254, 128>>, + Value = probe(C, Value), + <<"bytes">> = probe_type(C, Value), + stop(C). + +test_encoding_large_binary(Config) -> + C = new_ctx(Config), + Value = binary:copy(<<"abcdefghij">>, 20000), + Value = probe(C, Value), + stop(C). + +test_encoding_atom_becomes_str(Config) -> + C = new_ctx(Config), + <<"some_atom">> = probe(C, some_atom), + <<"str">> = probe_type(C, some_atom), + stop(C). + +test_encoding_empty_list(Config) -> + C = new_ctx(Config), + [] = probe(C, []), + <<"list">> = probe_type(C, []), + stop(C). + +test_encoding_erlang_string(Config) -> + C = new_ctx(Config), + "abc" = probe(C, "abc"), + <<"list">> = probe_type(C, "abc"), + <<"abc">> = probe(C, <<"abc">>), + <<"str">> = probe_type(C, <<"abc">>), + stop(C). + +test_encoding_nested_containers(Config) -> + C = new_ctx(Config), + Value = #{<<"k">> => [1, 2.5, {a, b}, #{<<"inner">> => [[], {}]}]}, + Expected = #{<<"k">> => [1, 2.5, {<<"a">>, <<"b">>}, #{<<"inner">> => [[], {}]}]}, + Expected = probe(C, Value), + <<"dict">> = probe_type(C, Value), + stop(C). + +test_encoding_pid_and_ref(Config) -> + C = new_ctx(Config), + Pid = self(), + Pid = probe(C, Pid), + <<"Pid">> = probe_type(C, Pid), + Ref = make_ref(), + Ref = probe(C, Ref), + <<"Ref">> = probe_type(C, Ref), + stop(C). + +test_encoding_floats(Config) -> + C = new_ctx(Config), + lists:foreach(fun(F) -> F = probe(C, F) end, + [3.14159265358979, 1.0e-300, 1.7976931348623157e308, -0.0]), + stop(C). + +test_encoding_booleans_and_none(Config) -> + C = new_ctx(Config), + true = probe(C, true), + false = probe(C, false), + <<"bool">> = probe_type(C, true), + lists:foreach(fun(A) -> + none = probe(C, A), + <<"NoneType">> = probe_type(C, A) + end, [undefined, nil, none]), + stop(C). + +test_encoding_python_types(Config) -> + C = new_ctx(Config), + <<"tuple">> = probe_type(C, {1, 2}), + <<"int">> = probe_type(C, 42), + <<"float">> = probe_type(C, 1.5), + stop(C). + +%%% ============================================================================ +%%% Python threads calling Erlang (py_thread_callback_SUITE) +%%% ============================================================================ + +test_threads_call_erlang(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_tid">>, fun([T, I]) -> T * 1000 + I end), + {ok, {<<"ok">>, true, 40}} = py_context:call(C, ?TEST_MOD, thread_calls, [<<"vm_tid">>, 4, 10], #{}, 30000), + py_callback:unregister(<<"vm_tid">>), + stop(C). + +test_threadpool_calls(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_double">>, fun([X]) -> X * 2 end), + {ok, true} = py_context:call(C, ?TEST_MOD, pool_calls, [<<"vm_double">>, 8, 200], #{}, 30000), + py_callback:unregister(<<"vm_double">>), + stop(C). + +test_threadpool_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_fail">>, fun(_) -> throw(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, pool_error, [<<"vm_fail">>]), + py_callback:unregister(<<"vm_fail">>), + stop(C). + +%% @doc From a pool thread: two erlang.call round trips nested in one +%% expression. (A callback that re-enters the context itself would deadlock +%% in every mode: the main thread is busy waiting on the pool.) +test_threadpool_nested(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_nested">>, fun([X]) -> X + 11 end), + {ok, 42} = py_context:call(C, ?TEST_MOD, pool_nested, [<<"vm_nested">>], #{}, 30000), + py_callback:unregister(<<"vm_nested">>), + stop(C). + +test_threads_high_concurrency(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_tid">>, fun([T, I]) -> T * 1000 + I end), + {ok, {<<"ok">>, true, 1600}} = py_context:call(C, ?TEST_MOD, thread_calls, [<<"vm_tid">>, 32, 50], #{}, 60000), + py_callback:unregister(<<"vm_tid">>), + stop(C). + +%%% ============================================================================ +%%% Actor-style state (py_actor_SUITE) +%%% ============================================================================ + +test_counter_actor(Config) -> + C = new_ctx(Config), + lists:foreach(fun(I) -> + {ok, I} = py_context:call(C, ?TEST_MOD, counter_increment, []) + end, lists:seq(1, 100)), + {ok, 110} = py_context:call(C, ?TEST_MOD, counter_increment, [10]), + {ok, 110} = py_context:call(C, ?TEST_MOD, counter_value, []), + stop(C). + +test_state_reset_on_restart(Config) -> + C1 = new_ctx(Config), + {ok, V0} = py_context:call(C1, ?TEST_MOD, counter_value, []), + V1 = V0 + 1, + V2 = V0 + 2, + {ok, V1} = py_context:call(C1, ?TEST_MOD, counter_increment, []), + {ok, V2} = py_context:call(C1, ?TEST_MOD, counter_increment, []), + stop(C1), + C2 = new_ctx(Config), + %% A new context: worker mode shares the interpreter, so the module + %% state persists; isolated mode starts a new process and it does not. + {ok, V} = py_context:call(C2, ?TEST_MOD, counter_value, []), + case ?config(mode, Config) of + isolated -> 0 = V; + worker -> V2 = V + end, + stop(C2). + +test_state_isolated_between_contexts(Config) -> + C1 = new_ctx(Config), + C2 = new_ctx(Config), + ok = py_context:exec(C1, <<"who = 'one'">>), + ok = py_context:exec(C2, <<"who = 'two'">>), + {ok, <<"one">>} = py_context:eval(C1, <<"who">>), + {ok, <<"two">>} = py_context:eval(C2, <<"who">>), + stop(C1), + stop(C2). + +%%% ============================================================================ +%%% Message flow both ways +%%% ============================================================================ + +%% @doc 1000 rounds of Erlang -> Python -> Erlang fun -> Python -> Erlang, +%% ordering asserted and no message left behind. +test_ping_pong(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_incr">>, fun([X]) -> X + 1 end), + {ok, 1000} = py_context:call(C, ?TEST_MOD, ping_pong, [<<"vm_incr">>, 1000], #{}, 60000), + py_callback:unregister(<<"vm_incr">>), + receive Any -> ct:fail({unexpected_message, Any}) after 0 -> ok end, + stop(C). + +%% @doc Erlang feeds terms to the child through a callback it polls. +test_feed_through_callback(Config) -> + C = new_ctx(Config), + Feeder = spawn_link(fun() -> feeder(lists:seq(1, 200)) end), + py_callback:register(<<"vm_next">>, fun([]) -> + Feeder ! {next, self()}, + receive {item, I} -> I after 5000 -> none end + end), + {ok, Items} = py_context:call(C, ?TEST_MOD, poll_feed, [<<"vm_next">>, 200], #{}, 60000), + Expected = lists:seq(1, 200), + Expected = Items, + py_callback:unregister(<<"vm_next">>), + Feeder ! stop, + stop(C). + +feeder([]) -> + receive stop -> ok; {next, From} -> From ! {item, none}, feeder([]) end; +feeder([H | T] = L) -> + receive + stop -> ok; + {next, From} -> From ! {item, H}, feeder(T) + after 10000 -> + feeder(L) + end. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +probe(C, Value) -> + py_callback:register(<<"vm_probe">>, fun(_) -> Value end), + {ok, Got} = py_context:call(C, ?TEST_MOD, callback, [<<"vm_probe">>]), + py_callback:unregister(<<"vm_probe">>), + Got. + +probe_type(C, Value) -> + py_callback:register(<<"vm_probe">>, fun(_) -> Value end), + {ok, Type} = py_context:call(C, ?TEST_MOD, callback_type, [<<"vm_probe">>]), + py_callback:unregister(<<"vm_probe">>), + Type. + +collect_items(Acc) -> + receive + {<<"item">>, I} -> collect_items([I | Acc]); + <<"done">> -> lists:reverse(Acc) + after 5000 -> + ct:fail({incomplete, length(Acc)}) + end. + +new_ctx(Config) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(#{mode => Mode, paths => [TestDir]}), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_test_isolated.py b/test/py_test_isolated.py new file mode 100644 index 0000000..1567b47 --- /dev/null +++ b/test/py_test_isolated.py @@ -0,0 +1,374 @@ +"""Helpers for the isolated-mode suites (py_isolated_SUITE, +py_isolated_vm_SUITE, py_isolated_async_SUITE). + +Every function here runs unchanged in worker and isolated mode; the suites +run both so a divergence between modes shows up as a failing pair. +""" + +import asyncio +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +import erlang + +# --------------------------------------------------------------------------- +# basics +# --------------------------------------------------------------------------- + +def add(a, b): + return a + b + + +def kwargs_probe(*args, **kwargs): + return (list(args), sorted(kwargs.items())) + + +def identity(x): + return x + + +def type_name(x): + return type(x).__name__ + + +def raise_value_error(msg): + raise ValueError(msg) + + +def big_payload(n): + return b'x' * n + + +def sleep_then(seconds, value): + time.sleep(seconds) + return value + + +def blocked_sleep(seconds): + """Sleep with every signal blocked: a soft interrupt cannot land, only + SIGKILL can end this. Exercises the kill backstop.""" + import signal + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGUSR1, signal.SIGINT}) + time.sleep(seconds) + return 'slept' + + +def segfault(): + import ctypes + ctypes.memset(0, 0, 1) + + +def close_control_socket(): + """Break the control socket from inside the child (mid-conversation), + to exercise the fail-loud discipline.""" + import os + import sys + rt = sys.modules['_erlang_impl._isolated'] + # The runtime installed in this process + import gc + for obj in gc.get_objects(): + if isinstance(obj, rt.Runtime): + os.close(obj.sock.fileno()) + break + time.sleep(5) + return 'unreachable' + + +def allocate(n_bytes): + data = bytearray(n_bytes) + return len(data) + + +def allocate_and_touch(n_bytes): + """Allocate and write every page, so the memory is resident (a bare + bytearray may stay untouched virtual memory on some platforms).""" + data = bytearray(n_bytes) + for i in range(0, n_bytes, 4096): + data[i] = 1 + return len(data) + + +def spin(seconds): + end = time.monotonic() + seconds + n = 0 + while time.monotonic() < end: + n += 1 + return n + + +def numpy_sum(n): + import numpy + return int(numpy.arange(n).sum()) + + +# --------------------------------------------------------------------------- +# VM interaction: pids, send, whereis, callbacks +# --------------------------------------------------------------------------- + +def is_pid(x): + return isinstance(x, erlang.Pid) + + +def pid_equal(a, b): + return a == b + + +def pid_hash_equal(a, b): + return hash(a) == hash(b) + + +def pid_in_structure(pid): + return {'owner': pid, 'list': [pid, (pid, 1)]} + + +def send(pid, msg): + erlang.send(pid, msg) + return True + + +def send_many(pid, n): + for i in range(n): + erlang.send(pid, ('item', i)) + erlang.send(pid, 'done') + return n + + +def send_timing(pid, n): + t0 = time.perf_counter() + for i in range(n): + erlang.send(pid, i) + return (time.perf_counter() - t0) * 1000.0 + + +def send_to_dead(pid): + try: + erlang.send(pid, 'msg') + return 'sent' + except erlang.ProcessError: + return 'process_error' + + +def send_bad_pid(): + try: + erlang.send('not_a_pid', 'msg') + return 'sent' + except TypeError: + return 'type_error' + + +def send_from_coroutine(pid, msg): + async def go(): + erlang.send(pid, msg) + return 'sent' + return asyncio.run(go()) + + +def whereis(name): + return erlang.whereis(name) + + +def suspension_is_base_exception(): + return (issubclass(erlang.SuspensionRequired, BaseException) + and not issubclass(erlang.SuspensionRequired, Exception)) + + +def call_inside_except_exception(name, arg): + """A callback inside `except Exception` must work in every mode.""" + try: + return ('ok', erlang.call(name, arg)) + except Exception as exc: + return ('caught', type(exc).__name__) + + +def callback(name, *args): + return erlang.call(name, *args) + + +def callback_error_type(name): + try: + erlang.call(name) + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +def callback_type(name): + return type(erlang.call(name)).__name__ + + +def ping_pong(name, rounds): + """Each round calls Erlang with the round number and checks the answer.""" + for i in range(rounds): + got = erlang.call(name, i) + if got != i + 1: + return ('mismatch', i, got) + return rounds + + +def poll_feed(name, expect): + """Pull terms from Erlang through a callback until `expect` items.""" + items = [] + while len(items) < expect: + item = erlang.call(name) + if item is None: + time.sleep(0.001) + continue + items.append(item) + return items + + +# --------------------------------------------------------------------------- +# Threads calling Erlang +# --------------------------------------------------------------------------- + +def thread_calls(name, n_threads, n_calls): + results = {} + errors = [] + + def worker(tid): + try: + results[tid] = [erlang.call(name, tid, i) for i in range(n_calls)] + except Exception as exc: + errors.append(repr(exc)) + + threads = [threading.Thread(target=worker, args=(t,)) for t in range(n_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + if errors: + return ('errors', errors) + ok = all(results[t] == [t * 1000 + i for i in range(n_calls)] + for t in range(n_threads)) + return ('ok', ok, n_threads * n_calls) + + +def pool_calls(name, n_workers, n_calls): + with ThreadPoolExecutor(max_workers=n_workers) as pool: + futures = [pool.submit(erlang.call, name, i) for i in range(n_calls)] + got = [f.result() for f in futures] + return got == [i * 2 for i in range(n_calls)] + + +def pool_error(name): + with ThreadPoolExecutor(max_workers=2) as pool: + fut = pool.submit(erlang.call, name) + try: + fut.result() + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +def pool_nested(name): + """A pool thread makes an erlang.call whose argument is itself the + result of an erlang.call: two round trips nested in one thread.""" + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(lambda: erlang.call(name, erlang.call(name, 20))).result() + + +# --------------------------------------------------------------------------- +# Actor-style state +# --------------------------------------------------------------------------- + +class Counter: + def __init__(self): + self.value = 0 + + def increment(self, by=1): + self.value += by + return self.value + + +_counter = Counter() + + +def counter_increment(by=1): + return _counter.increment(by) + + +def counter_value(): + return _counter.value + + +# --------------------------------------------------------------------------- +# asyncio +# --------------------------------------------------------------------------- + +async def async_add(a, b): + await asyncio.sleep(0) + return a + b + + +async def async_sleep_gather(n, seconds): + async def one(i): + await asyncio.sleep(seconds) + return i + return await asyncio.gather(*[one(i) for i in range(n)]) + + +async def async_raise(msg): + await asyncio.sleep(0) + raise KeyError(msg) + + +async def async_big(n): + await asyncio.sleep(0) + return b'y' * n + + +async def async_erlang_call(name, x): + return await erlang.async_call(name, x) + + +async def async_erlang_calls(name, n): + return await asyncio.gather(*[erlang.async_call(name, i) for i in range(n)]) + + +async def async_erlang_call_error(name): + try: + await erlang.async_call(name) + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +async def async_send(pid, msg): + erlang.send(pid, msg) + return 'sent' + + +async def stream_to(pid, n): + async def agen(): + for i in range(n): + await asyncio.sleep(0) + yield i + async for item in agen(): + erlang.send(pid, ('item', item)) + erlang.send(pid, 'done') + return n + + +async def task_value(i): + await asyncio.sleep(0.001 * (i % 5)) + return i * i + + +async def slow_task(seconds): + await asyncio.sleep(seconds) + return 'slow_done' + + +async def block_loop(seconds): + time.sleep(seconds) + return 'unblocked' + + +def run_helper_compat(): + """erlang.run / erlang.sleep / erlang.spawn_task behave like stdlib.""" + async def main(): + await erlang.sleep(0.001) + t = erlang.spawn_task(async_add(1, 2)) + return await t + return erlang.run(main())