diff --git a/CHANGELOG.md b/CHANGELOG.md index 8983353..37bdd1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,22 @@ storms, loop churn, 60 s mixed workload with resource counters checked. - Guide: `docs/isolated.md`, with what each of the three modes guarantees. +### Changed + +- The NIF side of every context request goes through one dispatcher + (`ctx_dispatch`, `ctx_dispatch_async` in `c_src/py_nif.c`) instead of a + per-request copy of the enqueue-and-wait loop; the execute functions are + `ctx_execute_*` and the thread functions `ctx_thread_main_*`, since both + serve worker and owngil contexts. Creating a process-local env and + applying imports or paths run on the context thread in `worker` mode too; + the scheduler-side copies of those paths are gone. +- The NIF function table is assembled from one `PY_*_NIFS` macro per area, + defined at the end of the file that owns the NIFs. +- `py_context` keeps the API and the reply protocol; the process body for + embedded modes moved to `py_context_embedded`. `py` delegates streaming, + virtual environments and shared dicts to `py_stream`, `py_venv` and + `py_shared_dict`. The public API is unchanged. + ### Removed - The legacy worker API (`py_nif:worker_new/0,1`, `worker_call`, `worker_eval`, diff --git a/c_src/README.md b/c_src/README.md index 8d1dc06..d3c02f8 100644 --- a/c_src/README.md +++ b/c_src/README.md @@ -14,7 +14,7 @@ where things are. | File | What it owns | Notes | |---|---|---| | `py_nif.h` | All shared types: `py_context_t` and its request queue, request types, callback and suspension state, runtime state machine, atoms, globals, declarations | 2.4k lines. The struct comments carry the locking rules; read `py_context_t` before touching threads | -| `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `worker_context_thread_main` and `owngil_context_thread_main`, `owngil_execute_*` (used by both thread kinds), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | +| `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `ctx_thread_main_worker` and `ctx_thread_main_owngil`, `ctx_execute_*` (one set for both thread kinds), `ctx_dispatch` / `ctx_dispatch_async` (the only way a NIF reaches a context thread), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end, assembled from the `PY_*_NIFS` macros of the other files | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | | `py_convert.c` | `py_to_term`, `term_to_py`, depth limits, tagged tuples (`{bytes, B}`, `{'$py_shm', ...}`), error tuples `{error, {Type, Msg}}` | The type mapping tables in the comments are the reference for `_etf.py` | | `py_exec.c` | Execution mode detection (free-threaded or GIL build) | | | `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `schedule*`, `Atom`/`Pid`/`Ref` types, callback delivery paths (suspension, blocking pipe, async pipe), channel and shared-dict methods, callback name registry | `erlang_call_impl` documents the path precedence | @@ -29,8 +29,8 @@ where things are. ## Where the live paths are - `py:call/3` in worker or owngil mode: `nif_context_call_async` (`py_nif.c`) - enqueues; `worker_context_thread_main` or `owngil_context_thread_main` - dequeues and calls `owngil_execute_request`; the reply goes out as + enqueues; `ctx_thread_main_worker` or `ctx_thread_main_owngil` + dequeues and calls `ctx_execute_request`; the reply goes out as `{py_result, Ref, Result}`. - `erlang.call` from Python: `erlang_call_impl` (`py_callback.c`). - Interrupt: `nif_context_interrupt` (`py_nif.c`), `interrupt_mutex` rules on @@ -57,7 +57,9 @@ where things are. 1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` next to related code. -2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of `py_nif.c`. +2. Add `{"x", Arity, nif_x, Flags}` to the `PY_*_NIFS` macro at the end of + that file (or to the `py_nif.c` block of `nif_funcs[]` for NIFs that + live there). 3. Add the stub and its `-spec` and doc to `src/py_nif.erl`. 4. Cover it in a suite; `rebar3 dialyzer` and `rebar3 xref` must stay clean. diff --git a/c_src/py_buffer.c b/c_src/py_buffer.c index f21f25e..1ee1927 100644 --- a/c_src/py_buffer.c +++ b/c_src/py_buffer.c @@ -1103,3 +1103,10 @@ static ERL_NIF_TERM nif_py_buffer_close(ErlNifEnv *env, int argc, return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_BUFFER_NIFS \ + {"py_buffer_create", 1, nif_py_buffer_create, 0}, \ + {"py_buffer_write", 2, nif_py_buffer_write, 0}, \ + {"py_buffer_close", 1, nif_py_buffer_close, 0} diff --git a/c_src/py_callback.c b/c_src/py_callback.c index b560673..772d817 100644 --- a/c_src/py_callback.c +++ b/c_src/py_callback.c @@ -4053,3 +4053,9 @@ static ERL_NIF_TERM nif_unregister_callback_name(ErlNifEnv *env, int argc, const return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_CALLBACK_NIFS \ + {"register_callback_name", 1, nif_register_callback_name, 0}, \ + {"unregister_callback_name", 1, nif_unregister_callback_name, 0} diff --git a/c_src/py_channel.c b/c_src/py_channel.c index ecfba9f..c78ba38 100644 --- a/c_src/py_channel.c +++ b/c_src/py_channel.c @@ -1033,3 +1033,21 @@ ERL_NIF_TERM nif_byte_channel_wait_bytes(ErlNifEnv *env, int argc, const ERL_NIF /* Return ok - Python will await Future */ return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_CHANNEL_NIFS \ + {"channel_create", 0, nif_channel_create, 0}, \ + {"channel_create", 1, nif_channel_create, 0}, \ + {"channel_send", 2, nif_channel_send, 0}, \ + {"channel_receive", 2, nif_channel_receive, 0}, \ + {"channel_try_receive", 1, nif_channel_try_receive, 0}, \ + {"channel_reply", 3, nif_channel_reply, 0}, \ + {"channel_close", 1, nif_channel_close, 0}, \ + {"channel_info", 1, nif_channel_info, 0}, \ + {"channel_wait", 3, nif_channel_wait, 0}, \ + {"channel_cancel_wait", 2, nif_channel_cancel_wait, 0}, \ + {"channel_register_sync_waiter", 1, nif_channel_register_sync_waiter, 0}, \ + {"byte_channel_send_bytes", 2, nif_byte_channel_send_bytes, 0}, \ + {"byte_channel_try_receive_bytes", 1, nif_byte_channel_try_receive_bytes, 0}, \ + {"byte_channel_wait_bytes", 3, nif_byte_channel_wait_bytes, 0} diff --git a/c_src/py_event_loop.c b/c_src/py_event_loop.c index 91b37ca..f88d46f 100644 --- a/c_src/py_event_loop.c +++ b/c_src/py_event_loop.c @@ -5735,7 +5735,7 @@ ERL_NIF_TERM nif_reactor_on_read_ready(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_read_to_owngil(env, ctx, fd, buffer); + return dispatch_reactor_read(env, ctx, fd, buffer); } #endif @@ -5831,7 +5831,7 @@ ERL_NIF_TERM nif_reactor_on_write_ready(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_write_to_owngil(env, ctx, fd); + return dispatch_reactor_write(env, ctx, fd); } #endif @@ -5917,7 +5917,7 @@ ERL_NIF_TERM nif_reactor_init_connection(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_init_to_owngil(env, ctx, fd, argv[2]); + return dispatch_reactor_init(env, ctx, fd, argv[2]); } #endif @@ -8528,3 +8528,74 @@ int init_subinterpreter_event_loop(ErlNifEnv *env) { } return 0; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_EVENT_LOOP_NIFS \ + {"set_event_loop_priv_dir", 1, nif_set_event_loop_priv_dir, 0}, \ + {"event_loop_new", 0, nif_event_loop_new, 0}, \ + {"event_loop_destroy", 1, nif_event_loop_destroy, 0}, \ + {"event_loop_set_router", 2, nif_event_loop_set_router, 0}, \ + {"event_loop_set_worker", 2, nif_event_loop_set_worker, 0}, \ + {"event_loop_set_id", 2, nif_event_loop_set_id, 0}, \ + {"event_loop_wakeup", 1, nif_event_loop_wakeup, 0}, \ + {"event_loop_run_async", 7, nif_event_loop_run_async, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"submit_task", 7, nif_submit_task, 0}, \ + {"submit_task_with_env", 8, nif_submit_task_with_env, 0}, \ + {"process_ready_tasks", 1, nif_process_ready_tasks, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"event_loop_set_py_loop", 2, nif_event_loop_set_py_loop, 0}, \ + {"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"add_reader", 3, nif_add_reader, 0}, \ + {"remove_reader", 2, nif_remove_reader, 0}, \ + {"add_writer", 3, nif_add_writer, 0}, \ + {"remove_writer", 2, nif_remove_writer, 0}, \ + {"call_later", 3, nif_call_later, 0}, \ + {"cancel_timer", 2, nif_cancel_timer, 0}, \ + {"poll_events", 2, nif_poll_events, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"get_pending", 1, nif_get_pending, 0}, \ + {"dispatch_callback", 3, nif_dispatch_callback, 0}, \ + {"dispatch_timer", 2, nif_dispatch_timer, 0}, \ + {"get_fd_callback_id", 2, nif_get_fd_callback_id, 0}, \ + {"reselect_reader", 2, nif_reselect_reader, 0}, \ + {"reselect_writer", 2, nif_reselect_writer, 0}, \ + {"reselect_reader_fd", 1, nif_reselect_reader_fd, 0}, \ + {"reselect_writer_fd", 1, nif_reselect_writer_fd, 0}, \ + {"handle_fd_event", 2, nif_handle_fd_event, 0}, \ + {"handle_fd_event_and_reselect", 2, nif_handle_fd_event_and_reselect, 0}, \ + {"fd_arm", 2, nif_fd_arm, 0}, \ + {"stop_reader", 1, nif_stop_reader, 0}, \ + {"start_reader", 1, nif_start_reader, 0}, \ + {"stop_writer", 1, nif_stop_writer, 0}, \ + {"start_writer", 1, nif_start_writer, 0}, \ + {"close_fd", 1, nif_close_fd, 0}, \ + {"create_test_pipe", 0, nif_create_test_pipe, 0}, \ + {"close_test_fd", 1, nif_close_test_fd, 0}, \ + {"dup_fd", 1, nif_dup_fd, 0}, \ + {"write_test_fd", 2, nif_write_test_fd, 0}, \ + {"read_test_fd", 2, nif_read_test_fd, 0}, \ + {"create_test_tcp_listener", 1, nif_create_test_tcp_listener, 0}, \ + {"accept_test_tcp", 1, nif_accept_test_tcp, 0}, \ + {"connect_test_tcp", 2, nif_connect_test_tcp, 0}, \ + {"create_test_udp_socket", 1, nif_create_test_udp_socket, 0}, \ + {"recvfrom_test_udp", 2, nif_recvfrom_test_udp, 0}, \ + {"sendto_test_udp", 4, nif_sendto_test_udp, 0}, \ + {"set_udp_broadcast", 2, nif_set_udp_broadcast, 0}, \ + {"set_python_event_loop", 1, nif_set_python_event_loop, 0}, \ + {"set_isolation_mode", 1, nif_set_isolation_mode, 0}, \ + {"set_shared_worker", 1, nif_set_shared_worker, 0}, \ + {"context_get_event_loop", 1, nif_context_get_event_loop, 0}, \ + {"reactor_register_fd", 3, nif_reactor_register_fd, 0}, \ + {"reactor_reselect_read", 1, nif_reactor_reselect_read, 0}, \ + {"reactor_select_write", 1, nif_reactor_select_write, 0}, \ + {"get_fd_from_resource", 1, nif_get_fd_from_resource, 0}, \ + {"reactor_on_read_ready", 2, nif_reactor_on_read_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_on_write_ready", 2, nif_reactor_on_write_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_init_connection", 3, nif_reactor_init_connection, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_close_fd", 2, nif_reactor_close_fd, 0}, \ + {"fd_read", 2, nif_fd_read, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"fd_write", 2, nif_fd_write, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"fd_select_read", 1, nif_fd_select_read, 0}, \ + {"fd_select_write", 1, nif_fd_select_write, 0}, \ + {"fd_close", 1, nif_fd_close, 0}, \ + {"socketpair", 0, nif_socketpair, 0} diff --git a/c_src/py_logging.c b/c_src/py_logging.c index 14b1f7e..1ca5077 100644 --- a/c_src/py_logging.c +++ b/c_src/py_logging.c @@ -453,3 +453,11 @@ static ERL_NIF_TERM nif_clear_trace_receiver(ErlNifEnv *env, int argc, const ERL return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_LOGGING_NIFS \ + {"set_log_receiver", 2, nif_set_log_receiver, 0}, \ + {"clear_log_receiver", 0, nif_clear_log_receiver, 0}, \ + {"set_trace_receiver", 1, nif_set_trace_receiver, 0}, \ + {"clear_trace_receiver", 0, nif_clear_trace_receiver, 0} diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 14c3b62..69fc110 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -670,218 +670,6 @@ static inline_continuation_t *create_inline_continuation( return cont; } -/** - * @brief NIF: Execute inline continuation - * - * This is the continuation function called by enif_schedule_nif(). - * It executes the Python function and handles the result: - * - InlineScheduleMarker: chain via another enif_schedule_nif - * - ScheduleMarker: return {schedule, ...} to Erlang - * - Suspension: return {suspended, ...} to Erlang - * - Normal result: return {ok, Result} - */ -static ERL_NIF_TERM nif_inline_continuation(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - - inline_continuation_t *cont; - if (!enif_get_resource(env, argv[0], INLINE_CONTINUATION_RESOURCE_TYPE, (void **)&cont)) { - return make_error(env, "invalid_continuation"); - } - - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - /* Check depth limit */ - if (cont->depth >= MAX_INLINE_CONTINUATION_DEPTH) { - return make_error(env, "inline_continuation_depth_exceeded"); - } - - py_context_t *ctx = cont->ctx; - if (ctx == NULL || ctx->destroyed) { - return make_error(env, "context_destroyed"); - } - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context for callback support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Set callback env for consume_time_slice */ - ErlNifEnv *prev_callback_env = tl_callback_env; - tl_callback_env = env; - - ERL_NIF_TERM result; - - /* Import module and get function */ - PyObject *func = NULL; - PyObject *module = NULL; - - /* Priority for __main__ lookups: - * 1. Captured globals/locals from the marker (caller's frame) - * 2. local_env globals (process-local environment) - * 3. ctx->globals/locals (context defaults) - */ - py_env_resource_t *local_env = (py_env_resource_t *)cont->local_env; - - if (strcmp(cont->module_name, "__main__") == 0) { - /* Try captured globals first (from caller's frame) */ - if (cont->globals != NULL) { - func = PyDict_GetItemString(cont->globals, cont->func_name); - } - /* Try captured locals */ - if (func == NULL && cont->locals != NULL) { - func = PyDict_GetItemString(cont->locals, cont->func_name); - } - /* Fallback to local_env globals */ - if (func == NULL && local_env != NULL) { - func = PyDict_GetItemString(local_env->globals, cont->func_name); - } - /* Fallback to context globals/locals */ - if (func == NULL) { - func = PyDict_GetItemString(ctx->globals, cont->func_name); - } - if (func == NULL) { - func = PyDict_GetItemString(ctx->locals, cont->func_name); - } - if (func != NULL) { - Py_INCREF(func); - } else { - PyErr_Format(PyExc_NameError, "name '%s' is not defined", cont->func_name); - } - } else { - module = PyImport_ImportModule(cont->module_name); - if (module != NULL) { - func = PyObject_GetAttrString(module, cont->func_name); - Py_DECREF(module); - } - } - - if (func == NULL) { - result = make_py_error(env); - goto cleanup; - } - - /* Build args tuple */ - PyObject *args = cont->args; - if (args == NULL) { - args = PyTuple_New(0); - if (args == NULL) { - Py_DECREF(func); - result = make_py_error(env); - goto cleanup; - } - } else { - Py_INCREF(args); - } - - /* Get kwargs */ - PyObject *kwargs = cont->kwargs; - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - - if (py_result == NULL) { - /* Check for pending callback */ - if (tl_pending_callback) { - PyErr_Clear(); - - /* Create suspended context state for callback handling */ - ErlNifBinary module_bin, func_bin; - enif_alloc_binary(cont->module_len, &module_bin); - memcpy(module_bin.data, cont->module_name, cont->module_len); - enif_alloc_binary(cont->func_len, &func_bin); - memcpy(func_bin.data, cont->func_name, cont->func_len); - - /* Convert args to Erlang term for replay */ - ERL_NIF_TERM args_term = enif_make_list(env, 0); - if (cont->args != NULL) { - args_term = py_to_term(env, cont->args); - } - - ERL_NIF_TERM kwargs_term = enif_make_new_map(env); - if (cont->kwargs != NULL) { - kwargs_term = py_to_term(env, cont->kwargs); - } - - suspended_context_state_t *suspended = create_suspended_context_state_for_call( - env, ctx, &module_bin, &func_bin, args_term, kwargs_term); - - enif_release_binary(&module_bin); - enif_release_binary(&func_bin); - - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Chain via another enif_schedule_nif */ - inline_continuation_t *next_cont = create_inline_continuation( - ctx, cont->local_env, py_result, cont->depth + 1); - Py_DECREF(py_result); - - if (next_cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, next_cont); - enif_release_resource(next_cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_callback_env = prev_callback_env; - clear_pending_callback_tls(); - - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Switch to schedule_py path */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - /* Normal result */ - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - -cleanup: - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_callback_env = prev_callback_env; - - /* Clear pending callback TLS */ - clear_pending_callback_tls(); - - /* Release thread state */ - py_context_release(&guard); - - return result; -} /* ============================================================================ * Initialization @@ -1571,7 +1359,7 @@ static void ctx_queue_cancel_all(py_context_t *ctx) { /** * @brief Execute a call request in the OWN_GIL thread */ -static void owngil_execute_call(py_context_t *ctx) { +static void ctx_execute_call(py_context_t *ctx) { /* Decode request from shared_env */ ERL_NIF_TERM module_term, func_term, args_term, kwargs_term; const ERL_NIF_TERM *tuple_terms; @@ -1713,7 +1501,7 @@ static void owngil_execute_call(py_context_t *ctx) { /** * @brief Execute an eval request in the OWN_GIL thread */ -static void owngil_execute_eval(py_context_t *ctx) { +static void ctx_execute_eval(py_context_t *ctx) { /* Decode request: {Code, Locals} */ const ERL_NIF_TERM *tuple_terms; int tuple_arity; @@ -1782,7 +1570,7 @@ static void owngil_execute_eval(py_context_t *ctx) { /** * @brief Execute an exec request in the OWN_GIL thread */ -static void owngil_execute_exec(py_context_t *ctx) { +static void ctx_execute_exec(py_context_t *ctx) { ErlNifBinary code_bin; if (!enif_inspect_binary(ctx->shared_env, ctx->request_term, &code_bin)) { ctx->response_term = enif_make_tuple2(ctx->shared_env, @@ -1829,7 +1617,7 @@ static void owngil_execute_exec(py_context_t *ctx) { /** * @brief Execute a reactor on_read_ready request in OWN_GIL thread */ -static void owngil_execute_reactor_read(py_context_t *ctx) { +static void ctx_execute_reactor_read(py_context_t *ctx) { /* Extract fd from request term (it's just an integer) */ int fd; if (!enif_get_int(ctx->shared_env, ctx->request_term, &fd)) { @@ -1860,7 +1648,7 @@ static void owngil_execute_reactor_read(py_context_t *ctx) { /** * @brief Execute a reactor on_write_ready request in OWN_GIL thread */ -static void owngil_execute_reactor_write(py_context_t *ctx) { +static void ctx_execute_reactor_write(py_context_t *ctx) { /* Extract fd from request term */ int fd; if (!enif_get_int(ctx->shared_env, ctx->request_term, &fd)) { @@ -1879,7 +1667,7 @@ static void owngil_execute_reactor_write(py_context_t *ctx) { /** * @brief Execute a reactor init_connection request in OWN_GIL thread */ -static void owngil_execute_reactor_init(py_context_t *ctx) { +static void ctx_execute_reactor_init(py_context_t *ctx) { /* Extract {Fd, ClientInfo} from request term */ const ERL_NIF_TERM *tuple; int arity; @@ -1910,7 +1698,7 @@ static void owngil_execute_reactor_init(py_context_t *ctx) { * * Uses penv->globals/locals instead of ctx->globals/locals */ -static void owngil_execute_exec_with_env(py_context_t *ctx) { +static void ctx_execute_exec_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -1987,7 +1775,7 @@ static void owngil_execute_exec_with_env(py_context_t *ctx) { * * Uses penv->globals/locals instead of ctx->globals/locals */ -static void owngil_execute_eval_with_env(py_context_t *ctx) { +static void ctx_execute_eval_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2250,7 +2038,7 @@ static void owngil_execute_eval_with_env(py_context_t *ctx) { * * Uses penv->globals for function lookup in __main__ module */ -static void owngil_execute_call_with_env(py_context_t *ctx) { +static void ctx_execute_call_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2427,7 +2215,7 @@ static void owngil_execute_call_with_env(py_context_t *ctx) { * Creates globals/locals dicts in the correct interpreter context. * The py_env_resource_t is passed via local_env_ptr. */ -static void owngil_execute_create_local_env(py_context_t *ctx) { +static void ctx_execute_create_local_env(py_context_t *ctx) { py_env_resource_t *res = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2498,7 +2286,7 @@ static void owngil_execute_create_local_env(py_context_t *ctx) { * Note: OWN_GIL contexts have their own dedicated interpreter, * so sys.modules is per-context in this mode. */ -static void owngil_execute_apply_imports(py_context_t *ctx) { +static void ctx_execute_apply_imports(py_context_t *ctx) { /* Process each import from request_term */ ERL_NIF_TERM head, tail = ctx->request_term; int arity; @@ -2547,7 +2335,7 @@ static void owngil_execute_apply_imports(py_context_t *ctx) { * * Paths are inserted at the beginning of sys.path. */ -static void owngil_execute_apply_paths(py_context_t *ctx) { +static void ctx_execute_apply_paths(py_context_t *ctx) { /* Get sys.path */ PyObject *sys_module = PyImport_ImportModule("sys"); if (sys_module == NULL) { @@ -2616,43 +2404,43 @@ static void owngil_execute_apply_paths(py_context_t *ctx) { /** * @brief Execute a request based on its type */ -static void owngil_execute_request(py_context_t *ctx) { +static void ctx_execute_request(py_context_t *ctx) { switch (ctx->request_type) { case CTX_REQ_CALL: - owngil_execute_call(ctx); + ctx_execute_call(ctx); break; case CTX_REQ_EVAL: - owngil_execute_eval(ctx); + ctx_execute_eval(ctx); break; case CTX_REQ_EXEC: - owngil_execute_exec(ctx); + ctx_execute_exec(ctx); break; case CTX_REQ_REACTOR_ON_READ_READY: - owngil_execute_reactor_read(ctx); + ctx_execute_reactor_read(ctx); break; case CTX_REQ_REACTOR_ON_WRITE_READY: - owngil_execute_reactor_write(ctx); + ctx_execute_reactor_write(ctx); break; case CTX_REQ_REACTOR_INIT_CONNECTION: - owngil_execute_reactor_init(ctx); + ctx_execute_reactor_init(ctx); break; case CTX_REQ_EXEC_WITH_ENV: - owngil_execute_exec_with_env(ctx); + ctx_execute_exec_with_env(ctx); break; case CTX_REQ_EVAL_WITH_ENV: - owngil_execute_eval_with_env(ctx); + ctx_execute_eval_with_env(ctx); break; case CTX_REQ_CALL_WITH_ENV: - owngil_execute_call_with_env(ctx); + ctx_execute_call_with_env(ctx); break; case CTX_REQ_CREATE_LOCAL_ENV: - owngil_execute_create_local_env(ctx); + ctx_execute_create_local_env(ctx); break; case CTX_REQ_APPLY_IMPORTS: - owngil_execute_apply_imports(ctx); + ctx_execute_apply_imports(ctx); break; case CTX_REQ_APPLY_PATHS: - owngil_execute_apply_paths(ctx); + ctx_execute_apply_paths(ctx); break; default: ctx->response_term = enif_make_tuple2(ctx->shared_env, @@ -2681,7 +2469,7 @@ static void owngil_execute_request(py_context_t *ctx) { * with other Python threads. The benefit is stable thread affinity and * compatibility with all Python extensions. */ -static void *worker_context_thread_main(void *arg) { +static void *ctx_thread_main_worker(void *arg) { py_context_t *ctx = (py_context_t *)arg; /* Create namespace dictionaries on the worker thread under GIL */ @@ -2696,7 +2484,7 @@ static void *worker_context_thread_main(void *arg) { if (ctx->globals == NULL || ctx->locals == NULL || ctx->module_cache == NULL) { PyGILState_Release(gstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -2717,7 +2505,7 @@ static void *worker_context_thread_main(void *arg) { PyGILState_Release(gstate); /* Signal that we're ready */ - atomic_store(&ctx->worker_running, true); + atomic_store(&ctx->thread_running, true); /* Main request loop - uses queue instead of single-slot */ while (!atomic_load(&ctx->shutdown_requested)) { @@ -2786,7 +2574,7 @@ static void *worker_context_thread_main(void *arg) { * invariant on py_context::interrupt_mutex). */ py_context_exec_enter(ctx); gstate = PyGILState_Ensure(); - owngil_execute_request(ctx); /* Reuse execute functions */ + ctx_execute_request(ctx); /* Reuse execute functions */ PyGILState_Release(gstate); py_context_exec_leave(ctx); @@ -2842,7 +2630,7 @@ static void *worker_context_thread_main(void *arg) { ctx->module_cache = NULL; PyGILState_Release(gstate); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -2853,10 +2641,10 @@ static void *worker_context_thread_main(void *arg) { * @return 0 on success, -1 on failure */ static int worker_context_init(py_context_t *ctx) { - ctx->uses_worker_thread = true; + ctx->has_thread = true; /* Initialize worker thread state */ - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); atomic_store(&ctx->shutdown_requested, false); atomic_store(&ctx->leaked, false); @@ -2898,7 +2686,7 @@ static int worker_context_init(py_context_t *ctx) { ctx->module_cache = NULL; /* Start the worker thread */ - if (pthread_create(&ctx->worker_thread, NULL, worker_context_thread_main, ctx) != 0) { + if (pthread_create(&ctx->thread, NULL, ctx_thread_main_worker, ctx) != 0) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; pthread_cond_destroy(&ctx->queue_not_empty); @@ -2908,16 +2696,16 @@ static int worker_context_init(py_context_t *ctx) { /* Wait for thread to initialize or fail */ int wait_count = 0; - while (!atomic_load(&ctx->worker_running) && + while (!atomic_load(&ctx->thread_running) && !atomic_load(&ctx->init_error) && wait_count < 2000) { usleep(1000); /* 1ms */ wait_count++; } - if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->worker_running)) { + if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->thread_running)) { /* Thread failed to start */ - pthread_join(ctx->worker_thread, NULL); + pthread_join(ctx->thread, NULL); if (ctx->msg_env != NULL) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; @@ -2939,10 +2727,10 @@ static int worker_context_init(py_context_t *ctx) { * * @param ctx Context to shutdown */ -#define WORKER_SHUTDOWN_TIMEOUT_SECS 30 +#define CTX_THREAD_JOIN_TIMEOUT_SECS 30 -static void worker_context_shutdown(py_context_t *ctx) { - if (!ctx->uses_worker_thread) { +static void ctx_thread_shutdown_worker(py_context_t *ctx) { + if (!ctx->has_thread) { return; } @@ -2969,19 +2757,19 @@ static void worker_context_shutdown(py_context_t *ctx) { #if defined(__linux__) struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += WORKER_SHUTDOWN_TIMEOUT_SECS; - int rc = pthread_timedjoin_np(ctx->worker_thread, NULL, &deadline); + deadline.tv_sec += CTX_THREAD_JOIN_TIMEOUT_SECS; + int rc = pthread_timedjoin_np(ctx->thread, NULL, &deadline); join_succeeded = (rc == 0); #else - /* macOS/other: poll worker_running flag with timeout */ + /* macOS/other: poll thread_running flag with timeout */ int wait_ms = 0; - while (atomic_load(&ctx->worker_running) && - wait_ms < WORKER_SHUTDOWN_TIMEOUT_SECS * 1000) { + while (atomic_load(&ctx->thread_running) && + wait_ms < CTX_THREAD_JOIN_TIMEOUT_SECS * 1000) { usleep(100000); /* 100ms */ wait_ms += 100; } - if (!atomic_load(&ctx->worker_running)) { - pthread_join(ctx->worker_thread, NULL); + if (!atomic_load(&ctx->thread_running)) { + pthread_join(ctx->thread, NULL); join_succeeded = true; } #endif @@ -2999,7 +2787,7 @@ static void worker_context_shutdown(py_context_t *ctx) { * !ctx->leaked for the same reason). Future cleanup happens * at VM exit. */ fprintf(stderr, "Worker thread shutdown timeout after %d seconds, leaking context\n", - WORKER_SHUTDOWN_TIMEOUT_SECS); + CTX_THREAD_JOIN_TIMEOUT_SECS); atomic_store(&ctx->leaked, true); enif_keep_resource(ctx); return; @@ -3014,7 +2802,7 @@ static void worker_context_shutdown(py_context_t *ctx) { pthread_cond_destroy(&ctx->queue_not_empty); pthread_mutex_destroy(&ctx->queue_mutex); - ctx->uses_worker_thread = false; + ctx->has_thread = false; } /** @@ -3029,94 +2817,95 @@ static void worker_context_shutdown(py_context_t *ctx) { * @param request_data Request data term * @return Result term copied back to caller's env */ -#define WORKER_DISPATCH_TIMEOUT_SECS 30 +#define CTX_DISPATCH_TIMEOUT_SECS 30 /** - * @brief Dispatch a request to the worker thread with optional local environment + * @brief Allocate a request for @p ctx, or return NULL with *err set * - * @param env NIF environment - * @param ctx Context to dispatch to - * @param req_type Request type - * @param request_data Request data term - * @param local_env Optional local environment (NULL for default) - * @return Result term + * Every dispatch starts here: the context must have a running thread + * and must not be destroyed. The caller fills the request fields and + * hands it to ctx_dispatch_wait() or ctx_dispatch_async(). */ -static ERL_NIF_TERM dispatch_to_worker_thread_impl( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data, - void *local_env -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); +static ctx_request_t *ctx_request_begin(ErlNifEnv *env, py_context_t *ctx, + ctx_request_type_t req_type, + ERL_NIF_TERM *err) { + if (!atomic_load(&ctx->thread_running)) { + *err = make_error(env, "thread_not_running"); + return NULL; } - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); + *err = make_error(env, "context_destroyed"); + return NULL; } - - /* Create request struct */ ctx_request_t *req = ctx_request_create(); if (req == NULL) { - return make_error(env, "alloc_failed"); + *err = make_error(env, "alloc_failed"); + return NULL; } - - /* Populate request */ req->type = req_type; - req->request_data = enif_make_copy(req->request_env, request_data); - req->local_env_ptr = local_env; + return req; +} - /* Add extra reference for queue (caller holds 1, queue holds 1) */ +/** + * @brief Enqueue a prepared request and block until the context thread + * answers it (or the dispatch timeout passes) + * + * Takes over the caller's reference on @p req. Used by the blocking NIFs + * and by the reactor callbacks; the async NIFs use ctx_dispatch_async(). + */ +static ERL_NIF_TERM ctx_dispatch_wait(ErlNifEnv *env, py_context_t *ctx, + ctx_request_t *req) { + /* Queue holds one reference, the caller keeps one */ ctx_request_addref(req); ctx_queue_enqueue(ctx, req); - /* Wait for completion with timeout */ struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += WORKER_DISPATCH_TIMEOUT_SECS; + deadline.tv_sec += CTX_DISPATCH_TIMEOUT_SECS; - ERL_NIF_TERM result; pthread_mutex_lock(&req->mutex); - while (!atomic_load(&req->completed)) { int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); if (rc == ETIMEDOUT) { - /* Timeout - mark as cancelled and return error */ + /* The thread may still be inside a long Python call: fail this + * request only, the thread will skip it as cancelled. */ atomic_store(&req->cancelled, true); pthread_mutex_unlock(&req->mutex); + fprintf(stderr, "context dispatch timeout after %d seconds (request type %d)\n", + CTX_DISPATCH_TIMEOUT_SECS, (int)req->type); ctx_request_release(req); return make_error(env, "worker_timeout"); } } - pthread_mutex_unlock(&req->mutex); - /* Copy result to caller's environment */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - /* Release caller's reference */ + ERL_NIF_TERM result = (req->result_env != NULL) + ? enif_make_copy(env, req->result) + : make_error(env, "no_result"); ctx_request_release(req); - return result; } /** - * @brief Convenience wrapper for dispatch without local environment + * @brief Blocking dispatch of a request whose data is one term + * + * @param local_env Process-local env resource for *_WITH_ENV requests, + * NULL otherwise. */ -static ERL_NIF_TERM dispatch_to_worker_thread( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data -) { - return dispatch_to_worker_thread_impl(env, ctx, req_type, request_data, NULL); +static ERL_NIF_TERM ctx_dispatch(ErlNifEnv *env, py_context_t *ctx, + ctx_request_type_t req_type, + ERL_NIF_TERM request_data, void *local_env) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, req_type, &err); + if (req == NULL) { + return err; + } + req->request_data = enif_make_copy(req->request_env, request_data); + req->local_env_ptr = local_env; + return ctx_dispatch_wait(env, ctx, req); } + /** * @brief Async dispatch to worker thread (non-blocking) * @@ -3143,10 +2932,10 @@ static inline bool ctx_uses_async_thread(const py_context_t *ctx) { return true; } #endif - return ctx->uses_worker_thread; + return ctx->has_thread; } -static ERL_NIF_TERM dispatch_to_worker_thread_async( +static ERL_NIF_TERM ctx_dispatch_async( ErlNifEnv *env, py_context_t *ctx, ctx_request_type_t req_type, @@ -3155,22 +2944,11 @@ static ERL_NIF_TERM dispatch_to_worker_thread_async( ERL_NIF_TERM request_id, void *local_env ) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, req_type, &err); if (req == NULL) { - return make_error(env, "alloc_failed"); + return err; } - - /* Populate request */ - req->type = req_type; req->request_data = enif_make_copy(req->request_env, request_data); req->local_env_ptr = local_env; @@ -3198,7 +2976,7 @@ static ERL_NIF_TERM dispatch_to_worker_thread_async( * The queue-based pattern replaces the old single-slot pattern which had race * conditions when multiple callers dispatched concurrently. */ -static void *owngil_context_thread_main(void *arg) { +static void *ctx_thread_main_owngil(void *arg) { py_context_t *ctx = (py_context_t *)arg; /* Attach to Python runtime to create the subinterpreter. @@ -3222,7 +3000,7 @@ static void *owngil_context_thread_main(void *arg) { status.err_msg ? status.err_msg : "unknown error"); PyGILState_Release(gstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3238,7 +3016,7 @@ static void *owngil_context_thread_main(void *arg) { PyErr_Print(); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3251,7 +3029,7 @@ static void *owngil_context_thread_main(void *arg) { PyErr_Print(); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } ctx->event_loop = get_current_interpreter_event_loop(); @@ -3271,7 +3049,7 @@ static void *owngil_context_thread_main(void *arg) { Py_XDECREF(ctx->module_cache); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3293,7 +3071,7 @@ static void *owngil_context_thread_main(void *arg) { PyEval_SaveThread(); /* Signal that we're ready */ - atomic_store(&ctx->worker_running, true); + atomic_store(&ctx->thread_running, true); /* Main request loop - uses queue instead of single-slot */ while (!atomic_load(&ctx->shutdown_requested)) { @@ -3360,7 +3138,7 @@ static void *owngil_context_thread_main(void *arg) { * invariant on py_context::interrupt_mutex). */ py_context_exec_enter(ctx); PyEval_RestoreThread(ctx->own_gil_tstate); - owngil_execute_request(ctx); + ctx_execute_request(ctx); PyEval_SaveThread(); py_context_exec_leave(ctx); @@ -3443,7 +3221,7 @@ static void *owngil_context_thread_main(void *arg) { * After Py_NewInterpreterFromConfig switched us to the OWN_GIL interpreter, * the original gstate is no longer valid. Py_EndInterpreter handles cleanup. */ - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3451,755 +3229,83 @@ static void *owngil_context_thread_main(void *arg) { * Timeout for OWN_GIL dispatch in seconds. * If worker thread doesn't respond within this time, assume it's dead. */ -#define OWNGIL_DISPATCH_TIMEOUT_SECS 30 + /** - * @brief Dispatch a request to the worker thread and wait for response - * - * Uses the queue-based pattern: creates a request, enqueues it, waits for - * completion, and copies the result back to the caller's environment. + * @brief Run the reactor on_read_ready handler on the context thread * - * This replaces the old single-slot pattern which had race conditions when - * multiple callers dispatched concurrently. - * - * @param env Caller's NIF environment - * @param ctx Context with worker thread - * @param req_type Request type (CTX_REQ_CALL, CTX_REQ_EVAL, CTX_REQ_EXEC, etc.) - * @param request_data Request data term - * @return Result term copied back to caller's env + * @param buffer_ptr Reactor buffer resource; ownership moves to the request. */ -static ERL_NIF_TERM dispatch_to_owngil_thread( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); +ERL_NIF_TERM dispatch_reactor_read(ErlNifEnv *env, py_context_t *ctx, + int fd, void *buffer_ptr) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_ON_READ_READY, &err); + if (req == NULL) { + return err; } + req->request_data = enif_make_int(req->request_env, fd); + req->reactor_buffer_ptr = buffer_ptr; + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); +/** @brief Run the reactor on_write_ready handler on the context thread */ +ERL_NIF_TERM dispatch_reactor_write(ErlNifEnv *env, py_context_t *ctx, int fd) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_ON_WRITE_READY, &err); + if (req == NULL) { + return err; } + req->request_data = enif_make_int(req->request_env, fd); + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); +/** @brief Run the reactor init_connection handler on the context thread */ +ERL_NIF_TERM dispatch_reactor_init(ErlNifEnv *env, py_context_t *ctx, + int fd, ERL_NIF_TERM client_info) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_INIT_CONNECTION, &err); if (req == NULL) { - return make_error(env, "alloc_failed"); + return err; } + req->request_data = enif_make_tuple2(req->request_env, + enif_make_int(req->request_env, fd), + enif_make_copy(req->request_env, client_info)); + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - /* Populate request */ - req->type = req_type; - req->request_data = enif_make_copy(req->request_env, request_data); - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - /* Worker thread is unresponsive - mark request as cancelled */ - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - /* Don't mark worker as dead - it might still be processing - * a long-running Python operation. Just fail this request. */ - fprintf(stderr, "OWN_GIL dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - ctx_request_release(req); /* Release caller's ref */ - return make_error(env, "worker_timeout"); - } - } - pthread_mutex_unlock(&req->mutex); - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - /* Release caller's ref */ - ctx_request_release(req); - return result; -} +#endif /* HAVE_SUBINTERPRETERS */ /** - * @brief Dispatch reactor on_read_ready to OWN_GIL thread + * @brief Initialize OWN_GIL fields in a context and start the worker thread * - * Uses queue-based dispatch with per-request synchronization. + * @param ctx Context to initialize + * @return 0 on success, -1 on failure */ -ERL_NIF_TERM dispatch_reactor_read_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, void *buffer_ptr) { - if (!atomic_load(&ctx->worker_running)) { - enif_release_resource(buffer_ptr); - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - enif_release_resource(buffer_ptr); - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - enif_release_resource(buffer_ptr); - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_ON_READ_READY; - req->request_data = enif_make_int(req->request_env, fd); - req->reactor_buffer_ptr = buffer_ptr; /* Transfer ownership */ - req->reactor_fd = fd; +#ifdef HAVE_SUBINTERPRETERS +static int owngil_context_init(py_context_t *ctx) { + ctx->uses_own_gil = true; + ctx->own_gil_tstate = NULL; + ctx->own_gil_interp = NULL; + ctx->event_loop = NULL; - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); + /* Initialize worker thread state */ + atomic_store(&ctx->thread_running, false); + atomic_store(&ctx->init_error, false); + atomic_store(&ctx->shutdown_requested, false); + atomic_store(&ctx->leaked, false); - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - /* Request timeout - mark as cancelled but don't release buffer - * (worker will handle it when it gets to this request) */ - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); /* Release caller's ref */ - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - /* Release caller's ref */ - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch reactor on_write_ready to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -ERL_NIF_TERM dispatch_reactor_write_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_ON_WRITE_READY; - req->request_data = enif_make_int(req->request_env, fd); - req->reactor_fd = fd; - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor write dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch reactor init_connection to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -ERL_NIF_TERM dispatch_reactor_init_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, ERL_NIF_TERM client_info) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_INIT_CONNECTION; - ERL_NIF_TERM fd_term = enif_make_int(req->request_env, fd); - ERL_NIF_TERM info_copy = enif_make_copy(req->request_env, client_info); - req->request_data = enif_make_tuple2(req->request_env, fd_term, info_copy); - req->reactor_fd = fd; - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor init dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch exec_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_exec_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM code, py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_EXEC_WITH_ENV; - req->request_data = enif_make_copy(req->request_env, code); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL exec_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch eval_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_eval_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM code, ERL_NIF_TERM locals, - py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request: {Code, Locals} */ - req->type = CTX_REQ_EVAL_WITH_ENV; - ERL_NIF_TERM code_copy = enif_make_copy(req->request_env, code); - ERL_NIF_TERM locals_copy = enif_make_copy(req->request_env, locals); - req->request_data = enif_make_tuple2(req->request_env, code_copy, locals_copy); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL eval_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch call_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_call_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM module, ERL_NIF_TERM func, - ERL_NIF_TERM args, ERL_NIF_TERM kwargs, - py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request: {Module, Func, Args, Kwargs} */ - req->type = CTX_REQ_CALL_WITH_ENV; - ERL_NIF_TERM module_copy = enif_make_copy(req->request_env, module); - ERL_NIF_TERM func_copy = enif_make_copy(req->request_env, func); - ERL_NIF_TERM args_copy = enif_make_copy(req->request_env, args); - ERL_NIF_TERM kwargs_copy = enif_make_copy(req->request_env, kwargs); - req->request_data = enif_make_tuple4(req->request_env, - module_copy, func_copy, args_copy, kwargs_copy); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL call_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch create_local_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_create_local_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - py_env_resource_t *res -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_CREATE_LOCAL_ENV; - req->local_env_ptr = res; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL create_local_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch apply_imports to OWN_GIL worker thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_apply_imports_to_owngil( - ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM imports_term -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_APPLY_IMPORTS; - req->request_data = enif_make_copy(req->request_env, imports_term); - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL apply_imports dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch apply_paths request to OWN_GIL worker thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_apply_paths_to_owngil( - ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM paths_term -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_APPLY_PATHS; - req->request_data = enif_make_copy(req->request_env, paths_term); - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL apply_paths dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -#endif /* HAVE_SUBINTERPRETERS */ - -/** - * @brief Initialize OWN_GIL fields in a context and start the worker thread - * - * @param ctx Context to initialize - * @return 0 on success, -1 on failure - */ -#ifdef HAVE_SUBINTERPRETERS -static int owngil_context_init(py_context_t *ctx) { - ctx->uses_own_gil = true; - ctx->own_gil_tstate = NULL; - ctx->own_gil_interp = NULL; - ctx->event_loop = NULL; - - /* Initialize worker thread state */ - atomic_store(&ctx->worker_running, false); - atomic_store(&ctx->init_error, false); - atomic_store(&ctx->shutdown_requested, false); - atomic_store(&ctx->leaked, false); - - /* Initialize request queue */ - ctx->queue_head = NULL; - ctx->queue_tail = NULL; + /* Initialize request queue */ + ctx->queue_head = NULL; + ctx->queue_tail = NULL; /* Initialize legacy compatibility fields */ ctx->shared_env = NULL; @@ -4230,7 +3336,7 @@ static int owngil_context_init(py_context_t *ctx) { } /* Start the worker thread */ - if (pthread_create(&ctx->worker_thread, NULL, owngil_context_thread_main, ctx) != 0) { + if (pthread_create(&ctx->thread, NULL, ctx_thread_main_owngil, ctx) != 0) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; pthread_cond_destroy(&ctx->queue_not_empty); @@ -4240,16 +3346,16 @@ static int owngil_context_init(py_context_t *ctx) { /* Wait for thread to initialize or fail */ int wait_count = 0; - while (!atomic_load(&ctx->worker_running) && + while (!atomic_load(&ctx->thread_running) && !atomic_load(&ctx->init_error) && wait_count < 2000) { usleep(1000); /* 1ms */ wait_count++; } - if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->worker_running)) { + if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->thread_running)) { /* Thread failed to start */ - pthread_join(ctx->worker_thread, NULL); + pthread_join(ctx->thread, NULL); if (ctx->msg_env != NULL) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; @@ -4273,13 +3379,13 @@ static int owngil_context_init(py_context_t *ctx) { */ #define OWNGIL_SHUTDOWN_TIMEOUT_SECS 30 -static void owngil_context_shutdown(py_context_t *ctx) { +static void ctx_thread_shutdown_owngil(py_context_t *ctx) { if (!ctx->uses_own_gil) { return; } /* Signal shutdown and wake any worker parked on the condvar. - * See worker_context_shutdown for why we broadcast instead of + * See ctx_thread_shutdown_worker for why we broadcast instead of * enqueuing a CTX_REQ_SHUTDOWN sentinel. */ atomic_store(&ctx->shutdown_requested, true); ctx_queue_cancel_all(ctx); @@ -4294,18 +3400,18 @@ static void owngil_context_shutdown(py_context_t *ctx) { struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += OWNGIL_SHUTDOWN_TIMEOUT_SECS; - int rc = pthread_timedjoin_np(ctx->worker_thread, NULL, &deadline); + int rc = pthread_timedjoin_np(ctx->thread, NULL, &deadline); join_succeeded = (rc == 0); #else - /* macOS/other: poll worker_running flag with timeout */ + /* macOS/other: poll thread_running flag with timeout */ int wait_ms = 0; - while (atomic_load(&ctx->worker_running) && + while (atomic_load(&ctx->thread_running) && wait_ms < OWNGIL_SHUTDOWN_TIMEOUT_SECS * 1000) { usleep(100000); /* 100ms */ wait_ms += 100; } - if (!atomic_load(&ctx->worker_running)) { - pthread_join(ctx->worker_thread, NULL); + if (!atomic_load(&ctx->thread_running)) { + pthread_join(ctx->thread, NULL); join_succeeded = true; } #endif @@ -4313,7 +3419,7 @@ static void owngil_context_shutdown(py_context_t *ctx) { if (!join_succeeded) { /* Worker thread is unresponsive - leak the context. Pin the * resource so the BEAM doesn't free its memory under the - * stuck pthread (UAF). See worker_context_shutdown for the + * stuck pthread (UAF). See ctx_thread_shutdown_worker for the * full rationale. */ fprintf(stderr, "OWN_GIL shutdown timeout after %d seconds, leaking context\n", OWNGIL_SHUTDOWN_TIMEOUT_SECS); @@ -4401,7 +3507,7 @@ static ERL_NIF_TERM nif_context_create(ErlNifEnv *env, int argc, const ERL_NIF_T ctx->globals = NULL; ctx->locals = NULL; ctx->module_cache = NULL; - ctx->uses_worker_thread = false; + ctx->has_thread = false; /* Interrupt support */ ctx->interrupt_mutex_init = (pthread_mutex_init(&ctx->interrupt_mutex, NULL) == 0); @@ -4640,7 +3746,7 @@ static ERL_NIF_TERM nif_context_destroy(ErlNifEnv *env, int argc, const ERL_NIF_ #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: shutdown the dedicated thread */ if (ctx->uses_own_gil) { - owngil_context_shutdown(ctx); + ctx_thread_shutdown_owngil(ctx); /* Close callback pipes only on a clean shutdown. If the * worker timed out (ctx->leaked == true) it may still write * to / read from these fds; closing them here would let the @@ -4662,8 +3768,8 @@ static ERL_NIF_TERM nif_context_destroy(ErlNifEnv *env, int argc, const ERL_NIF_ #endif /* Worker mode: shutdown the dedicated worker thread */ - if (ctx->uses_worker_thread) { - worker_context_shutdown(ctx); + if (ctx->has_thread) { + ctx_thread_shutdown_worker(ctx); /* Close callback pipes (see OWN_GIL branch for why this is * gated on !ctx->leaked). */ if (!atomic_load(&ctx->leaked)) { @@ -4736,36 +3842,15 @@ static ERL_NIF_TERM nif_context_call(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_CALL, request); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_worker_thread(env, ctx, CTX_REQ_CALL, request); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + /* Request tuple: {Module, Func, Args, Kwargs} */ + ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) + ? argv[4] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple4(env, argv[1], argv[2], argv[3], kwargs); + return ctx_dispatch(env, ctx, CTX_REQ_CALL, request, NULL); } /** @@ -4810,12 +3895,12 @@ static ERL_NIF_TERM nif_context_call_async(ErlNifEnv *env, int argc, const ERL_N argv[4], /* Func */ argv[5], /* Args */ kwargs); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_CALL, + return ctx_dispatch_async(env, ctx, CTX_REQ_CALL, request, caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4856,12 +3941,12 @@ static ERL_NIF_TERM nif_context_eval_async(ErlNifEnv *env, int argc, const ERL_N ERL_NIF_TERM locals = (argc > 4 && enif_is_map(env, argv[4])) ? argv[4] : enif_make_new_map(env); ERL_NIF_TERM request = enif_make_tuple2(env, argv[3], locals); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EVAL, + return ctx_dispatch_async(env, ctx, CTX_REQ_EVAL, request, caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4898,12 +3983,12 @@ static ERL_NIF_TERM nif_context_exec_async(ErlNifEnv *env, int argc, const ERL_N /* Dedicated thread (worker or OWN_GIL): dispatch async */ if (ctx_uses_async_thread(ctx)) { - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EXEC, + return ctx_dispatch_async(env, ctx, CTX_REQ_EXEC, argv[3], caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4941,7 +4026,7 @@ static ERL_NIF_TERM nif_context_call_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } ERL_NIF_TERM kwargs = enif_is_map(env, argv[6]) @@ -4951,7 +4036,7 @@ static ERL_NIF_TERM nif_context_call_with_env_async(ErlNifEnv *env, int argc, argv[4], /* Func */ argv[5], /* Args */ kwargs); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_CALL_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_CALL_WITH_ENV, request, caller_pid, request_id, penv); } @@ -4986,13 +4071,13 @@ static ERL_NIF_TERM nif_context_eval_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } ERL_NIF_TERM locals = enif_is_map(env, argv[4]) ? argv[4] : enif_make_new_map(env); ERL_NIF_TERM request = enif_make_tuple2(env, argv[3], locals); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EVAL_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, caller_pid, request_id, penv); } @@ -5027,10 +4112,10 @@ static ERL_NIF_TERM nif_context_exec_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EXEC_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[3], caller_pid, request_id, penv); } @@ -5055,28 +4140,15 @@ static ERL_NIF_TERM nif_context_eval(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_EVAL, request); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_worker_thread(env, ctx, CTX_REQ_EVAL, request); - } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); + } + /* Request tuple: {Code, Locals} */ + ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) + ? argv[2] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); + return ctx_dispatch(env, ctx, CTX_REQ_EVAL, request, NULL); } /** @@ -5098,20 +4170,11 @@ static ERL_NIF_TERM nif_context_exec(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_EXEC, argv[1]); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - return dispatch_to_worker_thread(env, ctx, CTX_REQ_EXEC, argv[1]); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + return ctx_dispatch(env, ctx, CTX_REQ_EXEC, argv[1], NULL); } /* ============================================================================ @@ -5152,79 +4215,32 @@ static ERL_NIF_TERM nif_create_local_env(ErlNifEnv *env, int argc, const ERL_NIF res->locals = NULL; res->interp_id = 0; -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread to create dicts */ - if (ctx->uses_own_gil) { - ERL_NIF_TERM dispatch_result = dispatch_create_local_env_to_owngil(env, ctx, res); - - /* Check if dispatch succeeded */ - ERL_NIF_TERM error_atom = enif_make_atom(env, "error"); - const ERL_NIF_TERM *tuple_elems; - int arity; - if (enif_get_tuple(env, dispatch_result, &arity, &tuple_elems) && - arity == 2 && enif_is_identical(tuple_elems[0], error_atom)) { - /* Dispatch failed - release resource and return error */ - enif_release_resource(res); - return dispatch_result; - } - - /* Success - return the resource */ - ERL_NIF_TERM ref = enif_make_resource(env, res); - enif_release_resource(res); /* Ref now owns it */ - return enif_make_tuple2(env, ATOM_OK, ref); - } -#endif - - /* Acquire context to switch to correct interpreter */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { + if (!ctx_uses_async_thread(ctx)) { enif_release_resource(res); - return make_error(env, "acquire_failed"); + return make_error(env, "context_has_no_thread"); } - /* Copy globals from context to inherit preloaded code */ - res->globals = PyDict_Copy(ctx->globals); - if (res->globals == NULL) { - py_context_release(&guard); + /* The dicts are created on the context thread so they belong to the + * right interpreter and allocator. */ + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_CREATE_LOCAL_ENV, &err); + if (req == NULL) { enif_release_resource(res); - return make_error(env, "globals_copy_failed"); - } - - /* Ensure __builtins__ is present (may not be in subinterpreter mode) */ - if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) { - PyObject *builtins = PyEval_GetBuiltins(); - if (builtins != NULL) { - PyDict_SetItemString(res->globals, "__builtins__", builtins); - } - } - - /* Ensure __name__ = '__main__' is set */ - if (PyDict_GetItemString(res->globals, "__name__") == NULL) { - PyObject *main_name = PyUnicode_FromString("__main__"); - if (main_name != NULL) { - PyDict_SetItemString(res->globals, "__name__", main_name); - Py_DECREF(main_name); - } + return err; } + req->local_env_ptr = res; + ERL_NIF_TERM dispatch_result = ctx_dispatch_wait(env, ctx, req); - /* Ensure erlang module is available */ - if (PyDict_GetItemString(res->globals, "erlang") == NULL) { - PyObject *erlang = PyImport_ImportModule("erlang"); - if (erlang != NULL) { - PyDict_SetItemString(res->globals, "erlang", erlang); - Py_DECREF(erlang); - } + const ERL_NIF_TERM *tuple_elems; + int arity; + if (enif_get_tuple(env, dispatch_result, &arity, &tuple_elems) && + arity == 2 && enif_is_identical(tuple_elems[0], enif_make_atom(env, "error"))) { + enif_release_resource(res); + return dispatch_result; } - /* Use the same dict for locals (module-level execution) */ - res->locals = res->globals; - Py_INCREF(res->locals); - - py_context_release(&guard); - ERL_NIF_TERM ref = enif_make_resource(env, res); enif_release_resource(res); /* Ref now owns it */ - return enif_make_tuple2(env, ATOM_OK, ref); } @@ -5257,60 +4273,11 @@ static ERL_NIF_TERM nif_interp_apply_imports(ErlNifEnv *env, int argc, const ERL return make_error(env, "context_destroyed"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_apply_imports_to_owngil(env, ctx, argv[1]); - } -#endif - - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Process each import - imports go into interpreter's sys.modules */ - ERL_NIF_TERM head, tail = argv[1]; - int arity; - const ERL_NIF_TERM *tuple; - - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (!enif_get_tuple(env, head, &arity, &tuple) || arity != 2) { - continue; - } - - ErlNifBinary module_bin; - if (!enif_inspect_binary(env, tuple[0], &module_bin)) { - continue; - } - - /* Convert to C string */ - char *module_name = enif_alloc(module_bin.size + 1); - if (module_name == NULL) continue; - memcpy(module_name, module_bin.data, module_bin.size); - module_name[module_bin.size] = '\0'; - - /* Skip __main__ */ - if (strcmp(module_name, "__main__") == 0) { - enif_free(module_name); - continue; - } - - /* Import the module - this caches in interpreter's sys.modules - * which is shared by all contexts using this interpreter */ - PyObject *mod = PyImport_ImportModule(module_name); - if (mod != NULL) { - Py_DECREF(mod); /* sys.modules holds the reference */ - } else { - /* Clear error - import failure is not fatal */ - PyErr_Clear(); - } - - enif_free(module_name); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - py_context_release(&guard); - return ATOM_OK; + return ctx_dispatch(env, ctx, CTX_REQ_APPLY_IMPORTS, argv[1], NULL); } /** @@ -5337,78 +4304,11 @@ static ERL_NIF_TERM nif_interp_apply_paths(ErlNifEnv *env, int argc, const ERL_N return make_error(env, "context_destroyed"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_apply_paths_to_owngil(env, ctx, argv[1]); - } -#endif - - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Get sys.path */ - PyObject *sys_module = PyImport_ImportModule("sys"); - if (sys_module == NULL) { - py_context_release(&guard); - return make_error(env, "sys_import_failed"); - } - - PyObject *sys_path = PyObject_GetAttrString(sys_module, "path"); - Py_DECREF(sys_module); - if (sys_path == NULL || !PyList_Check(sys_path)) { - Py_XDECREF(sys_path); - py_context_release(&guard); - return make_error(env, "sys_path_not_list"); - } - - /* Process each path - insert at beginning in reverse order */ - /* First, collect all paths */ - ERL_NIF_TERM head, tail = argv[1]; - int path_count = 0; - ERL_NIF_TERM paths_list = argv[1]; - - /* Count paths */ - while (enif_get_list_cell(env, tail, &head, &tail)) { - path_count++; - } - - /* Insert in reverse order so first path ends up first */ - tail = paths_list; - for (int i = 0; i < path_count; i++) { - /* Skip to the i-th element from the end */ - ERL_NIF_TERM current = paths_list; - for (int j = 0; j < path_count - 1 - i; j++) { - enif_get_list_cell(env, current, &head, ¤t); - } - enif_get_list_cell(env, current, &head, ¤t); - - ErlNifBinary path_bin; - if (!enif_inspect_binary(env, head, &path_bin)) { - continue; - } - - /* Convert to Python string */ - PyObject *path_str = PyUnicode_FromStringAndSize((char *)path_bin.data, path_bin.size); - if (path_str == NULL) { - PyErr_Clear(); - continue; - } - - /* Check if already in sys.path */ - int already_present = PySequence_Contains(sys_path, path_str); - if (already_present <= 0) { - /* Insert at position 0 */ - PyList_Insert(sys_path, 0, path_str); - } - Py_DECREF(path_str); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - Py_DECREF(sys_path); - py_context_release(&guard); - return ATOM_OK; + return ctx_dispatch(env, ctx, CTX_REQ_APPLY_PATHS, argv[1], NULL); } /** @@ -5428,79 +4328,29 @@ static ERL_NIF_TERM nif_context_exec_with_env(ErlNifEnv *env, int argc, const ER py_context_t *ctx; py_env_resource_t *penv; - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) { - return make_error(env, "invalid_context"); - } - - ErlNifBinary code_bin; - if (!enif_inspect_binary(env, argv[1], &code_bin)) { - return make_error(env, "invalid_code"); - } - - /* Get process-local environment */ - if (!enif_get_resource(env, argv[2], PY_ENV_RESOURCE_TYPE, (void **)&penv)) { - return make_error(env, "invalid_env"); - } - -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_exec_with_env_to_owngil(env, ctx, argv[1], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* For exec, we just pass the code binary */ - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[1], penv); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Always use process-local environment */ - PyObject *exec_globals = penv->globals; - PyObject *exec_locals = penv->globals; - - /* Execute statements */ - PyObject *py_result = PyRun_String(code, Py_file_input, exec_globals, exec_locals); + if (!runtime_is_running()) { + return make_error(env, "python_not_running"); + } - if (py_result == NULL) { - result = make_py_error(env); - } else { - Py_DECREF(py_result); - result = ATOM_OK; + if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) { + return make_error(env, "invalid_context"); } - /* Restore thread-local state */ - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; + ErlNifBinary code_bin; + if (!enif_inspect_binary(env, argv[1], &code_bin)) { + return make_error(env, "invalid_code"); + } - enif_free(code); - py_context_release(&guard); + /* Get process-local environment */ + if (!enif_get_resource(env, argv[2], PY_ENV_RESOURCE_TYPE, (void **)&penv)) { + return make_error(env, "invalid_env"); + } - return result; + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); + } + return ctx_dispatch(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[1], penv); } /** @@ -5534,135 +4384,14 @@ static ERL_NIF_TERM nif_context_eval_with_env(ErlNifEnv *env, int argc, const ER return make_error(env, "invalid_env"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_eval_with_env_to_owngil(env, ctx, argv[1], argv[2], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, penv); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Always use process-local environment */ - PyObject *eval_globals = penv->globals; - - /* Build locals dict from Erlang map (if provided) */ - PyObject *eval_locals = PyDict_Copy(eval_globals); - if (enif_is_map(env, argv[2])) { - ErlNifMapIterator iter; - ERL_NIF_TERM key, value; - - enif_map_iterator_create(env, argv[2], &iter, ERL_NIF_MAP_ITERATOR_FIRST); - while (enif_map_iterator_get_pair(env, &iter, &key, &value)) { - PyObject *py_key = term_to_py(env, key); - PyObject *py_value = term_to_py(env, value); - if (py_key != NULL && py_value != NULL) { - PyDict_SetItem(eval_locals, py_key, py_value); - } - Py_XDECREF(py_key); - Py_XDECREF(py_value); - enif_map_iterator_next(env, &iter); - } - enif_map_iterator_destroy(env, &iter); - } - - /* Evaluate expression */ - PyObject *py_result = PyRun_String(code, Py_eval_input, eval_globals, eval_locals); - Py_DECREF(eval_locals); - - if (py_result == NULL) { - /* Check for pending callback (flag-based detection) */ - if (tl_pending_callback) { - PyErr_Clear(); - /* Create suspended state for callback handling */ - suspended_context_state_t *suspended = create_suspended_context_state_for_eval( - env, ctx, &code_bin, argv[2]); - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif with local_env */ - inline_continuation_t *cont = create_inline_continuation(ctx, penv, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - clear_pending_callback_tls(); - enif_free(code); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - - clear_pending_callback_tls(); - enif_free(code); - py_context_release(&guard); - - return result; + ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) + ? argv[2] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); + return ctx_dispatch(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, penv); } /** @@ -5701,187 +4430,14 @@ static ERL_NIF_TERM nif_context_call_with_env(ErlNifEnv *env, int argc, const ER return make_error(env, "invalid_env"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_call_with_env_to_owngil(env, ctx, argv[1], argv[2], argv[3], argv[4], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_CALL_WITH_ENV, request, penv); - } - - char *module_name = binary_to_string(&module_bin); - char *func_name = binary_to_string(&func_bin); - if (module_name == NULL || func_name == NULL) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Always use process-local environment */ - PyObject *lookup_globals = penv->globals; - - PyObject *module = NULL; - PyObject *func = NULL; - - /* Special handling for __main__ module - look up in process-local globals */ - if (strcmp(module_name, "__main__") == 0) { - func = PyDict_GetItemString(lookup_globals, func_name); /* Borrowed ref */ - if (func != NULL) { - Py_INCREF(func); - } - } - - if (func == NULL) { - /* Get or import module from context cache */ - module = context_get_module(ctx, module_name); - if (module == NULL) { - result = make_py_error(env); - goto cleanup; - } - - /* Get function */ - func = PyObject_GetAttrString(module, func_name); - if (func == NULL) { - result = make_py_error(env); - goto cleanup; - } - } - - /* Convert args */ - unsigned int args_len; - if (!enif_get_list_length(env, argv[3], &args_len)) { - Py_DECREF(func); - result = make_error(env, "invalid_args"); - goto cleanup; - } - - PyObject *args = PyTuple_New(args_len); - if (args == NULL) { - Py_DECREF(func); - result = make_error(env, "alloc_failed"); - goto cleanup; - } - ERL_NIF_TERM head, tail = argv[3]; - for (unsigned int i = 0; i < args_len; i++) { - enif_get_list_cell(env, tail, &head, &tail); - PyObject *arg = term_to_py(env, head); - if (arg == NULL) { - Py_DECREF(args); - Py_DECREF(func); - result = make_error(env, "arg_conversion_failed"); - goto cleanup; - } - PyTuple_SET_ITEM(args, i, arg); - } - - /* Convert kwargs */ - PyObject *kwargs = NULL; - if (argc > 4 && enif_is_map(env, argv[4])) { - kwargs = term_to_py(env, argv[4]); - } - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (py_result == NULL) { - /* Check for pending callback */ - if (tl_pending_callback) { - PyErr_Clear(); - suspended_context_state_t *suspended = create_suspended_context_state_for_call( - env, ctx, &module_bin, &func_bin, argv[3], - argc > 4 ? argv[4] : enif_make_new_map(env)); - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif with local_env */ - inline_continuation_t *cont = create_inline_continuation(ctx, penv, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - clear_pending_callback_tls(); - enif_free(module_name); - enif_free(func_name); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - -cleanup: - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - - clear_pending_callback_tls(); - enif_free(module_name); - enif_free(func_name); - py_context_release(&guard); - - return result; + ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) + ? argv[4] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple4(env, argv[1], argv[2], argv[3], kwargs); + return ctx_dispatch(env, ctx, CTX_REQ_CALL_WITH_ENV, request, penv); } /** @@ -7330,20 +5886,11 @@ static ERL_NIF_TERM nif_os_kill(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg } static ErlNifFunc nif_funcs[] = { - /* Initialization */ + /* py_nif.c: runtime, contexts, process-local envs, py_ref */ {"init", 0, nif_py_init, 0}, {"init", 1, nif_py_init, 0}, {"finalize", 0, nif_finalize, 0}, - - - /* Python execution - dirty I/O NIFs */ - - /* Module operations */ - - /* Info */ {"version", 0, nif_version, 0}, - - /* Memory and GC */ {"memory_stats", 0, nif_memory_stats, 0}, {"get_debug_counters", 0, nif_get_debug_counters, 0}, {"gc", 0, nif_gc, 0}, @@ -7351,123 +5898,20 @@ static ErlNifFunc nif_funcs[] = { {"tracemalloc_start", 0, nif_tracemalloc_start, 0}, {"tracemalloc_start", 1, nif_tracemalloc_start, 0}, {"tracemalloc_stop", 0, nif_tracemalloc_stop, 0}, - - /* Callback support */ - - /* Async worker management */ - - /* Async execution - dirty I/O NIFs */ - - /* Subinterpreter capability probes */ {"subinterp_supported", 0, nif_subinterp_supported, 0}, {"owngil_supported", 0, nif_owngil_supported, 0}, - - /* OWN_GIL thread pool (used internally by py_event_loop_pool) */ {"subinterp_thread_pool_start", 0, nif_subinterp_thread_pool_start, 0}, {"subinterp_thread_pool_start", 1, nif_subinterp_thread_pool_start, 0}, {"subinterp_thread_pool_stop", 0, nif_subinterp_thread_pool_stop, 0}, {"subinterp_thread_pool_ready", 0, nif_subinterp_thread_pool_ready, 0}, {"subinterp_thread_pool_stats", 0, nif_subinterp_thread_pool_stats, 0}, - - /* OWN_GIL session management for event loop pool */ {"owngil_create_session", 1, nif_owngil_create_session, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_submit_task", 7, nif_owngil_submit_task, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_destroy_session", 2, nif_owngil_destroy_session, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_apply_imports", 3, nif_owngil_apply_imports, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_apply_paths", 3, nif_owngil_apply_paths, ERL_NIF_DIRTY_JOB_IO_BOUND}, - - /* Execution mode info */ {"execution_mode", 0, nif_execution_mode, 0}, - - /* Thread worker support (ThreadPoolExecutor). - * Writes are ERL_NIF_DIRTY_JOB_IO_BOUND because the response pipe - * has a non-blocking write end and the looped write may briefly - * wait for write-readiness when the Python reader is slow. */ - {"thread_worker_set_coordinator", 1, nif_thread_worker_set_coordinator, 0}, - {"thread_worker_write_with_id", 3, nif_thread_worker_write_with_id, - ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"thread_worker_signal_ready", 1, nif_thread_worker_signal_ready, 0}, - - /* Async callback support (for erlang.async_call). Same dirty-IO - * rationale as thread_worker_write_with_id above. */ - {"async_callback_response", 3, nif_async_callback_response, - ERL_NIF_DIRTY_JOB_IO_BOUND}, - - /* Callback name registry (prevents torch introspection issues) */ - {"register_callback_name", 1, nif_register_callback_name, 0}, - {"unregister_callback_name", 1, nif_unregister_callback_name, 0}, - - /* Logging and tracing */ - {"set_log_receiver", 2, nif_set_log_receiver, 0}, - {"clear_log_receiver", 0, nif_clear_log_receiver, 0}, - {"set_trace_receiver", 1, nif_set_trace_receiver, 0}, - {"clear_trace_receiver", 0, nif_clear_trace_receiver, 0}, - - /* Erlang-native event loop NIFs */ - {"set_event_loop_priv_dir", 1, nif_set_event_loop_priv_dir, 0}, - {"event_loop_new", 0, nif_event_loop_new, 0}, - {"event_loop_destroy", 1, nif_event_loop_destroy, 0}, - {"event_loop_set_router", 2, nif_event_loop_set_router, 0}, - {"event_loop_set_worker", 2, nif_event_loop_set_worker, 0}, - {"event_loop_set_id", 2, nif_event_loop_set_id, 0}, - {"event_loop_wakeup", 1, nif_event_loop_wakeup, 0}, - {"event_loop_run_async", 7, nif_event_loop_run_async, ERL_NIF_DIRTY_JOB_IO_BOUND}, - /* Async task queue NIFs (uvloop-inspired) */ - {"submit_task", 7, nif_submit_task, 0}, /* Thread-safe, no GIL needed */ - {"submit_task_with_env", 8, nif_submit_task_with_env, 0}, /* With process-local env */ - {"process_ready_tasks", 1, nif_process_ready_tasks, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"event_loop_set_py_loop", 2, nif_event_loop_set_py_loop, 0}, - /* Per-process namespace NIFs */ - {"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"add_reader", 3, nif_add_reader, 0}, - {"remove_reader", 2, nif_remove_reader, 0}, - {"add_writer", 3, nif_add_writer, 0}, - {"remove_writer", 2, nif_remove_writer, 0}, - {"call_later", 3, nif_call_later, 0}, - {"cancel_timer", 2, nif_cancel_timer, 0}, - {"poll_events", 2, nif_poll_events, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"get_pending", 1, nif_get_pending, 0}, - {"dispatch_callback", 3, nif_dispatch_callback, 0}, - {"dispatch_timer", 2, nif_dispatch_timer, 0}, - {"get_fd_callback_id", 2, nif_get_fd_callback_id, 0}, - {"reselect_reader", 2, nif_reselect_reader, 0}, - {"reselect_writer", 2, nif_reselect_writer, 0}, - {"reselect_reader_fd", 1, nif_reselect_reader_fd, 0}, - {"reselect_writer_fd", 1, nif_reselect_writer_fd, 0}, - /* FD lifecycle management (uvloop-like API) */ - {"handle_fd_event", 2, nif_handle_fd_event, 0}, - {"handle_fd_event_and_reselect", 2, nif_handle_fd_event_and_reselect, 0}, - {"fd_arm", 2, nif_fd_arm, 0}, - {"stop_reader", 1, nif_stop_reader, 0}, - {"start_reader", 1, nif_start_reader, 0}, - {"stop_writer", 1, nif_stop_writer, 0}, - {"start_writer", 1, nif_start_writer, 0}, - {"close_fd", 1, nif_close_fd, 0}, - /* Test helpers for fd monitoring (using pipes) */ - {"create_test_pipe", 0, nif_create_test_pipe, 0}, - {"close_test_fd", 1, nif_close_test_fd, 0}, - {"dup_fd", 1, nif_dup_fd, 0}, {"os_kill", 2, nif_os_kill, 0}, - {"write_test_fd", 2, nif_write_test_fd, 0}, - {"read_test_fd", 2, nif_read_test_fd, 0}, - /* TCP test helpers */ - {"create_test_tcp_listener", 1, nif_create_test_tcp_listener, 0}, - {"accept_test_tcp", 1, nif_accept_test_tcp, 0}, - {"connect_test_tcp", 2, nif_connect_test_tcp, 0}, - /* UDP test helpers */ - {"create_test_udp_socket", 1, nif_create_test_udp_socket, 0}, - {"recvfrom_test_udp", 2, nif_recvfrom_test_udp, 0}, - {"sendto_test_udp", 4, nif_sendto_test_udp, 0}, - {"set_udp_broadcast", 2, nif_set_udp_broadcast, 0}, - /* Python event loop integration */ - {"set_python_event_loop", 1, nif_set_python_event_loop, 0}, - {"set_isolation_mode", 1, nif_set_isolation_mode, 0}, - {"set_shared_worker", 1, nif_set_shared_worker, 0}, - - /* Worker pool */ - - /* Process-per-context API (no mutex) */ {"context_create", 1, nif_context_create, 0}, {"context_destroy", 1, nif_context_destroy, 0}, {"context_interrupt", 1, nif_context_interrupt, ERL_NIF_DIRTY_JOB_IO_BOUND}, @@ -7479,7 +5923,6 @@ static ErlNifFunc nif_funcs[] = { {"context_exec", 3, nif_context_exec_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_eval", 4, nif_context_eval_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_call", 6, nif_context_call_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - /* Async dispatch - non-blocking, returns immediately */ {"context_call_async", 7, nif_context_call_async, 0}, {"context_eval_async", 5, nif_context_eval_async, 0}, {"context_exec_async", 4, nif_context_exec_async, 0}, @@ -7497,9 +5940,6 @@ static ErlNifFunc nif_funcs[] = { {"context_write_callback_response", 2, nif_context_write_callback_response, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"context_resume", 3, nif_context_resume, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_cancel_resume", 2, nif_context_cancel_resume, 0}, - {"context_get_event_loop", 1, nif_context_get_event_loop, 0}, - - /* py_ref API (Python object references with interp_id) */ {"ref_wrap", 2, nif_ref_wrap, 0}, {"is_ref", 1, nif_is_ref, 0}, {"ref_interp_id", 1, nif_ref_interp_id, 0}, @@ -7507,54 +5947,14 @@ static ErlNifFunc nif_funcs[] = { {"ref_getattr", 2, nif_ref_getattr, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"ref_call_method", 3, nif_ref_call_method, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - /* Reactor NIFs - Erlang-as-Reactor architecture */ - {"reactor_register_fd", 3, nif_reactor_register_fd, 0}, - {"reactor_reselect_read", 1, nif_reactor_reselect_read, 0}, - {"reactor_select_write", 1, nif_reactor_select_write, 0}, - {"get_fd_from_resource", 1, nif_get_fd_from_resource, 0}, - {"reactor_on_read_ready", 2, nif_reactor_on_read_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_on_write_ready", 2, nif_reactor_on_write_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_init_connection", 3, nif_reactor_init_connection, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_close_fd", 2, nif_reactor_close_fd, 0}, - - /* Direct FD operations */ - {"fd_read", 2, nif_fd_read, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"fd_write", 2, nif_fd_write, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"fd_select_read", 1, nif_fd_select_read, 0}, - {"fd_select_write", 1, nif_fd_select_write, 0}, - {"fd_close", 1, nif_fd_close, 0}, - {"socketpair", 0, nif_socketpair, 0}, - - /* Channel API - bidirectional message passing */ - {"channel_create", 0, nif_channel_create, 0}, - {"channel_create", 1, nif_channel_create, 0}, - {"channel_send", 2, nif_channel_send, 0}, - {"channel_receive", 2, nif_channel_receive, 0}, - {"channel_try_receive", 1, nif_channel_try_receive, 0}, - {"channel_reply", 3, nif_channel_reply, 0}, - {"channel_close", 1, nif_channel_close, 0}, - {"channel_info", 1, nif_channel_info, 0}, - {"channel_wait", 3, nif_channel_wait, 0}, - {"channel_cancel_wait", 2, nif_channel_cancel_wait, 0}, - {"channel_register_sync_waiter", 1, nif_channel_register_sync_waiter, 0}, - - /* ByteChannel API - raw bytes, no term conversion */ - {"byte_channel_send_bytes", 2, nif_byte_channel_send_bytes, 0}, - {"byte_channel_try_receive_bytes", 1, nif_byte_channel_try_receive_bytes, 0}, - {"byte_channel_wait_bytes", 3, nif_byte_channel_wait_bytes, 0}, - - /* PyBuffer API - zero-copy input */ - {"py_buffer_create", 1, nif_py_buffer_create, 0}, - {"py_buffer_write", 2, nif_py_buffer_write, 0}, - {"py_buffer_close", 1, nif_py_buffer_close, 0}, - - /* SharedDict API - process-scoped shared dictionary */ - {"shared_dict_new", 0, nif_shared_dict_new, 0}, - {"shared_dict_get", 3, nif_shared_dict_get, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_set", 3, nif_shared_dict_set, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_del", 2, nif_shared_dict_del, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_keys", 1, nif_shared_dict_keys, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_destroy", 1, nif_shared_dict_destroy, 0} + /* One macro per area, defined at the end of the file that owns it */ + PY_CALLBACK_NIFS, + PY_THREAD_WORKER_NIFS, + PY_LOGGING_NIFS, + PY_EVENT_LOOP_NIFS, + PY_CHANNEL_NIFS, + PY_BUFFER_NIFS, + PY_SHARED_DICT_NIFS }; ERL_NIF_INIT(py_nif, nif_funcs, load, NULL, upgrade, unload) diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 6832c40..5dab26c 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -645,21 +645,21 @@ typedef struct { * @brief One Python execution environment served by one Erlang process * * A context has exactly one pthread that runs Python for it: the context - * thread (worker_context_thread_main for worker mode, - * owngil_context_thread_main for owngil mode). Erlang processes never run + * thread (ctx_thread_main_worker for worker mode, + * ctx_thread_main_owngil for owngil mode). Erlang processes never run * Python on a context; NIFs enqueue a ctx_request_t and return, the * context thread dequeues, executes and replies with `{py_result, Id, R}` * through msg_env. Isolated mode does not use this struct at all. * * Lock and ownership contract, by field group: * - * - Identity and lifecycle (interp_id, is_subinterp, uses_worker_thread, + * - Identity and lifecycle (interp_id, is_subinterp, has_thread, * uses_own_gil): written once by nif_context_create before the thread - * starts, read-only afterwards. destroyed, leaked, worker_running, + * starts, read-only afterwards. destroyed, leaked, thread_running, * shutdown_requested and init_error are atomics; any thread may read * them, the writers are nif_context_destroy (destroyed, leaked), the * shutdown helpers (shutdown_requested) and the context thread - * (worker_running, init_error). + * (thread_running, init_error). * * - Callback handler (has_callback_handler, callback_handler, * callback_pipe): set by the owning Erlang process through @@ -734,16 +734,16 @@ struct py_context { /* ========== Context thread (worker and owngil modes) ========== */ /** @brief Dedicated pthread for this context */ - pthread_t worker_thread; + pthread_t thread; /** @brief True when worker thread is running */ - _Atomic bool worker_running; + _Atomic bool thread_running; /** @brief True when shutdown has been requested */ _Atomic bool shutdown_requested; /** @brief True if this context uses a dedicated worker thread (worker mode) */ - bool uses_worker_thread; + bool has_thread; /** @brief True if thread initialization failed */ _Atomic bool init_error; @@ -2050,40 +2050,13 @@ static inline void log_and_clear_python_error(const char *context) { #ifdef HAVE_SUBINTERPRETERS -/** - * @brief Dispatch reactor on_read_ready to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @param buffer_ptr Reactor buffer resource (ownership transferred) - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_read_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, void *buffer_ptr); - -/** - * @brief Dispatch reactor on_write_ready to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_write_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd); - -/** - * @brief Dispatch reactor init_connection to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @param client_info Client info map term - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_init_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, ERL_NIF_TERM client_info); +/* Reactor callbacks run on the context thread through the request queue + * (ctx_dispatch_wait in py_nif.c); py_event_loop.c calls these. */ +ERL_NIF_TERM dispatch_reactor_read(ErlNifEnv *env, py_context_t *ctx, + int fd, void *buffer_ptr); +ERL_NIF_TERM dispatch_reactor_write(ErlNifEnv *env, py_context_t *ctx, int fd); +ERL_NIF_TERM dispatch_reactor_init(ErlNifEnv *env, py_context_t *ctx, + int fd, ERL_NIF_TERM client_info); #endif /* HAVE_SUBINTERPRETERS */ diff --git a/c_src/py_shared_dict.c b/c_src/py_shared_dict.c index a55c3d0..265b860 100644 --- a/c_src/py_shared_dict.c +++ b/c_src/py_shared_dict.c @@ -813,3 +813,13 @@ static PyObject *py_shared_dict_keys_impl(PyObject *self, PyObject *args) { pthread_mutex_unlock(&sd->mutex); return result; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_SHARED_DICT_NIFS \ + {"shared_dict_new", 0, nif_shared_dict_new, 0}, \ + {"shared_dict_get", 3, nif_shared_dict_get, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_set", 3, nif_shared_dict_set, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_del", 2, nif_shared_dict_del, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_keys", 1, nif_shared_dict_keys, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_destroy", 1, nif_shared_dict_destroy, 0} diff --git a/c_src/py_thread_worker.c b/c_src/py_thread_worker.c index 31fde37..7656583 100644 --- a/c_src/py_thread_worker.c +++ b/c_src/py_thread_worker.c @@ -844,3 +844,11 @@ static ERL_NIF_TERM nif_async_callback_response(ErlNifEnv *env, int argc, } return make_error(env, "write_failed"); } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_THREAD_WORKER_NIFS \ + {"thread_worker_set_coordinator", 1, nif_thread_worker_set_coordinator, 0}, \ + {"thread_worker_write_with_id", 3, nif_thread_worker_write_with_id, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"thread_worker_signal_ready", 1, nif_thread_worker_signal_ready, 0}, \ + {"async_callback_response", 3, nif_async_callback_response, ERL_NIF_DIRTY_JOB_IO_BOUND} diff --git a/docs/architecture.md b/docs/architecture.md index 7c30bf3..3ae029f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,7 @@ one Python execution environment and serves calls in order. Pools | | `worker` | `owngil` | `isolated` | |---|---|---|---| | Python runs in | the VM, main interpreter | the VM, a sub-interpreter with its own GIL | a child process | -| Thread | one pthread per context (`worker_context_thread_main`) | one pthread per context (`owngil_context_thread_main`) | the child's main thread | +| Thread | one pthread per context (`ctx_thread_main_worker`) | one pthread per context (`ctx_thread_main_owngil`) | the child's main thread | | Erlang process loop | the receive loop in `py_context` | the receive loop in `py_context` | `py_isolated` (`gen_statem`) | | Transport | NIF request queue on `py_context_t` | same | Unix socket, frames of the callback pipe format | | Python -> Erlang | suspension protocol | blocking callback pipe | socket frames | @@ -71,9 +71,9 @@ one Python execution environment and serves calls in order. Pools arguments (`term_to_py`, `c_src/py_convert.c`) into a request, enqueues it on the context's queue (`ctx_queue_enqueue`) and returns `{enqueued, Ref}` at once. The Erlang process is now free to serve callbacks. -3. The context's pthread (`worker_context_thread_main` or - `owngil_context_thread_main`, `c_src/py_nif.c`) dequeues the request and - runs it through `owngil_execute_request` (despite its name it serves both +3. The context's pthread (`ctx_thread_main_worker` or + `ctx_thread_main_owngil`, `c_src/py_nif.c`) dequeues the request and + runs it through `ctx_execute_request` (one function for both modes), which calls into Python with the GIL held. 4. The thread converts the result (`py_to_term`) and sends `{py_result, Ref, Result}` to the `py_context` process, which replies diff --git a/docs/code-map.md b/docs/code-map.md index 5b237d8..534ce81 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -9,8 +9,12 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | Module | Owns | Status | Guide | Suites | |---|---|---|---|---| -| `py` | Public API facade: call/eval/exec, streams, async helpers, venvs, memory, function registration | live | README, getting-started | `py_SUITE`, `py_api_SUITE`, `py_stream_SUITE`, `py_venv_SUITE` | -| `py_context` | The context process for embedded modes and the API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`); dispatch to `py_isolated` for isolated mode | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py` | Public API facade: call/eval/exec, async helpers, memory, function registration; delegates streams, venvs and shared dicts | live | README, getting-started | `py_SUITE`, `py_api_SUITE` | +| `py_stream` | Generator streaming behind `py:stream*` | live | streaming | `py_stream_SUITE` | +| `py_venv` | Virtual environments behind `py:ensure_venv` and friends | live | README (venvs) | `py_venv_SUITE` | +| `py_shared_dict` | `py:shared_dict_*` over the shared dict NIFs | live | shared-dict | `py_SUITE` | +| `py_context` | The API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`), the reply protocol and the pid to NIF reference table; `init/4` hands the process to `py_context_embedded` or `py_isolated` | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py_context_embedded` | Process body for `worker` and `owngil` mode: the receive loop, callbacks (suspension and pipe), worker loops | live | architecture, state-machines | same | | `py_isolated` | `gen_statem` driving a child process over the socket; restart policy | live | isolated | `py_isolated_*_SUITE` | | `py_context_router` | Pools and scheduler-affinity routing | live | pools, context-affinity | `py_context_router_SUITE`, `py_pool_SUITE` | | `py_context_sup`, `py_context_init` | Supervisor of contexts; starts the default pool at boot | live | pools | (through the above) | diff --git a/docs/contributing.md b/docs/contributing.md index 52c1226..756ed2e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -111,8 +111,9 @@ unreleased version. 1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` in the `c_src` file that owns the area (see `c_src/README.md`). -2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of - `c_src/py_nif.c`. `Flags` is `ERL_NIF_DIRTY_JOB_CPU_BOUND` or +2. Add `{"x", Arity, nif_x, Flags}` to the `PY_*_NIFS` macro at the end of + that file (`nif_funcs[]` in `c_src/py_nif.c` concatenates them; NIFs + that live in `py_nif.c` go in its own block there). `Flags` is `ERL_NIF_DIRTY_JOB_CPU_BOUND` or `ERL_NIF_DIRTY_JOB_IO_BOUND` when the NIF can block or run Python, `0` otherwise. 3. Add the stub, its `-spec` and a `@doc` to `src/py_nif.erl`, and the diff --git a/docs/decisions/0001-one-thread-per-context.md b/docs/decisions/0001-one-thread-per-context.md index 33fe2bc..45618ed 100644 --- a/docs/decisions/0001-one-thread-per-context.md +++ b/docs/decisions/0001-one-thread-per-context.md @@ -1,6 +1,6 @@ # 0001: One pthread per context, NIFs only enqueue -Since 3.0.0. Code: `worker_context_thread_main`, `owngil_context_thread_main`, +Since 3.0.0. Code: `ctx_thread_main_worker`, `ctx_thread_main_owngil`, the request queue on `py_context_t` (`c_src/py_nif.c`, `c_src/py_nif.h`). ## Situation diff --git a/docs/glossary.md b/docs/glossary.md index 439622e..7e675ca 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -22,7 +22,7 @@ descriptors for the reactor; "coordinator context" in C comments means the per context, shared GIL), `owngil` (a sub-interpreter with its own GIL per context, one pthread), `isolated` (a child process). `py_context:new(#{mode => ...})`. -Related flags on `py_context_t`: `uses_worker_thread` (has its own pthread; +Related flags on `py_context_t`: `has_thread` (has its own pthread; true for worker and owngil contexts created today), `is_subinterp` (has its own sub-interpreter), `uses_own_gil` (that sub-interpreter has its own GIL). `subinterp` in file and NIF names (`py_subinterp_thread.c`, @@ -39,7 +39,7 @@ The most overloaded word. Meanings, by file: | Where | Meaning | Prefer to say | |---|---|---| | `py_context:new(#{mode => worker})` | the context mode above | worker mode | -| `worker_context_thread_main`, `uses_worker_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | +| `ctx_thread_main_worker`, `has_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | | `thread_worker`, `thread_worker_call` (`py_thread_worker.c`), `py_thread_handler` | the channel a Python thread uses to call Erlang | thread callback bridge | | `py_event_worker` | the Erlang process that drives one asyncio loop (readiness, timers) | loop driver | | `docs/workers.md`, "worker loop" | a long-running asyncio loop on a context thread, gunicorn-style | worker loop | diff --git a/docs/owngil_internals.md b/docs/owngil_internals.md index 8d3b2e5..c7ae84f 100644 --- a/docs/owngil_internals.md +++ b/docs/owngil_internals.md @@ -68,7 +68,7 @@ All major erlang_python features work with OWN_GIL mode: │ │ │ │ └──────────┼───────────────────────────┼──────────────────────────────┘ │ │ - │ dispatch_to_owngil_thread │ + │ ctx_dispatch │ ▼ ▼ ┌──────────────────────┐ ┌──────────────────────┐ │ OWN_GIL Thread 1 │ │ OWN_GIL Thread 2 │ @@ -150,8 +150,8 @@ nif_context_create(env, "owngil") └── owngil_context_init(ctx) ├── Initialize mutex/condvars ├── Create shared_env - └── pthread_create(owngil_context_thread_main) - └── owngil_context_thread_main(ctx) + └── pthread_create(ctx_thread_main_owngil) + └── ctx_thread_main_owngil(ctx) ├── Py_NewInterpreterFromConfig(OWN_GIL) ├── Initialize globals/locals ├── Register py_event_loop module @@ -164,7 +164,7 @@ nif_context_create(env, "owngil") nif_context_call(env, ctx, module, func, args, kwargs) │ ├── [ctx->uses_own_gil == true] - │ └── dispatch_to_owngil_thread(env, ctx, CTX_REQ_CALL, request) + │ └── ctx_dispatch(env, ctx, CTX_REQ_CALL, request, NULL) │ ├── pthread_mutex_lock(&ctx->request_mutex) │ ├── Copy request term to shared_env │ ├── Set ctx->request_type = CTX_REQ_CALL @@ -180,15 +180,15 @@ nif_context_call(env, ctx, module, func, args, kwargs) ### 3. Request Processing (OWN_GIL Thread) ``` -owngil_context_thread_main(ctx) +ctx_thread_main_owngil(ctx) while (!shutdown_requested) { pthread_cond_wait(&ctx->request_ready) - owngil_execute_request(ctx) + ctx_execute_request(ctx) switch (ctx->request_type) { - case CTX_REQ_CALL: owngil_execute_call(ctx); break; - case CTX_REQ_EVAL: owngil_execute_eval(ctx); break; - case CTX_REQ_EXEC: owngil_execute_exec(ctx); break; + case CTX_REQ_CALL: ctx_execute_call(ctx); break; + case CTX_REQ_EVAL: ctx_execute_eval(ctx); break; + case CTX_REQ_EXEC: ctx_execute_exec(ctx); break; // ... other cases } @@ -224,8 +224,8 @@ OWN_GIL contexts support process-local environments for namespace isolation: ``` py_context:create_local_env(Ctx) └── nif_create_local_env(CtxRef) - └── dispatch_create_local_env_to_owngil(env, ctx, res) - └── owngil_execute_create_local_env(ctx) + └── ctx_dispatch_wait(env, ctx, req) (CTX_REQ_CREATE_LOCAL_ENV) + └── ctx_execute_create_local_env(ctx) ├── res->globals = PyDict_New() ├── res->locals = PyDict_New() └── res->interp_id = ctx->interp_id @@ -262,7 +262,7 @@ while (!shutdown_requested) { if (shutdown_requested) break; // Process request (GIL already held within subinterpreter) - owngil_execute_request(ctx); + ctx_execute_request(ctx); pthread_cond_signal(&response_ready); pthread_mutex_unlock(&request_mutex); @@ -336,7 +336,7 @@ Each Python subinterpreter has its own module namespace. The `py_event_loop` mod │ [ctx->uses_own_gil == true] ▼ ┌────────────────────────────────────────────────────────────────────────┐ -│ dispatch_reactor_read_to_owngil(env, ctx, fd, buffer_ptr) │ +│ dispatch_reactor_read(env, ctx, fd, buffer_ptr) │ │ │ │ │ ├── ctx->reactor_buffer_ptr = buffer_ptr │ │ ├── ctx->request_type = CTX_REQ_REACTOR_READ │ @@ -349,7 +349,7 @@ Each Python subinterpreter has its own module namespace. The `py_event_loop` mod │ OWN_GIL Thread │ ├────────────────────────────────────────────────────────────────────────┤ │ │ -│ owngil_execute_reactor_read(ctx) │ +│ ctx_execute_reactor_read(ctx) │ │ │ │ │ ├── Create ReactorBuffer Python object │ │ │ │ @@ -390,9 +390,9 @@ The `ensure_reactor_cached_for_interp()` function lazily imports `erlang.reactor | Request Type | Dispatch Function | Execute Function | |--------------|-------------------|------------------| -| `CTX_REQ_REACTOR_READ` | `dispatch_reactor_read_to_owngil` | `owngil_execute_reactor_read` | -| `CTX_REQ_REACTOR_WRITE` | `dispatch_reactor_write_to_owngil` | `owngil_execute_reactor_write` | -| `CTX_REQ_REACTOR_INIT` | `dispatch_reactor_init_to_owngil` | `owngil_execute_reactor_init` | +| `CTX_REQ_REACTOR_READ` | `dispatch_reactor_read` | `ctx_execute_reactor_read` | +| `CTX_REQ_REACTOR_WRITE` | `dispatch_reactor_write` | `ctx_execute_reactor_write` | +| `CTX_REQ_REACTOR_INIT` | `dispatch_reactor_init` | `ctx_execute_reactor_init` | ### Buffer Handling diff --git a/docs/state-machines.md b/docs/state-machines.md index 871bab1..f2fee76 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -25,7 +25,7 @@ UNINIT --init--> INITING --ok--> RUNNING --finalize--> SHUTTING_DOWN --> STOPPED Two cooperating machines: the Erlang process and the pthread in C. -Context thread (`worker_context_thread_main`, `owngil_context_thread_main` +Context thread (`ctx_thread_main_worker`, `ctx_thread_main_owngil` in `c_src/py_nif.c`): ``` @@ -38,11 +38,11 @@ starting --namespaces created--> waiting --dequeue--> executing --reply--> waiti - `executing` is bracketed by `py_context_exec_enter` / `exec_leave` (interrupt bookkeeping) around the GIL; the request mirror on `py_context_t` is valid only here. -- `exited` sets `worker_running = false`; `nif_context_destroy` joins with a +- `exited` sets `thread_running = false`; `nif_context_destroy` joins with a timeout and, if the join fails, marks the context `leaked` and pins the resource instead of freeing it. -Erlang process (`loop/1` in `py_context`): +Erlang process (`loop/1` in `py_context_embedded`): ``` idle --{call|eval|exec|submit}--> in_request --{py_result}--> idle diff --git a/src/py.erl b/src/py.erl index 8b9218a..f2c682e 100644 --- a/src/py.erl +++ b/src/py.erl @@ -145,7 +145,7 @@ -type py_args() :: [term()]. -type py_kwargs() :: #{atom() | binary() => term()}. --export_type([py_result/0, py_ref/0]). +-export_type([py_result/0, py_ref/0, py_module/0, py_func/0, py_args/0, py_kwargs/0]). %% Default timeout for synchronous calls (30 seconds) -define(DEFAULT_TIMEOUT, 30000). @@ -404,83 +404,24 @@ await(Ref, Timeout) -> %% @doc Stream results from a Python generator. %% Returns a list of all yielded values. -spec stream(py_module(), py_func(), py_args()) -> py_result(). -stream(Module, Func, Args) -> - stream(Module, Func, Args, #{}). +stream(A1, A2, A3) -> + py_stream:stream(A1, A2, A3). %% @doc Stream results from a Python generator with kwargs. -spec stream(py_module(), py_func(), py_args(), py_kwargs()) -> py_result(). -stream(Module, Func, Args, Kwargs) when map_size(Kwargs) == 0 -> - %% No kwargs - use stream_start and collect results - {ok, Ref} = stream_start(Module, Func, Args), - collect_stream(Ref, []); -stream(Module, Func, Args, Kwargs) -> - %% With kwargs - use eval approach - Ctx = py_context_router:get_context(), - ModuleBin = valid_py_module(ensure_binary(Module)), - FuncBin = valid_py_ident(ensure_binary(Func)), - KwargsCode = format_kwargs(Kwargs), - ArgsCode = format_args(Args), - Code = iolist_to_binary([ - <<"list(__import__('">>, ModuleBin, <<"').">>, FuncBin, - <<"(">>, ArgsCode, KwargsCode, <<"))">> - ]), - py_context:eval(Ctx, Code, #{}). - -%% @private Collect all stream events into a list -collect_stream(Ref, Acc) -> - receive - {py_stream, Ref, {data, Value}} -> - collect_stream(Ref, [Value | Acc]); - {py_stream, Ref, done} -> - {ok, lists:reverse(Acc)}; - {py_stream, Ref, {error, Reason}} -> - {error, Reason} - after 30000 -> - {error, timeout} - end. - -%% @private Format arguments for Python code -format_args([]) -> <<>>; -format_args(Args) -> - ArgStrs = [format_arg(A) || A <- Args], - iolist_to_binary(lists:join(<<", ">>, ArgStrs)). - -%% @private Format a single argument -format_arg(A) when is_integer(A) -> integer_to_binary(A); -format_arg(A) when is_float(A) -> float_to_binary(A); -format_arg(A) when is_binary(A) -> <<"'", (escape_py_literal(A))/binary, "'">>; -format_arg(A) when is_atom(A) -> <<"'", (escape_py_literal(atom_to_binary(A)))/binary, "'">>; -format_arg(A) when is_list(A) -> iolist_to_binary([<<"[">>, format_args(A), <<"]">>]); -format_arg(_) -> <<"None">>. - -%% @private Format kwargs for Python code -format_kwargs(Kwargs) when map_size(Kwargs) == 0 -> <<>>; -format_kwargs(Kwargs) -> - KwList = maps:fold(fun(K, V, Acc) -> - KB = valid_py_ident(if is_atom(K) -> atom_to_binary(K); is_binary(K) -> K end), - [<>, lists:join(<<", ">>, KwList)]). +stream(A1, A2, A3, A4) -> + py_stream:stream(A1, A2, A3, A4). %% @doc Stream results from a Python generator expression. %% Evaluates the expression and if it returns a generator, streams all values. -spec stream_eval(string() | binary()) -> py_result(). -stream_eval(Code) -> - stream_eval(Code, #{}). +stream_eval(A1) -> + py_stream:stream_eval(A1). %% @doc Stream results from a Python generator expression with local variables. -spec stream_eval(string() | binary(), map()) -> py_result(). -stream_eval(Code, Locals) -> - %% Route through the new process-per-context system - %% Wrap the code in list() to collect generator values - Ctx = py_context_router:get_context(), - CodeBin = ensure_binary(Code), - WrappedCode = <<"list(", CodeBin/binary, ")">>, - py_context:eval(Ctx, WrappedCode, Locals). - -%%% ============================================================================ -%%% True Streaming API (Event-driven) -%%% ============================================================================ +stream_eval(A1, A2) -> + py_stream:stream_eval(A1, A2). %% @doc Start a true streaming iteration from a Python generator. %% @@ -516,8 +457,8 @@ stream_eval(Code, Locals) -> %% end. %% ''' -spec stream_start(py_module(), py_func(), py_args()) -> {ok, reference()}. -stream_start(Module, Func, Args) -> - stream_start(Module, Func, Args, #{}). +stream_start(A1, A2, A3) -> + py_stream:stream_start(A1, A2, A3). %% @doc Start a true streaming iteration with options. %% @@ -530,83 +471,8 @@ stream_start(Module, Func, Args) -> %% @param Opts Options map %% @returns {ok, Ref} where Ref is used to identify stream events -spec stream_start(py_module(), py_func(), py_args(), map()) -> {ok, reference()}. -stream_start(Module, Func, Args, Opts) -> - Owner = maps:get(owner, Opts, self()), - Ref = make_ref(), - ModuleBin = ensure_binary(Module), - FuncBin = ensure_binary(Func), - RefHash = erlang:phash2(Ref), - %% Store owner and ref for Python to retrieve - %% Use binary keys because Python strings become binaries - py_state:store({<<"stream_owner">>, RefHash}, Owner), - py_state:store({<<"stream_ref">>, RefHash}, Ref), - py_state:store({<<"stream_args">>, RefHash}, Args), - %% Spawn an Erlang process to run the streaming iteration - spawn(fun() -> - stream_run_python(ModuleBin, FuncBin, RefHash) - end), - {ok, Ref}. - -%% @private Run the streaming via Python code -stream_run_python(ModuleBin0, FuncBin0, RefHash) -> - ModuleBin = valid_py_module(ModuleBin0), - FuncBin = valid_py_ident(FuncBin0), - RefHashBin = integer_to_binary(RefHash), - %% Build Python code that streams values using callbacks - Code = iolist_to_binary([ - <<"import erlang\n">>, - <<"_rh = ">>, RefHashBin, <<"\n">>, - <<"_args = erlang.call('state_get', ('stream_args', _rh))\n">>, - <<"if _args is None:\n">>, - <<" _args = []\n">>, - <<"try:\n">>, - <<" _mod = __import__('">>, ModuleBin, <<"')\n">>, - <<" _fn = getattr(_mod, '">>, FuncBin, <<"')\n">>, - <<" _gen = _fn(*_args) if _args else _fn()\n">>, - %% Async generators are driven on a private event loop. erlang.call is - %% a blocking pipe read, so it stalls that loop between yields, which - %% is fine for a sequential stream. - <<" if hasattr(_gen, '__anext__'):\n">>, - <<" import asyncio\n">>, - <<" async def _drive():\n">>, - <<" async for _val in _gen:\n">>, - <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, - <<" return\n">>, - <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, - <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, - <<" asyncio.run(_drive())\n">>, - <<" else:\n">>, - <<" for _val in _gen:\n">>, - <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, - <<" break\n">>, - <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, - <<" else:\n">>, - <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, - <<"except Exception as _e:\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', str(_e))\n">>, - <<"finally:\n">>, - <<" erlang.call('_py_stream_cleanup', _rh)\n">> - ]), - %% Execute the streaming code - case exec(Code) of - ok -> ok; - {error, Reason} -> - %% Try to notify owner of error - case py_state:fetch({<<"stream_owner">>, RefHash}) of - {ok, Owner} -> - case py_state:fetch({<<"stream_ref">>, RefHash}) of - {ok, Ref} -> - Owner ! {py_stream, Ref, {error, Reason}}, - py_state:remove({<<"stream_owner">>, RefHash}), - py_state:remove({<<"stream_ref">>, RefHash}), - py_state:remove({<<"stream_args">>, RefHash}); - _ -> ok - end; - _ -> ok - end - end. +stream_start(A1, A2, A3, A4) -> + py_stream:stream_start(A1, A2, A3, A4). %% @doc Cancel an active stream. %% @@ -616,13 +482,8 @@ stream_run_python(ModuleBin0, FuncBin0, RefHash) -> %% @param Ref The stream reference from stream_start/3,4 %% @returns ok -spec stream_cancel(reference()) -> ok. -stream_cancel(Ref) when is_reference(Ref) -> - %% Store cancellation flag that the streaming task checks - %% Use hash because we can't pass Erlang refs to Python callbacks easily - %% Use binary key because Python strings become binaries - RefHash = erlang:phash2(Ref), - py_state:store({<<"stream_cancelled_hash">>, RefHash}, true), - ok. +stream_cancel(A1) -> + py_stream:stream_cancel(A1). %%% ============================================================================ %%% Info @@ -844,6 +705,11 @@ parallel(Calls) when is_list(Calls) -> %%% Virtual Environment Support %%% ============================================================================ +%% @doc Kill the child of an isolated context. See py_context:kill/1. +-spec kill(pid()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + py_context:kill(Ctx). + %% @doc Ensure a virtual environment exists and activate it. %% %% Creates a venv at `Path' if it doesn't exist, installs dependencies from @@ -858,8 +724,8 @@ parallel(Calls) when is_list(Calls) -> %% ok = py:ensure_venv("priv/venv", "requirements.txt"). %% ''' -spec ensure_venv(string() | binary(), string() | binary()) -> ok | {error, term()}. -ensure_venv(Path, RequirementsFile) -> - ensure_venv(Path, RequirementsFile, []). +ensure_venv(A1, A2) -> + py_venv:ensure_venv(A1, A2). %% @doc Ensure a virtual environment exists with options. %% @@ -882,50 +748,8 @@ ensure_venv(Path, RequirementsFile) -> %% ]). %% ''' -spec ensure_venv(string() | binary(), string() | binary(), list()) -> ok | {error, term()}. -ensure_venv(Path, RequirementsFile, Opts) -> - PathStr = to_string(Path), - ReqFileStr = to_string(RequirementsFile), - Force = proplists:get_bool(force, Opts), - %% Create venv if needed - VenvReady = case venv_exists(PathStr) of - true when not Force -> - ok; - _ -> - create_venv(PathStr, Opts) - end, - case VenvReady of - ok -> - %% Always install/update dependencies (pip/uv skip existing) - case install_deps(PathStr, ReqFileStr, Opts) of - ok -> - activate_venv(PathStr); - {error, _} = Err -> - Err - end; - {error, _} = Err -> - Err - end. - -%% @private Check if venv exists by looking for pyvenv.cfg --spec venv_exists(string()) -> boolean(). -venv_exists(Path) -> - filelib:is_file(filename:join(Path, "pyvenv.cfg")). - -%% @private Create a new virtual environment --spec create_venv(string(), list()) -> ok | {error, term()}. -create_venv(Path, Opts) -> - Installer = detect_installer(Opts), - Python = case proplists:get_value(python, Opts, undefined) of - undefined -> get_python_executable(); - P -> P - end, - case Installer of - uv -> - %% uv venv is faster, use --python to match the running interpreter - run_cmd(uv_exe(), ["venv", "--python", Python, Path], []); - pip -> - run_cmd(Python, ["-m", "venv", Path], []) - end. +ensure_venv(A1, A2, A3) -> + py_venv:ensure_venv(A1, A2, A3). %% @private Get the Python executable path %% When embedded, sys.executable returns the embedding app (beam.smp) @@ -936,138 +760,7 @@ create_venv(Path, Opts) -> %% VM). Used as the default interpreter of isolated contexts and for venvs. -spec python_executable() -> string(). python_executable() -> - get_python_executable(). - -%% @doc Kill the child of an isolated context. See py_context:kill/1. --spec kill(pid()) -> ok | {error, not_isolated}. -kill(Ctx) when is_pid(Ctx) -> - py_context:kill(Ctx). - --spec get_python_executable() -> string(). -get_python_executable() -> - %% Use a single expression to find the Python executable - %% Searches for pythonX.Y, python3, python in sys.prefix/bin (Unix) - %% or python.exe in sys.prefix (Windows) - Expr = <<"(lambda: (__import__('os').path.join(__import__('sys').prefix, 'python.exe') if __import__('sys').platform == 'win32' and __import__('os').path.isfile(__import__('os').path.join(__import__('sys').prefix, 'python.exe')) else next((p for p in [__import__('os').path.join(__import__('sys').prefix, 'bin', f'python{__import__(\"sys\").version_info.major}.{__import__(\"sys\").version_info.minor}'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python3'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python')] if __import__('os').path.isfile(p)), 'python3')))()">>, - case eval(Expr) of - {ok, Path} when is_binary(Path) -> binary_to_list(Path); - _ -> "python3" - end. - -%% @private Install dependencies from requirements file --spec install_deps(string(), string(), list()) -> ok | {error, term()}. -install_deps(Path, RequirementsFile, Opts) -> - Installer = detect_installer(Opts), - {Exe, BaseArgs, PortOpts} = pip_command(Path, Installer), - Extras = proplists:get_value(extras, Opts, []), - - %% Determine file type and build the install argument list (no shell). - Args = case filename:extension(RequirementsFile) of - ".txt" -> - BaseArgs ++ ["install", "-r", RequirementsFile]; - ".toml" -> - %% pyproject.toml - install as editable. - %% filename:dirname returns "." for files without directory component - InstallPath = filename:dirname(RequirementsFile), - case Extras of - [] -> - BaseArgs ++ ["install", "-e", InstallPath]; - _ -> - ExtrasStr = string:join(Extras, ","), - BaseArgs ++ ["install", "-e", InstallPath ++ "[" ++ ExtrasStr ++ "]"] - end; - _ -> - BaseArgs ++ ["install", "-r", RequirementsFile] - end, - run_cmd(Exe, Args, PortOpts). - -%% @private Detect which installer to use (uv or pip) --spec detect_installer(list()) -> uv | pip. -detect_installer(Opts) -> - case proplists:get_value(installer, Opts, auto) of - auto -> - case os:find_executable("uv") of - false -> pip; - _ -> uv - end; - Installer -> - Installer - end. - -%% @private Resolve the installer into {Executable, BaseArgs, PortOpts}. -%% For uv the venv is selected via the VIRTUAL_ENV port env option (not a shell -%% prefix); for pip we use the venv's own pip binary. --spec pip_command(string(), uv | pip) -> {string(), [string()], list()}. -pip_command(VenvPath, uv) -> - {uv_exe(), ["pip"], [{env, [{"VIRTUAL_ENV", VenvPath}]}]}; -pip_command(VenvPath, pip) -> - PipExe = case os:type() of - {win32, _} -> - filename:join([VenvPath, "Scripts", "pip"]); - _ -> - filename:join([VenvPath, "bin", "pip"]) - end, - {PipExe, [], []}. - -%% @private Full path to the uv executable (falls back to the bare name). --spec uv_exe() -> string(). -uv_exe() -> - case os:find_executable("uv") of - false -> "uv"; - P -> P - end. - -%% @private Run an executable with an argv list (no shell) and return ok or error. --spec run_cmd(string(), [string()], list()) -> ok | {error, term()}. -run_cmd(Exe, Args, ExtraOpts) -> - case resolve_exe(Exe) of - {error, _} = Err -> - Err; - ExeFull -> - try open_port({spawn_executable, ExeFull}, - [exit_status, stderr_to_stdout, binary, {args, Args} | ExtraOpts]) of - Port -> collect_port(Port, []) - catch - error:Reason -> {error, {spawn_failed, Exe, Reason}} - end - end. - -%% @private Resolve an executable name/path to a full path (spawn_executable does -%% not search PATH). --spec resolve_exe(string()) -> string() | {error, term()}. -resolve_exe(Exe) -> - case filename:pathtype(Exe) of - absolute -> - case filelib:is_file(Exe) of - true -> Exe; - false -> {error, {executable_not_found, Exe}} - end; - _ -> - case os:find_executable(Exe) of - false -> {error, {executable_not_found, Exe}}; - Found -> Found - end - end. - -%% @private Collect a spawned port's output and exit status. --spec collect_port(port(), [binary()]) -> ok | {error, term()}. -collect_port(Port, Acc) -> - receive - {Port, {data, Data}} -> - collect_port(Port, [Data | Acc]); - {Port, {exit_status, 0}} -> - ok; - {Port, {exit_status, Code}} -> - {error, {exit_code, Code, iolist_to_binary(lists:reverse(Acc))}} - after 300000 -> - try port_close(Port) catch _:_ -> ok end, - {error, timeout} - end. - -%% @private Convert to string --spec to_string(string() | binary()) -> string(). -to_string(B) when is_binary(B) -> binary_to_list(B); -to_string(S) when is_list(S) -> S. + py_venv:python_executable(). %% @doc Activate a Python virtual environment. %% This modifies sys.path to use packages from the specified venv. @@ -1084,132 +777,20 @@ to_string(S) when is_list(S) -> S. %% {ok, _} = py:call(sentence_transformers, 'SentenceTransformer', [<<"all-MiniLM-L6-v2">>]). %% ''' -spec activate_venv(string() | binary()) -> ok | {error, term()}. -activate_venv(VenvPath) -> - VenvBin = ensure_binary(VenvPath), - %% Find site-packages directory dynamically (venv may use different Python version) - %% Uses a single expression to avoid multiline code issues - FindSitePackages = <<"(lambda vp: __import__('os').path.join(vp, 'Lib', 'site-packages') if __import__('os').path.exists(__import__('os').path.join(vp, 'Lib', 'site-packages')) else next((sp for name in (__import__('os').listdir(__import__('os').path.join(vp, 'lib')) if __import__('os').path.isdir(__import__('os').path.join(vp, 'lib')) else []) if name.startswith('python') for sp in [__import__('os').path.join(vp, 'lib', name, 'site-packages')] if __import__('os').path.isdir(sp)), None))(_venv_path)">>, - case eval(FindSitePackages, #{<<"_venv_path">> => VenvBin}) of - {ok, SitePackages} when SitePackages =/= none, SitePackages =/= null -> - activate_venv_with_site_packages(VenvBin, SitePackages); - {ok, _} -> - {error, {invalid_venv, no_site_packages_found}}; - Error -> - Error - end. - -%% @private Activate venv with known site-packages path -activate_venv_with_site_packages(VenvBin, SitePackages) -> - %% Verify site-packages exists - case eval(<<"__import__('os').path.isdir(sp)">>, #{sp => SitePackages}) of - {ok, true} -> - %% Save original path if not already saved - {ok, _} = eval(<<"setattr(__import__('sys'), '_original_path', __import__('sys').path.copy()) if not hasattr(__import__('sys'), '_original_path') else None">>), - %% Set venv info - {ok, _} = eval(<<"setattr(__import__('sys'), '_active_venv', vp)">>, #{vp => VenvBin}), - {ok, _} = eval(<<"setattr(__import__('sys'), '_venv_site_packages', sp)">>, #{sp => SitePackages}), - %% Add site-packages and process .pth files (editable installs) - %% Note: We embed the site-packages path directly since exec doesn't support - %% variables and sys attributes may not persist across calls in subinterpreters - SitePackagesStr = binary_to_list(SitePackages), - ExecCode = iolist_to_binary([ - <<"import site as _site, sys as _sys\n">>, - <<"_sp = '">>, escape_python_string(SitePackagesStr), <<"'\n">>, - <<"_b = frozenset(_sys.path)\n">>, - <<"_site.addsitedir(_sp)\n">>, - <<"_sys.path[:] = [p for p in _sys.path if p not in _b] + [p for p in _sys.path if p in _b]\n">>, - <<"del _site, _sys, _b, _sp\n">> - ]), - ok = exec(ExecCode), - ok; - {ok, false} -> - {error, {invalid_venv, SitePackages}}; - Error -> - Error - end. - -%% @private Escape a string for embedding in Python code -escape_python_string(Str) -> - lists:flatmap(fun($') -> "\\'"; - ($\\) -> "\\\\"; - (C) -> [C] - end, Str). - -%% @private Escape a binary for safe embedding inside a single-quoted Python -%% string literal: quote, backslash, and newline/CR/tab/other control bytes that -%% would otherwise break out of or corrupt the literal. -escape_py_literal(Bin) when is_binary(Bin) -> - << <<(escape_py_byte(B))/binary>> || <> <= Bin >>. - -escape_py_byte($') -> <<"\\'">>; -escape_py_byte($\\) -> <<"\\\\">>; -escape_py_byte($\n) -> <<"\\n">>; -escape_py_byte($\r) -> <<"\\r">>; -escape_py_byte($\t) -> <<"\\t">>; -escape_py_byte(B) when B < 16#20; B =:= 16#7f -> - list_to_binary(io_lib:format("\\x~2.16.0b", [B])); -escape_py_byte(B) -> <>. - -%% @private Validate a Python identifier ([A-Za-z_][A-Za-z0-9_]*). Crashes on a -%% non-conforming value so an attacker-controlled module/func/kwarg name can't -%% inject code at an identifier position (where quoting is meaningless). -valid_py_ident(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> - case ident_ok(Bin, first) of - true -> Bin; - false -> error({invalid_python_identifier, Bin}) - end; -valid_py_ident(Other) -> - error({invalid_python_identifier, Other}). - -%% @private Validate a dotted Python module path (each segment an identifier). -valid_py_module(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> - Segments = binary:split(Bin, <<".">>, [global]), - lists:foreach(fun valid_py_ident/1, Segments), - Bin; -valid_py_module(Other) -> - error({invalid_python_identifier, Other}). - -ident_ok(<<>>, first) -> false; %% empty segment (leading/trailing/double dot) -ident_ok(<<>>, rest) -> true; -ident_ok(<>, first) - when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); C =:= $_ -> - ident_ok(Rest, rest); -ident_ok(<>, rest) - when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); - (C >= $0 andalso C =< $9); C =:= $_ -> - ident_ok(Rest, rest); -ident_ok(_, _) -> false. +activate_venv(A1) -> + py_venv:activate_venv(A1). %% @doc Deactivate the current virtual environment. %% Restores sys.path to its original state. -spec deactivate_venv() -> ok | {error, term()}. deactivate_venv() -> - case eval(<<"hasattr(__import__('sys'), '_original_path')">>) of - {ok, true} -> - ok = exec(<<"import sys as _sys\n" - "_sys.path[:] = _sys._original_path\n" - "del _sys\n">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_original_path')">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_active_venv') if hasattr(__import__('sys'), '_active_venv') else None">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_venv_site_packages') if hasattr(__import__('sys'), '_venv_site_packages') else None">>), - ok; - {ok, false} -> - ok; - Error -> - Error - end. + py_venv:deactivate_venv(). %% @doc Get information about the currently active virtual environment. %% Returns a map with venv_path and site_packages, or none if no venv is active. -spec venv_info() -> {ok, map() | none} | {error, term()}. venv_info() -> - %% Check both attributes exist to handle partial activation/deactivation state - Code = <<"({'active': True, 'venv_path': __import__('sys')._active_venv, 'site_packages': __import__('sys')._venv_site_packages, 'sys_path': __import__('sys').path} if (hasattr(__import__('sys'), '_active_venv') and hasattr(__import__('sys'), '_venv_site_packages')) else {'active': False})">>, - eval(Code). - -%% @private -ensure_binary(S) -> - py_util:to_binary(S). + py_venv:venv_info(). %%% ============================================================================ %%% Execution Info @@ -1305,7 +886,7 @@ state_decr(Key, Amount) -> %% if any contexts failed. -spec reload(py_module()) -> ok | {error, [{context, term()}]}. reload(Module) -> - ModuleBin = ensure_binary(Module), + ModuleBin = py_util:to_binary(Module), %% Build Python code that: %% 1. Checks if module is loaded in sys.modules %% 2. If yes, reloads it with importlib.reload() @@ -1372,13 +953,13 @@ configure_logging(Opts) -> iolist_to_binary([ "__import__('erlang').setup_logging(", integer_to_binary(LevelInt), - ", '", escape_py_literal(F), "')" + ", '", py_util:escape_py_literal(F), "')" ]); F when is_list(F) -> iolist_to_binary([ "__import__('erlang').setup_logging(", integer_to_binary(LevelInt), - ", '", escape_py_literal(iolist_to_binary(F)), "')" + ", '", py_util:escape_py_literal(iolist_to_binary(F)), "')" ]) end, case eval(Code) of @@ -1529,7 +1110,7 @@ interrupt(Ctx) when is_pid(Ctx) -> %% @returns {ok, Result} | {error, Reason} -spec call_method(reference(), atom() | binary(), list()) -> py_result(). call_method(Ref, Method, Args) -> - MethodBin = ensure_binary(Method), + MethodBin = py_util:to_binary(Method), py_nif:ref_call_method(Ref, MethodBin, Args). %% @doc Get an attribute from a Python object reference. @@ -1539,7 +1120,7 @@ call_method(Ref, Method, Args) -> %% @returns {ok, Value} | {error, Reason} -spec getattr(reference(), atom() | binary()) -> py_result(). getattr(Ref, Name) -> - NameBin = ensure_binary(Name), + NameBin = py_util:to_binary(Name), py_nif:ref_getattr(Ref, NameBin). %% @doc Convert a Python object reference to an Erlang term. @@ -1645,7 +1226,7 @@ unregister_pool({Module, Func}) when is_atom(Module), is_atom(Func) -> %% @returns {ok, Reference} on success, {error, Reason} on failure -spec shared_dict_new() -> {ok, reference()} | {error, term()}. shared_dict_new() -> - py_nif:shared_dict_new(). + py_shared_dict:shared_dict_new(). %% @doc Get a value from SharedDict with default undefined. %% @@ -1653,8 +1234,8 @@ shared_dict_new() -> %% @param Key Binary key %% @returns Value or undefined if key not found -spec shared_dict_get(reference(), binary()) -> term(). -shared_dict_get(Handle, Key) -> - shared_dict_get(Handle, Key, undefined). +shared_dict_get(A1, A2) -> + py_shared_dict:shared_dict_get(A1, A2). %% @doc Get a value from SharedDict with custom default. %% @@ -1663,8 +1244,8 @@ shared_dict_get(Handle, Key) -> %% @param Default Default value if key not found %% @returns Value or Default -spec shared_dict_get(reference(), binary(), term()) -> term(). -shared_dict_get(Handle, Key, Default) when is_binary(Key) -> - py_nif:shared_dict_get(Handle, Key, Default). +shared_dict_get(A1, A2, A3) -> + py_shared_dict:shared_dict_get(A1, A2, A3). %% @doc Set a value in SharedDict. %% @@ -1675,8 +1256,8 @@ shared_dict_get(Handle, Key, Default) when is_binary(Key) -> %% @param Value Erlang term value (will be pickled) %% @returns ok on success -spec shared_dict_set(reference(), binary(), term()) -> ok | {error, term()}. -shared_dict_set(Handle, Key, Value) when is_binary(Key) -> - py_nif:shared_dict_set(Handle, Key, Value). +shared_dict_set(A1, A2, A3) -> + py_shared_dict:shared_dict_set(A1, A2, A3). %% @doc Delete a key from SharedDict. %% @@ -1684,16 +1265,16 @@ shared_dict_set(Handle, Key, Value) when is_binary(Key) -> %% @param Key Binary key %% @returns ok (even if key didn't exist) -spec shared_dict_del(reference(), binary()) -> ok. -shared_dict_del(Handle, Key) when is_binary(Key) -> - py_nif:shared_dict_del(Handle, Key). +shared_dict_del(A1, A2) -> + py_shared_dict:shared_dict_del(A1, A2). %% @doc Get all keys from SharedDict. %% %% @param Handle SharedDict reference %% @returns List of binary keys -spec shared_dict_keys(reference()) -> [binary()]. -shared_dict_keys(Handle) -> - py_nif:shared_dict_keys(Handle). +shared_dict_keys(A1) -> + py_shared_dict:shared_dict_keys(A1). %% @doc Explicitly destroy a SharedDict. %% @@ -1705,6 +1286,5 @@ shared_dict_keys(Handle) -> %% @param Handle SharedDict reference %% @returns ok -spec shared_dict_destroy(reference()) -> ok. -shared_dict_destroy(Handle) -> - py_nif:shared_dict_destroy(Handle). - +shared_dict_destroy(A1) -> + py_shared_dict:shared_dict_destroy(A1). diff --git a/src/py_context.erl b/src/py_context.erl index 385dff4..757df51 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -30,10 +30,10 @@ %%% back. Nested requests from the callback are served inline, so callbacks %%% can call Python again to any depth. %%% -%%% Owns: the context resource, the request in flight, its timeout and -%%% the process-local envs (`py:call(Ctx, ...)'). -%%% Talks to: `py_nif' (context NIFs), `py_isolated', `py_callback', -%%% `py_context_sup'. +%%% Owns: the public API, the reply protocol (`{MRef, Reply}', timeouts, +%%% interrupt on timeout) and the pid to NIF reference table. +%%% Talks to: `py_context_embedded' (the process body for worker and owngil +%%% mode), `py_isolated' (isolated mode), `py_nif' (interrupt). %%% Never: runs Python on a scheduler thread; the context thread does. %%% %%% @end @@ -78,9 +78,9 @@ -export([kill/1, pass_fd/2, child_info/1]). -export([init/3, init/4, init_ref_tab/0]). +%% Used by py_context_embedded +-export([register_nif_ref/1, unregister_nif_ref/0]). -%% Exported for py_reactor_context --export([extend_erlang_module_in_context/1]). %% Maps context pid -> NIF context reference. Read by interrupt/1, which must %% reach the NIF reference while the context process is blocked in a NIF and @@ -90,29 +90,17 @@ %% How long to wait for an interrupted call to unwind and reply, so the late %% reply is drained instead of being left in the caller's mailbox. -define(INTERRUPT_GRACE_MS, 1000). +%% Time given to a running loop to exit after interrupt/1 (also in +%% py_context_embedded, which drives the stop) +-define(LOOP_INTERRUPT_GRACE_MS, 3000). -type context_mode() :: worker | owngil | isolated. -type context() :: pid(). -export_type([context_mode/0, context/0]). --record(state, { - ref :: reference(), - id :: pos_integer(), - interp_id :: non_neg_integer(), - event_state = #{} :: map(), %% #{loop_ref => ref(), worker_pid => pid()} - callback_handler :: pid() | undefined, %% For thread-model callback handling - %% Worker loop (start_loop/1): request id of the run_forever exec, the - %% owner that gets {py_loop_exit, Ctx, Result}, its monitor, and the - %% callers waiting in stop_loop/2 - loop_req :: reference() | undefined, - loop_owner :: pid() | undefined, - loop_owner_mon :: reference() | undefined, - loop_stop_waiters = [] :: [{pid(), reference()}] -}). -%% Time given to a running loop to exit after py_context:interrupt/1 --define(LOOP_INTERRUPT_GRACE_MS, 3000). + %% ============================================================================ %% API @@ -735,867 +723,7 @@ init(Parent, Id, Mode) -> init(Parent, Id, isolated, Opts) -> py_isolated:init(Parent, Id, isolated, Opts); init(Parent, Id, Mode, Opts) -> - process_flag(trap_exit, true), - case create_context(Mode) of - {ok, Ref, InterpId} -> - %% Publish the NIF reference so interrupt/1 can reach it while - %% this process is blocked in a NIF - register_nif_ref(Ref), - case apply_memory_limit(Ref, Opts) of - ok -> - init_started(Parent, Id, Ref, InterpId, Opts); - {error, LimitError} -> - unregister_nif_ref(), - try py_nif:context_destroy(Ref) catch _:_ -> ok end, - Parent ! {self(), {error, LimitError}} - end; - {error, Reason} -> - Parent ! {self(), {error, Reason}} - end. - -%% @private -apply_memory_limit(Ref, Opts) -> - case maps:get(memory_limit, Opts, undefined) of - undefined -> - ok; - Bytes when is_integer(Bytes), Bytes >= 0 -> - py_nif:context_set_memory_limit(Ref, Bytes); - Other -> - {error, {invalid_memory_limit, Other}} - end. - -%% @private -init_started(Parent, Id, Ref, InterpId, Opts) -> - %% Apply all registered imports and paths to this interpreter - apply_registered_imports(Ref), - apply_registered_paths(Ref), - %% Apply preload code (populates globals for process-local envs) - apply_preload(Ref), - %% Per-context preload from new/1 (imports the app once per worker) - case maps:get(preload, Opts, undefined) of - undefined -> ok; - PreCode when is_binary(PreCode); is_list(PreCode) -> - case handle_exec_with_async(Ref, iolist_to_binary(PreCode)) of - ok -> ok; - {error, PreErr} -> - error_logger:warning_msg( - "py_context ~p: preload failed: ~p~n", [InterpId, PreErr]) - end - end, - %% For subinterpreters, create a dedicated event worker - EventState = setup_event_worker(Ref, InterpId), - %% For thread-model subinterpreters, spawn a dedicated callback handler - %% because the main context process will be blocked in the NIF - CallbackHandler = case maps:get(mode, EventState, normal) of - thread_model -> - Handler = spawn_callback_handler(Ref), - ok = py_nif:context_set_callback_handler(Ref, Handler), - Handler; - _ -> - undefined - end, - Parent ! {self(), started}, - State = #state{ - ref = Ref, - id = Id, - interp_id = InterpId, - event_state = EventState, - callback_handler = CallbackHandler - }, - loop(State). - -%% @private Create event worker for subinterpreter contexts -setup_event_worker(Ref, InterpId) -> - case py_nif:context_get_event_loop(Ref) of - {ok, LoopRef} -> - %% This is a subinterpreter - create dedicated event worker - WorkerId = iolist_to_binary(["ctx_", integer_to_list(InterpId)]), - case py_event_worker:start_link(WorkerId, LoopRef) of - {ok, WorkerPid} -> - ok = py_nif:event_loop_set_worker(LoopRef, WorkerPid), - %% Extend erlang module with event loop functions - extend_erlang_module_in_context(Ref), - #{loop_ref => LoopRef, worker_pid => WorkerPid}; - {error, WorkerError} -> - error_logger:warning_msg( - "py_context ~p: Failed to start event worker: ~p~n", - [InterpId, WorkerError]), - #{} - end; - {error, not_subinterp} -> - %% Worker mode - uses shared router (lazy initialization) - #{}; - {error, event_loop_owned_by_thread} -> - %% Thread-model subinterpreter: event loop is managed by dedicated thread. - %% This is expected behavior, not a failure. - #{mode => thread_model}; - {error, Reason} -> - error_logger:warning_msg( - "py_context ~p: Failed to get event loop: ~p~n", - [InterpId, Reason]), - #{} - end. - -%% @private Extend the erlang module with event loop functions in a subinterpreter -extend_erlang_module_in_context(Ref) -> - PrivDir = code:priv_dir(erlang_python), - Code = iolist_to_binary([ - "import sys\n", - "priv_dir = '", PrivDir, "'\n", - "if priv_dir not in sys.path:\n", - " sys.path.insert(0, priv_dir)\n", - "import erlang\n", - "if hasattr(erlang, '_extend_erlang_module'):\n", - " erlang._extend_erlang_module(priv_dir)\n" - ]), - case py_nif:context_exec(Ref, Code) of - ok -> ok; - {error, Reason} -> - error_logger:warning_msg( - "py_context: Failed to extend erlang module: ~p~n", [Reason]), - ok - end. - -%% @private Apply all imports from the global registry to this interpreter. -%% -%% Called when a new interpreter is created to pre-warm the module cache -%% with all modules registered via py_import:ensure_imported/1,2. -apply_registered_imports(Ref) -> - case py_import:all_imports() of - [] -> ok; - Imports -> py_nif:interp_apply_imports(Ref, Imports) - end. - -%% @private Apply all paths from the global registry to this interpreter. -%% -%% Called when a new interpreter is created to add all registered paths -%% to sys.path. -apply_registered_paths(Ref) -> - case py_import:all_paths() of - [] -> ok; - Paths -> py_nif:interp_apply_paths(Ref, Paths) - end. - -%% @private Apply preload code to the interpreter's globals. -%% -%% Called when a new interpreter is created. The preload code populates -%% the context's globals dict, which process-local environments inherit. -apply_preload(Ref) -> - py_preload:apply_preload(Ref). - -%% @private -create_context(worker) -> - py_nif:context_create(worker); -create_context(owngil) -> - %% OWN_GIL mode requires Python 3.14+ due to C extension bugs in earlier versions - case py_nif:owngil_supported() of - true -> py_nif:context_create(owngil); - false -> {error, owngil_requires_python314} - end. - -%% @private -%% Main context loop. Handles requests and uses suspension-based callback support. -loop(#state{ref = Ref, interp_id = InterpId, loop_req = LoopReq} = State) -> - receive - %% ---- worker loop management (start_loop/stop_loop/loop_ref) ---- - {start_loop, From, MRef, _Owner} when LoopReq =/= undefined -> - From ! {MRef, {error, already_running}}, - loop(State); - - {start_loop, From, MRef, Owner} -> - {Reply, NewState} = do_start_loop(Owner, State), - From ! {MRef, Reply}, - loop(NewState); - - {stop_loop, From, MRef, _GraceMs} when LoopReq =:= undefined -> - From ! {MRef, {error, no_loop}}, - loop(State); - - {stop_loop, From, MRef, GraceMs} -> - loop(begin_stop_loop(From, MRef, GraceMs, State)); - - {loop_ref, From, MRef} -> - From ! {MRef, context_loop_ref(State)}, - loop(State); - - {py_result, LoopReq, Result} when LoopReq =/= undefined -> - loop(loop_exited(Result, State)); - - {loop_stop_deadline, LoopReq} when LoopReq =/= undefined -> - %% Cooperative stop did not land: interrupt the thread - _ = py_nif:context_interrupt(Ref), - erlang:send_after(?LOOP_INTERRUPT_GRACE_MS, self(), - {loop_interrupt_deadline, LoopReq}), - loop(State); - - {loop_interrupt_deadline, LoopReq} when LoopReq =/= undefined -> - [W ! {M, {error, timeout}} || {W, M} <- State#state.loop_stop_waiters], - loop(State#state{loop_stop_waiters = []}); - - {loop_stop_deadline, _} -> - loop(State); - {loop_interrupt_deadline, _} -> - loop(State); - - {'DOWN', Mon, process, _Owner, _Reason} - when Mon =:= State#state.loop_owner_mon, LoopReq =/= undefined -> - %% Owner is gone: nobody will hear the exit, stop the loop - loop(begin_stop_loop(undefined, undefined, 5000, - State#state{loop_owner_mon = undefined})); - - {async_result, _TaskRef, _} -> - %% Result of a coroutine this process submitted (loop stop) - drop - loop(State); - - %% ---- while a worker loop runs, the thread is not available ---- - {call, From, MRef, _, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {call, From, MRef, _, _, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {eval, From, MRef, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {eval, From, MRef, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {exec, From, MRef, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {exec, From, MRef, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {call_method, From, MRef, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - - {stop, From, MRef} when LoopReq =/= undefined -> - %% Get the thread out of the loop before destroying the context, - %% otherwise context_destroy waits for a thread that never returns - terminate(normal, stop_running_loop(State)), - From ! {MRef, ok}; - - {'EXIT', _Pid, Reason} = Exit when LoopReq =/= undefined, - (Reason =:= shutdown orelse Reason =:= kill orelse - (is_tuple(Reason) andalso element(1, Reason) =:= shutdown)) -> - self() ! Exit, - loop(stop_running_loop(State)); - - {call, From, MRef, Module, Func, Args, Kwargs} -> - Result = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), - From ! {MRef, Result}, - loop(State); - - %% Call with process-local environment (worker mode) - {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> - Result = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), - From ! {MRef, Result}, - loop(State); - - {eval, From, MRef, Code, Locals} -> - Result = handle_eval_with_suspension(Ref, Code, Locals), - From ! {MRef, Result}, - loop(State); - - %% Eval with process-local environment (worker mode) - {eval, From, MRef, Code, Locals, EnvRef} -> - Result = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), - From ! {MRef, Result}, - loop(State); - - {exec, From, MRef, Code} -> - Result = handle_exec_with_async(Ref, Code), - From ! {MRef, Result}, - loop(State); - - %% Exec with process-local environment (worker mode). - %% Async dispatch with sync fallback (mirrors call/eval). - {exec, From, MRef, Code, EnvRef} -> - Result = handle_exec_with_async_and_env(Ref, Code, EnvRef), - From ! {MRef, Result}, - loop(State); - - {call_method, From, MRef, ObjRef, Method, Args} -> - Result = py_nif:context_call_method(Ref, ObjRef, Method, Args), - From ! {MRef, Result}, - loop(State); - - {get_interp_id, From, MRef} -> - From ! {MRef, {ok, InterpId}}, - loop(State); - - {is_subinterp, From, MRef} -> - %% Check the interp_id to determine if this is a subinterpreter - %% Subinterpreters have interp_id > 0 (main interpreter is 0) - %% But actually we need to check the mode, not just interp_id - IsSubinterp = is_context_subinterp(Ref), - From ! {MRef, IsSubinterp}, - loop(State); - - {create_local_env, From, MRef} -> - %% Create env inside this context's interpreter - Result = py_nif:create_local_env(Ref), - From ! {MRef, Result}, - loop(State); - - {get_nif_ref, From, MRef} -> - From ! {MRef, Ref}, - loop(State); - - {stop, From, MRef} -> - terminate(normal, State), - From ! {MRef, ok}; - - {'EXIT', Pid, Reason} -> - %% Handle EXIT from linked processes - case State#state.callback_handler of - Pid -> - %% Callback handler died - restart it for thread-model contexts - error_logger:warning_msg( - "py_context ~p: Callback handler died: ~p, restarting~n", - [InterpId, Reason]), - NewHandler = spawn_callback_handler(Ref), - ok = py_nif:context_set_callback_handler(Ref, NewHandler), - NewState = State#state{callback_handler = NewHandler}, - loop(NewState); - _ -> - case State#state.event_state of - #{worker_pid := Pid} -> - %% Event worker died - log and continue (degraded asyncio support) - error_logger:warning_msg( - "py_context ~p: Event worker died: ~p~n", - [InterpId, Reason]), - NewState = State#state{event_state = #{}}, - loop(NewState); - _ when Reason =:= shutdown; Reason =:= kill -> - %% Supervisor shutdown or kill signal - clean exit - terminate(Reason, State); - _ when is_tuple(Reason), element(1, Reason) =:= shutdown -> - %% Supervisor shutdown with extra info: {shutdown, _} - terminate(Reason, State); - _ -> - %% Ignore EXIT from other processes - loop(State) - end - end - end. - -%% ============================================================================ -%% Worker loop helpers -%% ============================================================================ - -%% @private Loop reference: the context's own loop (owngil) or the shared -%% main-interpreter loop (worker mode) -context_loop_ref(#state{event_state = #{loop_ref := LoopRef}}) -> - {ok, LoopRef}; -context_loop_ref(_State) -> - py_event_loop:get_loop(). - -%% @private Start run_forever on the context thread through the async exec -%% path, so this process stays free to serve loop_ref/stop_loop and the -%% dirty schedulers are not held. -do_start_loop(Owner, #state{ref = Ref} = State) -> - case context_loop_ref(State) of - {ok, _} -> - LoopReq = make_ref(), - case py_nif:context_call_async(Ref, self(), LoopReq, <<"erlang">>, - <<"_run_loop_forever">>, [self()], #{}) of - {enqueued, LoopReq} -> - %% Wait for the loop to actually run before answering, so - %% a submit right after start_loop finds it - receive - {py_loop_started} -> - Mon = case is_pid(Owner) of - true -> erlang:monitor(process, Owner); - false -> undefined - end, - {ok, State#state{loop_req = LoopReq, loop_owner = Owner, - loop_owner_mon = Mon, loop_stop_waiters = []}}; - {py_result, LoopReq, {error, Reason}} -> - {{error, Reason}, State}; - {py_result, LoopReq, Other} -> - {{error, {loop_exited, Other}}, State} - after 10000 -> - {{error, loop_start_timeout}, State} - end; - {error, Reason} -> - {{error, Reason}, State} - end; - {error, Reason} -> - {{error, Reason}, State} - end. - -%% @private Ask the running loop to stop from inside, arm the interrupt -%% deadline, and remember who to answer once it has exited. -begin_stop_loop(From, MRef, GraceMs, #state{loop_req = LoopReq} = State) -> - Waiters = case From of - undefined -> State#state.loop_stop_waiters; - _ -> [{From, MRef} | State#state.loop_stop_waiters] - end, - case context_loop_ref(State) of - {ok, LoopRef} -> - _ = py_nif:submit_task(LoopRef, self(), make_ref(), - <<"erlang">>, <<"_stop_loop">>, [], #{}); - _ -> - ok - end, - erlang:send_after(GraceMs, self(), {loop_stop_deadline, LoopReq}), - State#state{loop_stop_waiters = Waiters}. - -%% @private The exec running the loop returned: tell the owner and the -%% stop_loop callers, clear the loop state. -loop_exited(Result, #state{loop_owner = Owner, loop_owner_mon = Mon, - loop_stop_waiters = Waiters} = State) -> - case Mon of - undefined -> ok; - _ -> erlang:demonitor(Mon, [flush]) - end, - case is_pid(Owner) of - true -> Owner ! {py_loop_exit, self(), Result}; - false -> ok - end, - [W ! {M, ok} || {W, M} <- Waiters], - State#state{loop_req = undefined, loop_owner = undefined, - loop_owner_mon = undefined, loop_stop_waiters = []}. - -%% @private Synchronous stop used before terminate: interrupt and wait a -%% bounded time for the exec to return. -stop_running_loop(#state{ref = Ref, loop_req = LoopReq} = State) -> - _ = py_nif:context_interrupt(Ref), - receive - {py_result, LoopReq, Result} -> - loop_exited(Result, State) - after ?LOOP_INTERRUPT_GRACE_MS -> - loop_exited({error, timeout}, State) - end. - -%% @private Clean up resources on termination -terminate(_Reason, #state{ref = Ref, event_state = EventState, callback_handler = CallbackHandler}) -> - unregister_nif_ref(), - %% Stop the callback handler if it exists - case CallbackHandler of - Pid when is_pid(Pid) -> - Pid ! stop; - _ -> - ok - end, - %% Stop the event worker first (if it exists and is still alive) - case EventState of - #{worker_pid := WorkerPid} -> - try gen_server:stop(WorkerPid, normal, 5000) catch _:_ -> ok end; - _ -> - ok - end, - %% Destroy the Python context - try py_nif:context_destroy(Ref) catch _:_ -> ok end, - ok. - -%% ============================================================================ -%% Blocking callback handling (for thread-model subinterpreters) -%% ============================================================================ -%% -%% Thread-model subinterpreters use blocking pipe-based callbacks because -%% the suspension mechanism doesn't work when Python runs in a dedicated thread. -%% The Python thread blocks waiting for a response on the callback pipe. -%% -%% A separate callback handler process is spawned because the main context -%% process is blocked in the NIF (dispatch_to_thread) and cannot receive messages. - -%% @private -%% Spawn a dedicated callback handler process for thread-model subinterpreters. -spawn_callback_handler(Ref) -> - spawn_link(fun() -> callback_handler_loop(Ref) end). - -%% @private -%% Callback handler loop - receives erlang_callback messages and responds. -callback_handler_loop(Ref) -> - receive - {erlang_callback, _CallbackId, FuncName, Args} -> - handle_blocking_callback(Ref, FuncName, Args), - callback_handler_loop(Ref); - stop -> - ok - end. - -%% @private -%% Handle a blocking callback from a thread-model subinterpreter. -%% Executes the callback and writes the response to the callback pipe. -handle_blocking_callback(Ref, FuncName, Args) -> - %% Convert Args from tuple to list if needed - ArgsList = case Args of - T when is_tuple(T) -> tuple_to_list(T); - L when is_list(L) -> L; - _ -> [Args] - end, - %% Execute the registered function - Response = case py_callback:execute(FuncName, ArgsList) of - {ok, Result} -> - %% Format: status_byte (2=ok, ETF) + external term format - <<2, (term_to_binary(Result))/binary>>; - {error, {not_found, Name}} -> - ErrMsg = iolist_to_binary( - io_lib:format("Function '~s' not registered", [Name])), - <<1, ErrMsg/binary>>; - {error, {Class, Reason, _Stack}} -> - ErrMsg = iolist_to_binary( - io_lib:format("~p: ~p", [Class, Reason])), - <<1, ErrMsg/binary>> - end, - %% Write response to context's callback pipe - py_nif:context_write_callback_response(Ref, Response). - -%% ============================================================================ -%% Suspension-based callback handling -%% ============================================================================ -%% -%% When Python calls erlang.call(), the NIF returns {suspended, ...} instead of -%% blocking. We handle the callback inline and then resume Python execution. -%% This enables unlimited nesting depth without deadlock. - -%% @private -%% Handle call with potential suspension for callbacks -%% Uses async dispatch to avoid blocking dirty schedulers when possible. -handle_call_with_suspension(Ref, Module, Func, Args, Kwargs) -> - RequestId = make_ref(), - case py_nif:context_call_async(Ref, self(), RequestId, Module, Func, Args, Kwargs) of - {enqueued, RequestId} -> - %% Async dispatch succeeded - wait for result message - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - %% Fall back to blocking call for non-worker-thread contexts - handle_call_blocking(Ref, Module, Func, Args, Kwargs); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Blocking call handler (used when async is not available) -handle_call_blocking(Ref, Module, Func, Args, Kwargs) -> - case py_nif:context_call(Ref, Module, Func, Args, Kwargs) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - %% Callback needed - handle it with recursive receive - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - %% Resume and potentially get more suspensions - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - %% Schedule marker: Python returned erlang.schedule() - %% Execute the callback and return its result - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle eval with potential suspension for callbacks -%% Uses async dispatch to avoid blocking dirty schedulers when possible. -handle_eval_with_suspension(Ref, Code, Locals) -> - RequestId = make_ref(), - case py_nif:context_eval_async(Ref, self(), RequestId, Code, Locals) of - {enqueued, RequestId} -> - %% Async dispatch succeeded - wait for result message - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - %% Fall back to blocking call for non-worker-thread contexts - handle_eval_blocking(Ref, Code, Locals); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Handle exec with async dispatch -handle_exec_with_async(Ref, Code) -> - RequestId = make_ref(), - case py_nif:context_exec_async(Ref, self(), RequestId, Code) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - py_nif:context_exec(Ref, Code); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Blocking eval handler (used when async is not available) -handle_eval_blocking(Ref, Code, Locals) -> - case py_nif:context_eval(Ref, Code, Locals) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - %% Callback needed - handle it with recursive receive - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - %% Resume and potentially get more suspensions - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - %% Schedule marker: Python returned erlang.schedule() - %% Execute the callback and return its result - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Wait for async result from worker thread -%% The worker thread sends {py_result, RequestId, Result} when done. -%% -%% Drains stale {py_result, _, _} messages from prior timed-out -%% requests before the matching receive so a context that experiences -%% repeat timeouts doesn't grow an unbounded mailbox: when -%% wait_for_async_result/2 returns {error, async_timeout}, the C -%% worker can still finish later and deliver the result; without the -%% drain those messages would accumulate forever. -%% -%% Safe because the context process is the sole receiver for its own -%% async results and only one wait_for_async_result/2 is in flight at -%% a time, so the drain cannot consume the result of a concurrent live -%% request. -wait_for_async_result(Ref, RequestId) -> - drain_stale_async_results(RequestId), - receive - {py_result, RequestId, Result} -> - process_async_result(Ref, Result) - after 300000 -> %% 5 minute timeout - {error, async_timeout} - end. - -%% @private -drain_stale_async_results(CurrentId) -> - receive - {py_result, OldId, _} when OldId =/= CurrentId -> - drain_stale_async_results(CurrentId) - after 0 -> - ok - end. - -%% @private -%% Process the result from async dispatch -%% Handles suspension, schedule markers, and normal results. -process_async_result(Ref, {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}}) -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); -process_async_result(Ref, {schedule, CallbackName, CallbackArgs}) -> - handle_schedule(Ref, CallbackName, CallbackArgs); -process_async_result(_Ref, Result) -> - Result. - -%% @private -%% Handle call with process-local environment. -%% Tries async dispatch first (no 30 s NIF timeout); falls back to the -%% blocking NIF only when the worker thread isn't available. -handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_call_with_env_async(Ref, self(), RequestId, - Module, Func, Args, Kwargs, - EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - handle_call_with_env_blocking(Ref, Module, Func, Args, Kwargs, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -handle_call_with_env_blocking(Ref, Module, Func, Args, Kwargs, EnvRef) -> - case py_nif:context_call(Ref, Module, Func, Args, Kwargs, EnvRef) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle eval with process-local environment. -%% Tries async dispatch first; falls back to the blocking NIF only when -%% the worker thread isn't available. -handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_eval_with_env_async(Ref, self(), RequestId, - Code, Locals, EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - handle_eval_with_env_blocking(Ref, Code, Locals, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -handle_eval_with_env_blocking(Ref, Code, Locals, EnvRef) -> - case py_nif:context_eval(Ref, Code, Locals, EnvRef) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle exec with process-local environment via the same async-first -%% path used for call/eval. -handle_exec_with_async_and_env(Ref, Code, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_exec_with_env_async(Ref, self(), RequestId, - Code, EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - py_nif:context_exec(Ref, Code, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Check if a context is a subinterpreter (has interp_id > 0) -is_context_subinterp(Ref) -> - py_nif:context_interp_id(Ref) > 0. - -%% @private -%% Handle schedule marker - Python returned erlang.schedule() or schedule_py() -%% Execute the callback and return its result transparently to the caller. -%% -%% Special case for _execute_py: this callback is used by schedule_py() to -%% call back into Python with a different function. We handle it directly -%% using context_call to avoid recursion through py:call. -handle_schedule(Ref, <<"_execute_py">>, {Module, Func, Args, Kwargs}) -> - %% schedule_py callback: call Python function via context - CallArgs = case Args of - none -> []; - undefined -> []; - List when is_list(List) -> List; - Tuple when is_tuple(Tuple) -> tuple_to_list(Tuple); - _ -> [Args] - end, - CallKwargs = case Kwargs of - none -> #{}; - undefined -> #{}; - Map when is_map(Map) -> Map; - _ -> #{} - end, - handle_call_with_suspension(Ref, Module, Func, CallArgs, CallKwargs); -handle_schedule(_Ref, CallbackName, CallbackArgs) when is_binary(CallbackName) -> - %% Regular callback: execute via py_callback:execute - ArgsList = tuple_to_list(CallbackArgs), - case py_callback:execute(CallbackName, ArgsList) of - {ok, Result} -> - {ok, Result}; - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Handle callback, allowing nested py:eval/call to be processed. -%% We spawn a process to execute the callback so we can stay in a receive loop -%% for nested calls while the callback runs. -handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs) -> - Parent = self(), - CallbackPid = spawn_link(fun() -> - Result = try - ArgsList = tuple_to_list(CallbackArgs), - case py_callback:execute(FuncName, ArgsList) of - {ok, Value} -> - {ok, <<2, (term_to_binary(Value))/binary>>}; - {error, Reason} -> - ErrMsg = iolist_to_binary(io_lib:format("~p", [Reason])), - {ok, <<1, ErrMsg/binary>>} - end - catch - Class:ExcReason:Stacktrace -> - ErrorMsg = iolist_to_binary(io_lib:format("~p:~p~n~p", - [Class, ExcReason, Stacktrace])), - {ok, <<1, ErrorMsg/binary>>} - end, - Parent ! {callback_result, self(), Result} - end), - %% Wait for callback, processing nested requests - wait_for_callback(Ref, CallbackPid). - -%% @private -%% Wait for callback result while processing nested py:call/eval requests. -%% This enables arbitrarily deep callback nesting. -wait_for_callback(Ref, CallbackPid) -> - receive - {callback_result, CallbackPid, Result} -> - Result; - - %% Handle nested py:call while waiting for callback - {call, From, MRef, Module, Func, Args, Kwargs} -> - NestedResult = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:call while waiting for callback (with EnvRef) - {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> - NestedResult = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:eval while waiting for callback (without EnvRef) - {eval, From, MRef, Code, Locals} -> - NestedResult = handle_eval_with_suspension(Ref, Code, Locals), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:eval while waiting for callback (with EnvRef) - {eval, From, MRef, Code, Locals, EnvRef} -> - NestedResult = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:exec while waiting for callback - {exec, From, MRef, Code} -> - NestedResult = py_nif:context_exec(Ref, Code), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:exec while waiting for callback (with EnvRef) - {exec, From, MRef, Code, EnvRef} -> - NestedResult = py_nif:context_exec(Ref, Code, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested call_method while waiting for callback - {call_method, From, MRef, ObjRef, Method, Args} -> - NestedResult = py_nif:context_call_method(Ref, ObjRef, Method, Args), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle get_interp_id while waiting - {get_interp_id, From, MRef} -> - InterpId = py_nif:context_interp_id(Ref), - From ! {MRef, {ok, InterpId}}, - wait_for_callback(Ref, CallbackPid); - - %% Handle create_local_env while waiting - {create_local_env, From, MRef} -> - Result = py_nif:create_local_env(Ref), - From ! {MRef, Result}, - wait_for_callback(Ref, CallbackPid); - - {get_nif_ref, From, MRef} -> - From ! {MRef, Ref}, - wait_for_callback(Ref, CallbackPid) - end. - -%% @private -%% Resume suspended state, handle additional suspensions (nested callbacks) -resume_and_continue(Ref, StateRef, {ok, ResultBin}) -> - case py_nif:context_resume(Ref, StateRef, ResultBin) of - {suspended, _CallbackId2, StateRef2, {FuncName2, Args2}} -> - %% Another callback during resume - recursive handling - CallbackResult2 = handle_callback_with_nested_receive(Ref, FuncName2, Args2), - resume_and_continue(Ref, StateRef2, CallbackResult2); - FinalResult -> - FinalResult - end; -resume_and_continue(Ref, StateRef, {error, _} = Err) -> - _ = py_nif:context_cancel_resume(Ref, StateRef), - Err. - -%% ============================================================================ -%% Utility functions -%% ============================================================================ - -%% Callback results cross to Python as external term format (status byte 2) -%% and are decoded by term_to_py() in c_src/py_convert.c, the same -%% converter used for call arguments. The former Python-repr encoder was -%% removed in favour of it. + py_context_embedded:init(Parent, Id, Mode, Opts). %% @private to_binary(Atom) when is_atom(Atom) -> diff --git a/src/py_context_embedded.erl b/src/py_context_embedded.erl new file mode 100644 index 0000000..fddeb48 --- /dev/null +++ b/src/py_context_embedded.erl @@ -0,0 +1,847 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Process body of a context in `worker' or `owngil' mode. +%%% +%%% `py_context:init/4' hands the process here once the mode is known. The +%%% NIF context is created, the registry imports, paths and preload are +%%% applied, and the process enters `loop/1': one request at a time, +%%% forwarded to the context thread with the `*_async' NIFs and answered by +%%% `{py_result, Ref, Result}'. While the thread waits for an Erlang +%%% callback the process runs it here, serving nested requests, so +%%% callbacks can call Python again to any depth. +%%% +%%% Messages and replies are those documented in `py_context'; callers never +%%% address this module. +%%% +%%% Owns: the NIF context resource, the request in flight, the worker loop +%%% state and the callback handler process. +%%% Talks to: `py_nif' (context NIFs), `py_callback' (registered funs), +%%% `py_event_worker' (owngil loops), `py_import', `py_preload'. +%%% Never: answers a caller directly with anything but `{MRef, Reply}'. +%%% +%%% @private +-module(py_context_embedded). + +-export([init/4]). +%% Used by py_reactor_context +-export([extend_erlang_module_in_context/1]). + +-record(state, { + ref :: reference(), + id :: pos_integer(), + interp_id :: non_neg_integer(), + event_state = #{} :: map(), %% #{loop_ref => ref(), worker_pid => pid()} + callback_handler :: pid() | undefined, %% For thread-model callback handling + %% Worker loop (start_loop/1): request id of the run_forever exec, the + %% owner that gets {py_loop_exit, Ctx, Result}, its monitor, and the + %% callers waiting in stop_loop/2 + loop_req :: reference() | undefined, + loop_owner :: pid() | undefined, + loop_owner_mon :: reference() | undefined, + loop_stop_waiters = [] :: [{pid(), reference()}] +}). + +%% Time given to a running loop to exit after py_context:interrupt/1 +-define(LOOP_INTERRUPT_GRACE_MS, 3000). + +%% @private +init(Parent, Id, Mode, Opts) -> + process_flag(trap_exit, true), + case create_context(Mode) of + {ok, Ref, InterpId} -> + %% Publish the NIF reference so interrupt/1 can reach it while + %% this process is blocked in a NIF + py_context:register_nif_ref(Ref), + case apply_memory_limit(Ref, Opts) of + ok -> + init_started(Parent, Id, Ref, InterpId, Opts); + {error, LimitError} -> + py_context:unregister_nif_ref(), + try py_nif:context_destroy(Ref) catch _:_ -> ok end, + Parent ! {self(), {error, LimitError}} + end; + {error, Reason} -> + Parent ! {self(), {error, Reason}} + end. + +%% @private +apply_memory_limit(Ref, Opts) -> + case maps:get(memory_limit, Opts, undefined) of + undefined -> + ok; + Bytes when is_integer(Bytes), Bytes >= 0 -> + py_nif:context_set_memory_limit(Ref, Bytes); + Other -> + {error, {invalid_memory_limit, Other}} + end. + +%% @private +init_started(Parent, Id, Ref, InterpId, Opts) -> + %% Apply all registered imports and paths to this interpreter + apply_registered_imports(Ref), + apply_registered_paths(Ref), + %% Apply preload code (populates globals for process-local envs) + apply_preload(Ref), + %% Per-context preload from new/1 (imports the app once per worker) + case maps:get(preload, Opts, undefined) of + undefined -> ok; + PreCode when is_binary(PreCode); is_list(PreCode) -> + case handle_exec_with_async(Ref, iolist_to_binary(PreCode)) of + ok -> ok; + {error, PreErr} -> + error_logger:warning_msg( + "py_context ~p: preload failed: ~p~n", [InterpId, PreErr]) + end + end, + %% For subinterpreters, create a dedicated event worker + EventState = setup_event_worker(Ref, InterpId), + %% For thread-model subinterpreters, spawn a dedicated callback handler + %% because the main context process will be blocked in the NIF + CallbackHandler = case maps:get(mode, EventState, normal) of + thread_model -> + Handler = spawn_callback_handler(Ref), + ok = py_nif:context_set_callback_handler(Ref, Handler), + Handler; + _ -> + undefined + end, + Parent ! {self(), started}, + State = #state{ + ref = Ref, + id = Id, + interp_id = InterpId, + event_state = EventState, + callback_handler = CallbackHandler + }, + loop(State). + +%% @private Create event worker for subinterpreter contexts +setup_event_worker(Ref, InterpId) -> + case py_nif:context_get_event_loop(Ref) of + {ok, LoopRef} -> + %% This is a subinterpreter - create dedicated event worker + WorkerId = iolist_to_binary(["ctx_", integer_to_list(InterpId)]), + case py_event_worker:start_link(WorkerId, LoopRef) of + {ok, WorkerPid} -> + ok = py_nif:event_loop_set_worker(LoopRef, WorkerPid), + %% Extend erlang module with event loop functions + extend_erlang_module_in_context(Ref), + #{loop_ref => LoopRef, worker_pid => WorkerPid}; + {error, WorkerError} -> + error_logger:warning_msg( + "py_context ~p: Failed to start event worker: ~p~n", + [InterpId, WorkerError]), + #{} + end; + {error, not_subinterp} -> + %% Worker mode - uses shared router (lazy initialization) + #{}; + {error, event_loop_owned_by_thread} -> + %% Thread-model subinterpreter: event loop is managed by dedicated thread. + %% This is expected behavior, not a failure. + #{mode => thread_model}; + {error, Reason} -> + error_logger:warning_msg( + "py_context ~p: Failed to get event loop: ~p~n", + [InterpId, Reason]), + #{} + end. + +%% @private Extend the erlang module with event loop functions in a subinterpreter +extend_erlang_module_in_context(Ref) -> + PrivDir = code:priv_dir(erlang_python), + Code = iolist_to_binary([ + "import sys\n", + "priv_dir = '", PrivDir, "'\n", + "if priv_dir not in sys.path:\n", + " sys.path.insert(0, priv_dir)\n", + "import erlang\n", + "if hasattr(erlang, '_extend_erlang_module'):\n", + " erlang._extend_erlang_module(priv_dir)\n" + ]), + case py_nif:context_exec(Ref, Code) of + ok -> ok; + {error, Reason} -> + error_logger:warning_msg( + "py_context: Failed to extend erlang module: ~p~n", [Reason]), + ok + end. + +%% @private Apply all imports from the global registry to this interpreter. +%% +%% Called when a new interpreter is created to pre-warm the module cache +%% with all modules registered via py_import:ensure_imported/1,2. +apply_registered_imports(Ref) -> + case py_import:all_imports() of + [] -> ok; + Imports -> py_nif:interp_apply_imports(Ref, Imports) + end. + +%% @private Apply all paths from the global registry to this interpreter. +%% +%% Called when a new interpreter is created to add all registered paths +%% to sys.path. +apply_registered_paths(Ref) -> + case py_import:all_paths() of + [] -> ok; + Paths -> py_nif:interp_apply_paths(Ref, Paths) + end. + +%% @private Apply preload code to the interpreter's globals. +%% +%% Called when a new interpreter is created. The preload code populates +%% the context's globals dict, which process-local environments inherit. +apply_preload(Ref) -> + py_preload:apply_preload(Ref). + +%% @private +create_context(worker) -> + py_nif:context_create(worker); +create_context(owngil) -> + %% OWN_GIL mode requires Python 3.14+ due to C extension bugs in earlier versions + case py_nif:owngil_supported() of + true -> py_nif:context_create(owngil); + false -> {error, owngil_requires_python314} + end. + +%% @private +%% Main context loop. Handles requests and uses suspension-based callback support. +loop(#state{ref = Ref, interp_id = InterpId, loop_req = LoopReq} = State) -> + receive + %% ---- worker loop management (start_loop/stop_loop/loop_ref) ---- + {start_loop, From, MRef, _Owner} when LoopReq =/= undefined -> + From ! {MRef, {error, already_running}}, + loop(State); + + {start_loop, From, MRef, Owner} -> + {Reply, NewState} = do_start_loop(Owner, State), + From ! {MRef, Reply}, + loop(NewState); + + {stop_loop, From, MRef, _GraceMs} when LoopReq =:= undefined -> + From ! {MRef, {error, no_loop}}, + loop(State); + + {stop_loop, From, MRef, GraceMs} -> + loop(begin_stop_loop(From, MRef, GraceMs, State)); + + {loop_ref, From, MRef} -> + From ! {MRef, context_loop_ref(State)}, + loop(State); + + {py_result, LoopReq, Result} when LoopReq =/= undefined -> + loop(loop_exited(Result, State)); + + {loop_stop_deadline, LoopReq} when LoopReq =/= undefined -> + %% Cooperative stop did not land: interrupt the thread + _ = py_nif:context_interrupt(Ref), + erlang:send_after(?LOOP_INTERRUPT_GRACE_MS, self(), + {loop_interrupt_deadline, LoopReq}), + loop(State); + + {loop_interrupt_deadline, LoopReq} when LoopReq =/= undefined -> + [W ! {M, {error, timeout}} || {W, M} <- State#state.loop_stop_waiters], + loop(State#state{loop_stop_waiters = []}); + + {loop_stop_deadline, _} -> + loop(State); + {loop_interrupt_deadline, _} -> + loop(State); + + {'DOWN', Mon, process, _Owner, _Reason} + when Mon =:= State#state.loop_owner_mon, LoopReq =/= undefined -> + %% Owner is gone: nobody will hear the exit, stop the loop + loop(begin_stop_loop(undefined, undefined, 5000, + State#state{loop_owner_mon = undefined})); + + {async_result, _TaskRef, _} -> + %% Result of a coroutine this process submitted (loop stop) - drop + loop(State); + + %% ---- while a worker loop runs, the thread is not available ---- + {call, From, MRef, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call, From, MRef, _, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call_method, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + + {stop, From, MRef} when LoopReq =/= undefined -> + %% Get the thread out of the loop before destroying the context, + %% otherwise context_destroy waits for a thread that never returns + terminate(normal, stop_running_loop(State)), + From ! {MRef, ok}; + + {'EXIT', _Pid, Reason} = Exit when LoopReq =/= undefined, + (Reason =:= shutdown orelse Reason =:= kill orelse + (is_tuple(Reason) andalso element(1, Reason) =:= shutdown)) -> + self() ! Exit, + loop(stop_running_loop(State)); + + {call, From, MRef, Module, Func, Args, Kwargs} -> + Result = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), + From ! {MRef, Result}, + loop(State); + + %% Call with process-local environment (worker mode) + {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> + Result = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), + From ! {MRef, Result}, + loop(State); + + {eval, From, MRef, Code, Locals} -> + Result = handle_eval_with_suspension(Ref, Code, Locals), + From ! {MRef, Result}, + loop(State); + + %% Eval with process-local environment (worker mode) + {eval, From, MRef, Code, Locals, EnvRef} -> + Result = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), + From ! {MRef, Result}, + loop(State); + + {exec, From, MRef, Code} -> + Result = handle_exec_with_async(Ref, Code), + From ! {MRef, Result}, + loop(State); + + %% Exec with process-local environment (worker mode). + %% Async dispatch with sync fallback (mirrors call/eval). + {exec, From, MRef, Code, EnvRef} -> + Result = handle_exec_with_async_and_env(Ref, Code, EnvRef), + From ! {MRef, Result}, + loop(State); + + {call_method, From, MRef, ObjRef, Method, Args} -> + Result = py_nif:context_call_method(Ref, ObjRef, Method, Args), + From ! {MRef, Result}, + loop(State); + + {get_interp_id, From, MRef} -> + From ! {MRef, {ok, InterpId}}, + loop(State); + + {is_subinterp, From, MRef} -> + %% Check the interp_id to determine if this is a subinterpreter + %% Subinterpreters have interp_id > 0 (main interpreter is 0) + %% But actually we need to check the mode, not just interp_id + IsSubinterp = is_context_subinterp(Ref), + From ! {MRef, IsSubinterp}, + loop(State); + + {create_local_env, From, MRef} -> + %% Create env inside this context's interpreter + Result = py_nif:create_local_env(Ref), + From ! {MRef, Result}, + loop(State); + + {get_nif_ref, From, MRef} -> + From ! {MRef, Ref}, + loop(State); + + {stop, From, MRef} -> + terminate(normal, State), + From ! {MRef, ok}; + + {'EXIT', Pid, Reason} -> + %% Handle EXIT from linked processes + case State#state.callback_handler of + Pid -> + %% Callback handler died - restart it for thread-model contexts + error_logger:warning_msg( + "py_context ~p: Callback handler died: ~p, restarting~n", + [InterpId, Reason]), + NewHandler = spawn_callback_handler(Ref), + ok = py_nif:context_set_callback_handler(Ref, NewHandler), + NewState = State#state{callback_handler = NewHandler}, + loop(NewState); + _ -> + case State#state.event_state of + #{worker_pid := Pid} -> + %% Event worker died - log and continue (degraded asyncio support) + error_logger:warning_msg( + "py_context ~p: Event worker died: ~p~n", + [InterpId, Reason]), + NewState = State#state{event_state = #{}}, + loop(NewState); + _ when Reason =:= shutdown; Reason =:= kill -> + %% Supervisor shutdown or kill signal - clean exit + terminate(Reason, State); + _ when is_tuple(Reason), element(1, Reason) =:= shutdown -> + %% Supervisor shutdown with extra info: {shutdown, _} + terminate(Reason, State); + _ -> + %% Ignore EXIT from other processes + loop(State) + end + end + end. + +%% ============================================================================ +%% Worker loop helpers +%% ============================================================================ + +%% @private Loop reference: the context's own loop (owngil) or the shared +%% main-interpreter loop (worker mode) +context_loop_ref(#state{event_state = #{loop_ref := LoopRef}}) -> + {ok, LoopRef}; +context_loop_ref(_State) -> + py_event_loop:get_loop(). + +%% @private Start run_forever on the context thread through the async exec +%% path, so this process stays free to serve loop_ref/stop_loop and the +%% dirty schedulers are not held. +do_start_loop(Owner, #state{ref = Ref} = State) -> + case context_loop_ref(State) of + {ok, _} -> + LoopReq = make_ref(), + case py_nif:context_call_async(Ref, self(), LoopReq, <<"erlang">>, + <<"_run_loop_forever">>, [self()], #{}) of + {enqueued, LoopReq} -> + %% Wait for the loop to actually run before answering, so + %% a submit right after start_loop finds it + receive + {py_loop_started} -> + Mon = case is_pid(Owner) of + true -> erlang:monitor(process, Owner); + false -> undefined + end, + {ok, State#state{loop_req = LoopReq, loop_owner = Owner, + loop_owner_mon = Mon, loop_stop_waiters = []}}; + {py_result, LoopReq, {error, Reason}} -> + {{error, Reason}, State}; + {py_result, LoopReq, Other} -> + {{error, {loop_exited, Other}}, State} + after 10000 -> + {{error, loop_start_timeout}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end. + +%% @private Ask the running loop to stop from inside, arm the interrupt +%% deadline, and remember who to answer once it has exited. +begin_stop_loop(From, MRef, GraceMs, #state{loop_req = LoopReq} = State) -> + Waiters = case From of + undefined -> State#state.loop_stop_waiters; + _ -> [{From, MRef} | State#state.loop_stop_waiters] + end, + case context_loop_ref(State) of + {ok, LoopRef} -> + _ = py_nif:submit_task(LoopRef, self(), make_ref(), + <<"erlang">>, <<"_stop_loop">>, [], #{}); + _ -> + ok + end, + erlang:send_after(GraceMs, self(), {loop_stop_deadline, LoopReq}), + State#state{loop_stop_waiters = Waiters}. + +%% @private The exec running the loop returned: tell the owner and the +%% stop_loop callers, clear the loop state. +loop_exited(Result, #state{loop_owner = Owner, loop_owner_mon = Mon, + loop_stop_waiters = Waiters} = State) -> + case Mon of + undefined -> ok; + _ -> erlang:demonitor(Mon, [flush]) + end, + case is_pid(Owner) of + true -> Owner ! {py_loop_exit, self(), Result}; + false -> ok + end, + [W ! {M, ok} || {W, M} <- Waiters], + State#state{loop_req = undefined, loop_owner = undefined, + loop_owner_mon = undefined, loop_stop_waiters = []}. + +%% @private Synchronous stop used before terminate: interrupt and wait a +%% bounded time for the exec to return. +stop_running_loop(#state{ref = Ref, loop_req = LoopReq} = State) -> + _ = py_nif:context_interrupt(Ref), + receive + {py_result, LoopReq, Result} -> + loop_exited(Result, State) + after ?LOOP_INTERRUPT_GRACE_MS -> + loop_exited({error, timeout}, State) + end. + +%% @private Clean up resources on termination +terminate(_Reason, #state{ref = Ref, event_state = EventState, callback_handler = CallbackHandler}) -> + py_context:unregister_nif_ref(), + %% Stop the callback handler if it exists + case CallbackHandler of + Pid when is_pid(Pid) -> + Pid ! stop; + _ -> + ok + end, + %% Stop the event worker first (if it exists and is still alive) + case EventState of + #{worker_pid := WorkerPid} -> + try gen_server:stop(WorkerPid, normal, 5000) catch _:_ -> ok end; + _ -> + ok + end, + %% Destroy the Python context + try py_nif:context_destroy(Ref) catch _:_ -> ok end, + ok. + +%% ============================================================================ +%% Blocking callback handling (for thread-model subinterpreters) +%% ============================================================================ +%% +%% Thread-model subinterpreters use blocking pipe-based callbacks because +%% the suspension mechanism doesn't work when Python runs in a dedicated thread. +%% The Python thread blocks waiting for a response on the callback pipe. +%% +%% A separate callback handler process is spawned because the main context +%% process is blocked in the NIF (dispatch_to_thread) and cannot receive messages. + +%% @private +%% Spawn a dedicated callback handler process for thread-model subinterpreters. +spawn_callback_handler(Ref) -> + spawn_link(fun() -> callback_handler_loop(Ref) end). + +%% @private +%% Callback handler loop - receives erlang_callback messages and responds. +callback_handler_loop(Ref) -> + receive + {erlang_callback, _CallbackId, FuncName, Args} -> + handle_blocking_callback(Ref, FuncName, Args), + callback_handler_loop(Ref); + stop -> + ok + end. + +%% @private +%% Handle a blocking callback from a thread-model subinterpreter. +%% Executes the callback and writes the response to the callback pipe. +handle_blocking_callback(Ref, FuncName, Args) -> + %% Convert Args from tuple to list if needed + ArgsList = case Args of + T when is_tuple(T) -> tuple_to_list(T); + L when is_list(L) -> L; + _ -> [Args] + end, + %% Execute the registered function + Response = case py_callback:execute(FuncName, ArgsList) of + {ok, Result} -> + %% Format: status_byte (2=ok, ETF) + external term format + <<2, (term_to_binary(Result))/binary>>; + {error, {not_found, Name}} -> + ErrMsg = iolist_to_binary( + io_lib:format("Function '~s' not registered", [Name])), + <<1, ErrMsg/binary>>; + {error, {Class, Reason, _Stack}} -> + ErrMsg = iolist_to_binary( + io_lib:format("~p: ~p", [Class, Reason])), + <<1, ErrMsg/binary>> + end, + %% Write response to context's callback pipe + py_nif:context_write_callback_response(Ref, Response). + +%% ============================================================================ +%% Suspension-based callback handling +%% ============================================================================ +%% +%% When Python calls erlang.call(), the NIF returns {suspended, ...} instead of +%% blocking. We handle the callback inline and then resume Python execution. +%% This enables unlimited nesting depth without deadlock. + +%% @private +%% Handle call with potential suspension for callbacks +handle_call_with_suspension(Ref, Module, Func, Args, Kwargs) -> + RequestId = make_ref(), + case py_nif:context_call_async(Ref, self(), RequestId, Module, Func, Args, Kwargs) of + {enqueued, RequestId} -> + %% Async dispatch succeeded - wait for result message + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle eval with potential suspension for callbacks +handle_eval_with_suspension(Ref, Code, Locals) -> + RequestId = make_ref(), + case py_nif:context_eval_async(Ref, self(), RequestId, Code, Locals) of + {enqueued, RequestId} -> + %% Async dispatch succeeded - wait for result message + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Handle exec with async dispatch +handle_exec_with_async(Ref, Code) -> + RequestId = make_ref(), + case py_nif:context_exec_async(Ref, self(), RequestId, Code) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Wait for async result from worker thread +%% The worker thread sends {py_result, RequestId, Result} when done. +%% +%% Drains stale {py_result, _, _} messages from prior timed-out +%% requests before the matching receive so a context that experiences +%% repeat timeouts doesn't grow an unbounded mailbox: when +%% wait_for_async_result/2 returns {error, async_timeout}, the C +%% worker can still finish later and deliver the result; without the +%% drain those messages would accumulate forever. +%% +%% Safe because the context process is the sole receiver for its own +%% async results and only one wait_for_async_result/2 is in flight at +%% a time, so the drain cannot consume the result of a concurrent live +%% request. +wait_for_async_result(Ref, RequestId) -> + drain_stale_async_results(RequestId), + receive + {py_result, RequestId, Result} -> + process_async_result(Ref, Result) + after 300000 -> %% 5 minute timeout + {error, async_timeout} + end. + +%% @private +drain_stale_async_results(CurrentId) -> + receive + {py_result, OldId, _} when OldId =/= CurrentId -> + drain_stale_async_results(CurrentId) + after 0 -> + ok + end. + +%% @private +%% Process the result from async dispatch +%% Handles suspension, schedule markers, and normal results. +process_async_result(Ref, {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}}) -> + CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), + resume_and_continue(Ref, StateRef, CallbackResult); +process_async_result(Ref, {schedule, CallbackName, CallbackArgs}) -> + handle_schedule(Ref, CallbackName, CallbackArgs); +process_async_result(_Ref, Result) -> + Result. + +%% @private +%% Handle call with process-local environment. +handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_call_with_env_async(Ref, self(), RequestId, + Module, Func, Args, Kwargs, + EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle eval with process-local environment. +handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_eval_with_env_async(Ref, self(), RequestId, + Code, Locals, EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle exec with process-local environment via the same async-first +%% path used for call/eval. +handle_exec_with_async_and_env(Ref, Code, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_exec_with_env_async(Ref, self(), RequestId, + Code, EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Check if a context is a subinterpreter (has interp_id > 0) +is_context_subinterp(Ref) -> + py_nif:context_interp_id(Ref) > 0. + +%% @private +%% Handle schedule marker - Python returned erlang.schedule() or schedule_py() +%% Execute the callback and return its result transparently to the caller. +%% +%% Special case for _execute_py: this callback is used by schedule_py() to +%% call back into Python with a different function. We handle it directly +%% using context_call to avoid recursion through py:call. +handle_schedule(Ref, <<"_execute_py">>, {Module, Func, Args, Kwargs}) -> + %% schedule_py callback: call Python function via context + CallArgs = case Args of + none -> []; + undefined -> []; + List when is_list(List) -> List; + Tuple when is_tuple(Tuple) -> tuple_to_list(Tuple); + _ -> [Args] + end, + CallKwargs = case Kwargs of + none -> #{}; + undefined -> #{}; + Map when is_map(Map) -> Map; + _ -> #{} + end, + handle_call_with_suspension(Ref, Module, Func, CallArgs, CallKwargs); +handle_schedule(_Ref, CallbackName, CallbackArgs) when is_binary(CallbackName) -> + %% Regular callback: execute via py_callback:execute + ArgsList = tuple_to_list(CallbackArgs), + case py_callback:execute(CallbackName, ArgsList) of + {ok, Result} -> + {ok, Result}; + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Handle callback, allowing nested py:eval/call to be processed. +%% We spawn a process to execute the callback so we can stay in a receive loop +%% for nested calls while the callback runs. +handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs) -> + Parent = self(), + CallbackPid = spawn_link(fun() -> + Result = try + ArgsList = tuple_to_list(CallbackArgs), + case py_callback:execute(FuncName, ArgsList) of + {ok, Value} -> + {ok, <<2, (term_to_binary(Value))/binary>>}; + {error, Reason} -> + ErrMsg = iolist_to_binary(io_lib:format("~p", [Reason])), + {ok, <<1, ErrMsg/binary>>} + end + catch + Class:ExcReason:Stacktrace -> + ErrorMsg = iolist_to_binary(io_lib:format("~p:~p~n~p", + [Class, ExcReason, Stacktrace])), + {ok, <<1, ErrorMsg/binary>>} + end, + Parent ! {callback_result, self(), Result} + end), + %% Wait for callback, processing nested requests + wait_for_callback(Ref, CallbackPid). + +%% @private +%% Wait for callback result while processing nested py:call/eval requests. +%% This enables arbitrarily deep callback nesting. +wait_for_callback(Ref, CallbackPid) -> + receive + {callback_result, CallbackPid, Result} -> + Result; + + %% Handle nested py:call while waiting for callback + {call, From, MRef, Module, Func, Args, Kwargs} -> + NestedResult = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:call while waiting for callback (with EnvRef) + {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> + NestedResult = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:eval while waiting for callback (without EnvRef) + {eval, From, MRef, Code, Locals} -> + NestedResult = handle_eval_with_suspension(Ref, Code, Locals), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:eval while waiting for callback (with EnvRef) + {eval, From, MRef, Code, Locals, EnvRef} -> + NestedResult = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:exec while waiting for callback + {exec, From, MRef, Code} -> + NestedResult = py_nif:context_exec(Ref, Code), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:exec while waiting for callback (with EnvRef) + {exec, From, MRef, Code, EnvRef} -> + NestedResult = py_nif:context_exec(Ref, Code, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested call_method while waiting for callback + {call_method, From, MRef, ObjRef, Method, Args} -> + NestedResult = py_nif:context_call_method(Ref, ObjRef, Method, Args), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle get_interp_id while waiting + {get_interp_id, From, MRef} -> + InterpId = py_nif:context_interp_id(Ref), + From ! {MRef, {ok, InterpId}}, + wait_for_callback(Ref, CallbackPid); + + %% Handle create_local_env while waiting + {create_local_env, From, MRef} -> + Result = py_nif:create_local_env(Ref), + From ! {MRef, Result}, + wait_for_callback(Ref, CallbackPid); + + {get_nif_ref, From, MRef} -> + From ! {MRef, Ref}, + wait_for_callback(Ref, CallbackPid) + end. + +%% @private +%% Resume suspended state, handle additional suspensions (nested callbacks) +resume_and_continue(Ref, StateRef, {ok, ResultBin}) -> + case py_nif:context_resume(Ref, StateRef, ResultBin) of + {suspended, _CallbackId2, StateRef2, {FuncName2, Args2}} -> + %% Another callback during resume - recursive handling + CallbackResult2 = handle_callback_with_nested_receive(Ref, FuncName2, Args2), + resume_and_continue(Ref, StateRef2, CallbackResult2); + FinalResult -> + FinalResult + end; +resume_and_continue(Ref, StateRef, {error, _} = Err) -> + _ = py_nif:context_cancel_resume(Ref, StateRef), + Err. + +%% ============================================================================ +%% Utility functions +%% ============================================================================ + +%% Callback results cross to Python as external term format (status byte 2) +%% and are decoded by term_to_py() in c_src/py_convert.c, the same +%% converter used for call arguments. The former Python-repr encoder was +%% removed in favour of it. diff --git a/src/py_reactor_context.erl b/src/py_reactor_context.erl index 0261f8e..52fe763 100644 --- a/src/py_reactor_context.erl +++ b/src/py_reactor_context.erl @@ -187,7 +187,7 @@ init(Parent, Id, Mode, Opts) -> py_nif:context_set_callback_handler(Ref, self()), %% Extend erlang module to make erlang.reactor available - py_context:extend_erlang_module_in_context(Ref), + py_context_embedded:extend_erlang_module_in_context(Ref), MaxConns = maps:get(max_connections, Opts, ?DEFAULT_MAX_CONNECTIONS), AppModule = maps:get(app_module, Opts, undefined), diff --git a/src/py_shared_dict.erl b/src/py_shared_dict.erl new file mode 100644 index 0000000..8ee103b --- /dev/null +++ b/src/py_shared_dict.erl @@ -0,0 +1,110 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Process-scoped shared dictionaries (`py:shared_dict_*'). +%%% Thin wrappers over the `shared_dict_*' NIFs; use the `py' functions. +%%% @private +-module(py_shared_dict). + +-export([ + shared_dict_new/0, + shared_dict_get/2, + shared_dict_get/3, + shared_dict_set/3, + shared_dict_del/2, + shared_dict_keys/1, + shared_dict_destroy/1 +]). + + +%% @doc Create a new process-scoped SharedDict. +%% +%% Creates a SharedDict owned by the calling process. The dict is automatically +%% destroyed when the owning process terminates. Values are stored as pickled +%% bytes for cross-interpreter safety. +%% +%% == Example == +%% ``` +%% {ok, SD} = py:shared_dict_new(). +%% ok = py:shared_dict_set(SD, <<"config">>, #{host => <<"localhost">>}). +%% #{<<"host">> := <<"localhost">>} = py:shared_dict_get(SD, <<"config">>). +%% ''' +%% +%% @returns {ok, Reference} on success, {error, Reason} on failure +-spec shared_dict_new() -> {ok, reference()} | {error, term()}. +shared_dict_new() -> + py_nif:shared_dict_new(). + +%% @doc Get a value from SharedDict with default undefined. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @returns Value or undefined if key not found +-spec shared_dict_get(reference(), binary()) -> term(). +shared_dict_get(Handle, Key) -> + shared_dict_get(Handle, Key, undefined). + +%% @doc Get a value from SharedDict with custom default. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @param Default Default value if key not found +%% @returns Value or Default +-spec shared_dict_get(reference(), binary(), term()) -> term(). +shared_dict_get(Handle, Key, Default) when is_binary(Key) -> + py_nif:shared_dict_get(Handle, Key, Default). + +%% @doc Set a value in SharedDict. +%% +%% The value is pickled for cross-interpreter safety. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @param Value Erlang term value (will be pickled) +%% @returns ok on success +-spec shared_dict_set(reference(), binary(), term()) -> ok | {error, term()}. +shared_dict_set(Handle, Key, Value) when is_binary(Key) -> + py_nif:shared_dict_set(Handle, Key, Value). + +%% @doc Delete a key from SharedDict. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @returns ok (even if key didn't exist) +-spec shared_dict_del(reference(), binary()) -> ok. +shared_dict_del(Handle, Key) when is_binary(Key) -> + py_nif:shared_dict_del(Handle, Key). + +%% @doc Get all keys from SharedDict. +%% +%% @param Handle SharedDict reference +%% @returns List of binary keys +-spec shared_dict_keys(reference()) -> [binary()]. +shared_dict_keys(Handle) -> + py_nif:shared_dict_keys(Handle). + +%% @doc Explicitly destroy a SharedDict. +%% +%% Marks the SharedDict as destroyed and clears its Python dict. +%% After destruction, any further operations on this SharedDict will +%% return badarg. This is idempotent - calling on an already-destroyed +%% dict returns ok. +%% +%% @param Handle SharedDict reference +%% @returns ok +-spec shared_dict_destroy(reference()) -> ok. +shared_dict_destroy(Handle) -> + py_nif:shared_dict_destroy(Handle). + + diff --git a/src/py_stream.erl b/src/py_stream.erl new file mode 100644 index 0000000..badb8f8 --- /dev/null +++ b/src/py_stream.erl @@ -0,0 +1,255 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Streaming results from Python generators into Erlang messages. +%%% Implementation of `py:stream/3,4', `py:stream_eval/1,2', `py:stream_start/3,4' +%%% and `py:stream_cancel/1'; use those. Owns the `{py_stream, Ref, ...}' +%%% event protocol and the Python-side generator driver. +%%% @private +-module(py_stream). + +-export([ + stream/3, + stream/4, + stream_eval/1, + stream_eval/2, + stream_start/3, + stream_start/4, + stream_cancel/1 +]). + + +%% @doc Stream results from a Python generator. +%% Returns a list of all yielded values. +-spec stream(py:py_module(), py:py_func(), py:py_args()) -> py:py_result(). +stream(Module, Func, Args) -> + stream(Module, Func, Args, #{}). + +%% @doc Stream results from a Python generator with kwargs. +-spec stream(py:py_module(), py:py_func(), py:py_args(), py:py_kwargs()) -> py:py_result(). +stream(Module, Func, Args, Kwargs) when map_size(Kwargs) == 0 -> + %% No kwargs - use stream_start and collect results + {ok, Ref} = stream_start(Module, Func, Args), + collect_stream(Ref, []); +stream(Module, Func, Args, Kwargs) -> + %% With kwargs - use eval approach + Ctx = py_context_router:get_context(), + ModuleBin = py_util:valid_py_module(py_util:to_binary(Module)), + FuncBin = py_util:valid_py_ident(py_util:to_binary(Func)), + KwargsCode = format_kwargs(Kwargs), + ArgsCode = format_args(Args), + Code = iolist_to_binary([ + <<"list(__import__('">>, ModuleBin, <<"').">>, FuncBin, + <<"(">>, ArgsCode, KwargsCode, <<"))">> + ]), + py_context:eval(Ctx, Code, #{}). + +%% @private Collect all stream events into a list +collect_stream(Ref, Acc) -> + receive + {py_stream, Ref, {data, Value}} -> + collect_stream(Ref, [Value | Acc]); + {py_stream, Ref, done} -> + {ok, lists:reverse(Acc)}; + {py_stream, Ref, {error, Reason}} -> + {error, Reason} + after 30000 -> + {error, timeout} + end. + +%% @private Format arguments for Python code +format_args([]) -> <<>>; +format_args(Args) -> + ArgStrs = [format_arg(A) || A <- Args], + iolist_to_binary(lists:join(<<", ">>, ArgStrs)). + +%% @private Format a single argument +format_arg(A) when is_integer(A) -> integer_to_binary(A); +format_arg(A) when is_float(A) -> float_to_binary(A); +format_arg(A) when is_binary(A) -> <<"'", (py_util:escape_py_literal(A))/binary, "'">>; +format_arg(A) when is_atom(A) -> <<"'", (py_util:escape_py_literal(atom_to_binary(A)))/binary, "'">>; +format_arg(A) when is_list(A) -> iolist_to_binary([<<"[">>, format_args(A), <<"]">>]); +format_arg(_) -> <<"None">>. + +%% @private Format kwargs for Python code +format_kwargs(Kwargs) when map_size(Kwargs) == 0 -> <<>>; +format_kwargs(Kwargs) -> + KwList = maps:fold(fun(K, V, Acc) -> + KB = py_util:valid_py_ident(if is_atom(K) -> atom_to_binary(K); is_binary(K) -> K end), + [<>, lists:join(<<", ">>, KwList)]). + +%% @doc Stream results from a Python generator expression. +%% Evaluates the expression and if it returns a generator, streams all values. +-spec stream_eval(string() | binary()) -> py:py_result(). +stream_eval(Code) -> + stream_eval(Code, #{}). + +%% @doc Stream results from a Python generator expression with local variables. +-spec stream_eval(string() | binary(), map()) -> py:py_result(). +stream_eval(Code, Locals) -> + %% Route through the new process-per-context system + %% Wrap the code in list() to collect generator values + Ctx = py_context_router:get_context(), + CodeBin = py_util:to_binary(Code), + WrappedCode = <<"list(", CodeBin/binary, ")">>, + py_context:eval(Ctx, WrappedCode, Locals). + +%%% ============================================================================ +%%% True Streaming API (Event-driven) +%%% ============================================================================ + +%% @doc Start a true streaming iteration from a Python generator. +%% +%% Unlike stream/3,4 which collects all values at once, this function +%% returns immediately with a reference and sends values as events +%% to the calling process as they are yielded. +%% +%% Events sent to the owner process: +%% - `{py_stream, Ref, {data, Value}}' - Each yielded value +%% - `{py_stream, Ref, done}' - Stream completed +%% - `{py_stream, Ref, {error, Reason}}' - Stream error +%% +%% Accepts sync generators and async generators. An async generator is driven +%% on a private event loop, one value at a time; delivering a value blocks that +%% loop, so other coroutines on it do not progress between yields. +%% +%% Example: +%% ``` +%% {ok, Ref} = py:stream_start(builtins, iter, [[1,2,3,4,5]]), +%% receive_loop(Ref). +%% +%% receive_loop(Ref) -> +%% receive +%% {py_stream, Ref, {data, Value}} -> +%% io:format("Got: ~p~n", [Value]), +%% receive_loop(Ref); +%% {py_stream, Ref, done} -> +%% io:format("Complete~n"); +%% {py_stream, Ref, {error, Reason}} -> +%% io:format("Error: ~p~n", [Reason]) +%% after 30000 -> +%% timeout +%% end. +%% ''' +-spec stream_start(py:py_module(), py:py_func(), py:py_args()) -> {ok, reference()}. +stream_start(Module, Func, Args) -> + stream_start(Module, Func, Args, #{}). + +%% @doc Start a true streaming iteration with options. +%% +%% Options: +%% - `owner => pid()' - Process to receive events (default: self()) +%% +%% @param Module Python module name +%% @param Func Python function name +%% @param Args Function arguments +%% @param Opts Options map +%% @returns {ok, Ref} where Ref is used to identify stream events +-spec stream_start(py:py_module(), py:py_func(), py:py_args(), map()) -> {ok, reference()}. +stream_start(Module, Func, Args, Opts) -> + Owner = maps:get(owner, Opts, self()), + Ref = make_ref(), + ModuleBin = py_util:to_binary(Module), + FuncBin = py_util:to_binary(Func), + RefHash = erlang:phash2(Ref), + %% Store owner and ref for Python to retrieve + %% Use binary keys because Python strings become binaries + py_state:store({<<"stream_owner">>, RefHash}, Owner), + py_state:store({<<"stream_ref">>, RefHash}, Ref), + py_state:store({<<"stream_args">>, RefHash}, Args), + %% Spawn an Erlang process to run the streaming iteration + spawn(fun() -> + stream_run_python(ModuleBin, FuncBin, RefHash) + end), + {ok, Ref}. + +%% @private Run the streaming via Python code +stream_run_python(ModuleBin0, FuncBin0, RefHash) -> + ModuleBin = py_util:valid_py_module(ModuleBin0), + FuncBin = py_util:valid_py_ident(FuncBin0), + RefHashBin = integer_to_binary(RefHash), + %% Build Python code that streams values using callbacks + Code = iolist_to_binary([ + <<"import erlang\n">>, + <<"_rh = ">>, RefHashBin, <<"\n">>, + <<"_args = erlang.call('state_get', ('stream_args', _rh))\n">>, + <<"if _args is None:\n">>, + <<" _args = []\n">>, + <<"try:\n">>, + <<" _mod = __import__('">>, ModuleBin, <<"')\n">>, + <<" _fn = getattr(_mod, '">>, FuncBin, <<"')\n">>, + <<" _gen = _fn(*_args) if _args else _fn()\n">>, + %% Async generators are driven on a private event loop. erlang.call is + %% a blocking pipe read, so it stalls that loop between yields, which + %% is fine for a sequential stream. + <<" if hasattr(_gen, '__anext__'):\n">>, + <<" import asyncio\n">>, + <<" async def _drive():\n">>, + <<" async for _val in _gen:\n">>, + <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, + <<" return\n">>, + <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, + <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, + <<" asyncio.run(_drive())\n">>, + <<" else:\n">>, + <<" for _val in _gen:\n">>, + <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, + <<" break\n">>, + <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, + <<" else:\n">>, + <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, + <<"except Exception as _e:\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', str(_e))\n">>, + <<"finally:\n">>, + <<" erlang.call('_py_stream_cleanup', _rh)\n">> + ]), + %% Execute the streaming code + case py:exec(Code) of + ok -> ok; + {error, Reason} -> + %% Try to notify owner of error + case py_state:fetch({<<"stream_owner">>, RefHash}) of + {ok, Owner} -> + case py_state:fetch({<<"stream_ref">>, RefHash}) of + {ok, Ref} -> + Owner ! {py_stream, Ref, {error, Reason}}, + py_state:remove({<<"stream_owner">>, RefHash}), + py_state:remove({<<"stream_ref">>, RefHash}), + py_state:remove({<<"stream_args">>, RefHash}); + _ -> ok + end; + _ -> ok + end + end. + +%% @doc Cancel an active stream. +%% +%% Sends a cancellation signal to stop the stream iteration. +%% Any pending values may still be delivered before the stream stops. +%% +%% @param Ref The stream reference from stream_start/3,4 +%% @returns ok +-spec stream_cancel(reference()) -> ok. +stream_cancel(Ref) when is_reference(Ref) -> + %% Store cancellation flag that the streaming task checks + %% Use hash because we can't pass Erlang refs to Python callbacks easily + %% Use binary key because Python strings become binaries + RefHash = erlang:phash2(Ref), + py_state:store({<<"stream_cancelled_hash">>, RefHash}, true), + ok. + diff --git a/src/py_util.erl b/src/py_util.erl index 5ed4319..9ae86b1 100644 --- a/src/py_util.erl +++ b/src/py_util.erl @@ -18,7 +18,10 @@ -module(py_util). -export([ - to_binary/1 + to_binary/1, + escape_py_literal/1, + valid_py_module/1, + valid_py_ident/1 ]). %%% ============================================================================ @@ -33,3 +36,47 @@ to_binary(List) when is_list(List) -> list_to_binary(List); to_binary(Bin) when is_binary(Bin) -> Bin. + +%% @doc Escape a binary for use inside a single-quoted Python string literal. +-spec escape_py_literal(binary()) -> binary(). +escape_py_literal(Bin) when is_binary(Bin) -> + << <<(escape_py_byte(B))/binary>> || <> <= Bin >>. + +escape_py_byte($') -> <<"\\'">>; +escape_py_byte($\\) -> <<"\\\\">>; +escape_py_byte($\n) -> <<"\\n">>; +escape_py_byte($\r) -> <<"\\r">>; +escape_py_byte($\t) -> <<"\\t">>; +escape_py_byte(B) when B < 16#20; B =:= 16#7f -> + list_to_binary(io_lib:format("\\x~2.16.0b", [B])); +escape_py_byte(B) -> <>. + +%% @private Validate a dotted Python module path (each segment an identifier). +valid_py_module(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> + Segments = binary:split(Bin, <<".">>, [global]), + lists:foreach(fun valid_py_ident/1, Segments), + Bin; +valid_py_module(Other) -> + error({invalid_python_identifier, Other}). + +ident_ok(<<>>, first) -> false; %% empty segment (leading/trailing/double dot) +ident_ok(<<>>, rest) -> true; +ident_ok(<>, first) + when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); C =:= $_ -> + ident_ok(Rest, rest); +ident_ok(<>, rest) + when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); + (C >= $0 andalso C =< $9); C =:= $_ -> + ident_ok(Rest, rest); +ident_ok(_, _) -> false. + +%% @private Validate a Python identifier ([A-Za-z_][A-Za-z0-9_]*). Crashes on a +%% non-conforming value so an attacker-controlled module/func/kwarg name can't +%% inject code at an identifier position (where quoting is meaningless). +valid_py_ident(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> + case ident_ok(Bin, first) of + true -> Bin; + false -> error({invalid_python_identifier, Bin}) + end; +valid_py_ident(Other) -> + error({invalid_python_identifier, Other}). diff --git a/src/py_venv.erl b/src/py_venv.erl new file mode 100644 index 0000000..731f6cd --- /dev/null +++ b/src/py_venv.erl @@ -0,0 +1,350 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Virtual environments: create, install dependencies, activate. +%%% Implementation of `py:ensure_venv/2,3', `py:activate_venv/1', +%%% `py:deactivate_venv/0', `py:venv_info/0' and `py:python_executable/0'; +%%% use those. Owns the venv layout under the configured directory and the +%%% pip/uv invocation. +%%% @private +-module(py_venv). + +-export([ + ensure_venv/2, + ensure_venv/3, + python_executable/0, + activate_venv/1, + deactivate_venv/0, + venv_info/0 +]). + + +%% @doc Ensure a virtual environment exists and activate it. +%% +%% Creates a venv at `Path' if it doesn't exist, installs dependencies from +%% `RequirementsFile', and activates the venv. +%% +%% RequirementsFile can be: +%% - `"requirements.txt"' - standard pip requirements file +%% - `"pyproject.toml"' - PEP 621 project file (installs with -e .) +%% +%% Example: +%% ``` +%% ok = py:ensure_venv("priv/venv", "requirements.txt"). +%% ''' +-spec ensure_venv(string() | binary(), string() | binary()) -> ok | {error, term()}. +ensure_venv(Path, RequirementsFile) -> + ensure_venv(Path, RequirementsFile, []). + +%% @doc Ensure a virtual environment exists with options. +%% +%% Options: +%% - `{extras, [string()]}' - Install optional dependencies (pyproject.toml) +%% - `{installer, uv | pip}' - Package installer (default: auto-detect) +%% - `{python, string()}' - Python executable for venv creation +%% - `force' - Recreate venv even if it exists +%% +%% Example: +%% ``` +%% %% With pyproject.toml and dev extras +%% ok = py:ensure_venv("priv/venv", "pyproject.toml", [ +%% {extras, ["dev", "test"]} +%% ]). +%% +%% %% Force uv installer +%% ok = py:ensure_venv("priv/venv", "requirements.txt", [ +%% {installer, uv} +%% ]). +%% ''' +-spec ensure_venv(string() | binary(), string() | binary(), list()) -> ok | {error, term()}. +ensure_venv(Path, RequirementsFile, Opts) -> + PathStr = to_string(Path), + ReqFileStr = to_string(RequirementsFile), + Force = proplists:get_bool(force, Opts), + %% Create venv if needed + VenvReady = case venv_exists(PathStr) of + true when not Force -> + ok; + _ -> + create_venv(PathStr, Opts) + end, + case VenvReady of + ok -> + %% Always install/update dependencies (pip/uv skip existing) + case install_deps(PathStr, ReqFileStr, Opts) of + ok -> + activate_venv(PathStr); + {error, _} = Err -> + Err + end; + {error, _} = Err -> + Err + end. + +%% @private Check if venv exists by looking for pyvenv.cfg +-spec venv_exists(string()) -> boolean(). +venv_exists(Path) -> + filelib:is_file(filename:join(Path, "pyvenv.cfg")). + +%% @private Create a new virtual environment +-spec create_venv(string(), list()) -> ok | {error, term()}. +create_venv(Path, Opts) -> + Installer = detect_installer(Opts), + Python = case proplists:get_value(python, Opts, undefined) of + undefined -> get_python_executable(); + P -> P + end, + case Installer of + uv -> + %% uv venv is faster, use --python to match the running interpreter + run_cmd(uv_exe(), ["venv", "--python", Python, Path], []); + pip -> + run_cmd(Python, ["-m", "venv", Path], []) + end. + +%% @private Get the Python executable path +%% When embedded, sys.executable returns the embedding app (beam.smp) +%% so we reconstruct the path from sys.prefix and version info +%% @doc Path of the Python interpreter matching the embedded runtime. +%% +%% Reconstructed from `sys.prefix' (when embedded, `sys.executable' is the +%% VM). Used as the default interpreter of isolated contexts and for venvs. +-spec python_executable() -> string(). +python_executable() -> + get_python_executable(). + + +-spec get_python_executable() -> string(). +get_python_executable() -> + %% Use a single expression to find the Python executable + %% Searches for pythonX.Y, python3, python in sys.prefix/bin (Unix) + %% or python.exe in sys.prefix (Windows) + Expr = <<"(lambda: (__import__('os').path.join(__import__('sys').prefix, 'python.exe') if __import__('sys').platform == 'win32' and __import__('os').path.isfile(__import__('os').path.join(__import__('sys').prefix, 'python.exe')) else next((p for p in [__import__('os').path.join(__import__('sys').prefix, 'bin', f'python{__import__(\"sys\").version_info.major}.{__import__(\"sys\").version_info.minor}'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python3'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python')] if __import__('os').path.isfile(p)), 'python3')))()">>, + case py:eval(Expr) of + {ok, Path} when is_binary(Path) -> binary_to_list(Path); + _ -> "python3" + end. + +%% @private Install dependencies from requirements file +-spec install_deps(string(), string(), list()) -> ok | {error, term()}. +install_deps(Path, RequirementsFile, Opts) -> + Installer = detect_installer(Opts), + {Exe, BaseArgs, PortOpts} = pip_command(Path, Installer), + Extras = proplists:get_value(extras, Opts, []), + + %% Determine file type and build the install argument list (no shell). + Args = case filename:extension(RequirementsFile) of + ".txt" -> + BaseArgs ++ ["install", "-r", RequirementsFile]; + ".toml" -> + %% pyproject.toml - install as editable. + %% filename:dirname returns "." for files without directory component + InstallPath = filename:dirname(RequirementsFile), + case Extras of + [] -> + BaseArgs ++ ["install", "-e", InstallPath]; + _ -> + ExtrasStr = string:join(Extras, ","), + BaseArgs ++ ["install", "-e", InstallPath ++ "[" ++ ExtrasStr ++ "]"] + end; + _ -> + BaseArgs ++ ["install", "-r", RequirementsFile] + end, + run_cmd(Exe, Args, PortOpts). + +%% @private Detect which installer to use (uv or pip) +-spec detect_installer(list()) -> uv | pip. +detect_installer(Opts) -> + case proplists:get_value(installer, Opts, auto) of + auto -> + case os:find_executable("uv") of + false -> pip; + _ -> uv + end; + Installer -> + Installer + end. + +%% @private Resolve the installer into {Executable, BaseArgs, PortOpts}. +%% For uv the venv is selected via the VIRTUAL_ENV port env option (not a shell +%% prefix); for pip we use the venv's own pip binary. +-spec pip_command(string(), uv | pip) -> {string(), [string()], list()}. +pip_command(VenvPath, uv) -> + {uv_exe(), ["pip"], [{env, [{"VIRTUAL_ENV", VenvPath}]}]}; +pip_command(VenvPath, pip) -> + PipExe = case os:type() of + {win32, _} -> + filename:join([VenvPath, "Scripts", "pip"]); + _ -> + filename:join([VenvPath, "bin", "pip"]) + end, + {PipExe, [], []}. + +%% @private Full path to the uv executable (falls back to the bare name). +-spec uv_exe() -> string(). +uv_exe() -> + case os:find_executable("uv") of + false -> "uv"; + P -> P + end. + +%% @private Run an executable with an argv list (no shell) and return ok or error. +-spec run_cmd(string(), [string()], list()) -> ok | {error, term()}. +run_cmd(Exe, Args, ExtraOpts) -> + case resolve_exe(Exe) of + {error, _} = Err -> + Err; + ExeFull -> + try open_port({spawn_executable, ExeFull}, + [exit_status, stderr_to_stdout, binary, {args, Args} | ExtraOpts]) of + Port -> collect_port(Port, []) + catch + error:Reason -> {error, {spawn_failed, Exe, Reason}} + end + end. + +%% @private Resolve an executable name/path to a full path (spawn_executable does +%% not search PATH). +-spec resolve_exe(string()) -> string() | {error, term()}. +resolve_exe(Exe) -> + case filename:pathtype(Exe) of + absolute -> + case filelib:is_file(Exe) of + true -> Exe; + false -> {error, {executable_not_found, Exe}} + end; + _ -> + case os:find_executable(Exe) of + false -> {error, {executable_not_found, Exe}}; + Found -> Found + end + end. + +%% @private Collect a spawned port's output and exit status. +-spec collect_port(port(), [binary()]) -> ok | {error, term()}. +collect_port(Port, Acc) -> + receive + {Port, {data, Data}} -> + collect_port(Port, [Data | Acc]); + {Port, {exit_status, 0}} -> + ok; + {Port, {exit_status, Code}} -> + {error, {exit_code, Code, iolist_to_binary(lists:reverse(Acc))}} + after 300000 -> + try port_close(Port) catch _:_ -> ok end, + {error, timeout} + end. + +%% @private Convert to string +-spec to_string(string() | binary()) -> string(). +to_string(B) when is_binary(B) -> binary_to_list(B); +to_string(S) when is_list(S) -> S. + +%% @doc Activate a Python virtual environment. +%% This modifies sys.path to use packages from the specified venv. +%% The venv path should be the root directory (containing bin/lib folders). +%% +%% `.pth' files in the venv's site-packages directory are processed, so +%% editable installs created by uv, pip, or any PEP 517/660 compliant tool +%% work correctly. New paths are inserted at the front of sys.path so that +%% venv packages take priority over system packages. +%% +%% Example: +%% ``` +%% ok = py:activate_venv(<<"/path/to/myenv">>). +%% {ok, _} = py:call(sentence_transformers, 'SentenceTransformer', [<<"all-MiniLM-L6-v2">>]). +%% ''' +-spec activate_venv(string() | binary()) -> ok | {error, term()}. +activate_venv(VenvPath) -> + VenvBin = py_util:to_binary(VenvPath), + %% Find site-packages directory dynamically (venv may use different Python version) + %% Uses a single expression to avoid multiline code issues + FindSitePackages = <<"(lambda vp: __import__('os').path.join(vp, 'Lib', 'site-packages') if __import__('os').path.exists(__import__('os').path.join(vp, 'Lib', 'site-packages')) else next((sp for name in (__import__('os').listdir(__import__('os').path.join(vp, 'lib')) if __import__('os').path.isdir(__import__('os').path.join(vp, 'lib')) else []) if name.startswith('python') for sp in [__import__('os').path.join(vp, 'lib', name, 'site-packages')] if __import__('os').path.isdir(sp)), None))(_venv_path)">>, + case py:eval(FindSitePackages, #{<<"_venv_path">> => VenvBin}) of + {ok, SitePackages} when SitePackages =/= none, SitePackages =/= null -> + activate_venv_with_site_packages(VenvBin, SitePackages); + {ok, _} -> + {error, {invalid_venv, no_site_packages_found}}; + Error -> + Error + end. + +%% @private Activate venv with known site-packages path +activate_venv_with_site_packages(VenvBin, SitePackages) -> + %% Verify site-packages exists + case py:eval(<<"__import__('os').path.isdir(sp)">>, #{sp => SitePackages}) of + {ok, true} -> + %% Save original path if not already saved + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_original_path', __import__('sys').path.copy()) if not hasattr(__import__('sys'), '_original_path') else None">>), + %% Set venv info + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_active_venv', vp)">>, #{vp => VenvBin}), + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_venv_site_packages', sp)">>, #{sp => SitePackages}), + %% Add site-packages and process .pth files (editable installs) + %% Note: We embed the site-packages path directly since exec doesn't support + %% variables and sys attributes may not persist across calls in subinterpreters + SitePackagesStr = binary_to_list(SitePackages), + ExecCode = iolist_to_binary([ + <<"import site as _site, sys as _sys\n">>, + <<"_sp = '">>, escape_python_string(SitePackagesStr), <<"'\n">>, + <<"_b = frozenset(_sys.path)\n">>, + <<"_site.addsitedir(_sp)\n">>, + <<"_sys.path[:] = [p for p in _sys.path if p not in _b] + [p for p in _sys.path if p in _b]\n">>, + <<"del _site, _sys, _b, _sp\n">> + ]), + ok = py:exec(ExecCode), + ok; + {ok, false} -> + {error, {invalid_venv, SitePackages}}; + Error -> + Error + end. + +%% @private Escape a string for embedding in Python code +escape_python_string(Str) -> + lists:flatmap(fun($') -> "\\'"; + ($\\) -> "\\\\"; + (C) -> [C] + end, Str). + + + + +%% @doc Deactivate the current virtual environment. +%% Restores sys.path to its original state. +-spec deactivate_venv() -> ok | {error, term()}. +deactivate_venv() -> + case py:eval(<<"hasattr(__import__('sys'), '_original_path')">>) of + {ok, true} -> + ok = py:exec(<<"import sys as _sys\n" + "_sys.path[:] = _sys._original_path\n" + "del _sys\n">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_original_path')">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_active_venv') if hasattr(__import__('sys'), '_active_venv') else None">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_venv_site_packages') if hasattr(__import__('sys'), '_venv_site_packages') else None">>), + ok; + {ok, false} -> + ok; + Error -> + Error + end. + +%% @doc Get information about the currently active virtual environment. +%% Returns a map with venv_path and site_packages, or none if no venv is active. +-spec venv_info() -> {ok, map() | none} | {error, term()}. +venv_info() -> + %% Check both attributes exist to handle partial activation/deactivation state + Code = <<"({'active': True, 'venv_path': __import__('sys')._active_venv, 'site_packages': __import__('sys')._venv_site_packages, 'sys_path': __import__('sys').path} if (hasattr(__import__('sys'), '_active_venv') and hasattr(__import__('sys'), '_venv_site_packages')) else {'active': False})">>, + py:eval(Code). + +