diff --git a/Makefile b/Makefile index 6f7e0e2..52b8725 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,17 @@ C_SRC := $(shell pwd)/c_src CPPFLAGS := -shared -fPIC -fvisibility=hidden -std=c++17 -Wall -Wextra -Wno-unused-parameter -Wno-comment CPPFLAGS += -I$(ERTS_INCLUDE_DIR) -I$(FINE_INCLUDE_DIR) +# PYTHONX_BINARIES is always set by make_env in mix.exs. +# "safe" copies bytes (no dangling pointer risk after finalization). +# "fast" uses zero-copy resource binaries (existing behavior). +ifeq ($(PYTHONX_BINARIES),safe) + CPPFLAGS += -DPYTHONX_SAFE_BINARIES +else ifeq ($(PYTHONX_BINARIES),fast) + CPPFLAGS += -DPYTHONX_FAST_BINARIES +else +$(error PYTHONX_BINARIES must be "safe" or "fast") +endif + ifdef DEBUG CPPFLAGS += -g else @@ -22,9 +33,14 @@ endif SOURCES := $(wildcard $(C_SRC)/*.cpp) HEADERS := $(wildcard $(C_SRC)/*.hpp) +# Stamp file written by make_env (mix.exs) recording the current +# PYTHONX_BINARIES value. A change in the config triggers a recompile +# because the NIF target depends on this file's timestamp. +BINARIES_STAMP := $(MIX_MANIFEST_PATH)/pythonx_binaries.stamp + all: $(NIF_PATH) @ echo > /dev/null # Dummy command to avoid the default output "Nothing to be done" -$(NIF_PATH): $(SOURCES) $(HEADERS) +$(NIF_PATH): $(SOURCES) $(HEADERS) $(BINARIES_STAMP) @ mkdir -p $(PRIV_DIR) $(CXX) $(CPPFLAGS) $(SOURCES) -o $(NIF_PATH) diff --git a/README.md b/README.md index 7f516ba..a89b9fe 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,42 @@ Note that currently the `~PY` sigil does not work as part of Mix project code. This limitation is intentional, since in actual applications it is preferable to manage the Python globals explicitly. +### Finalization + +When a normal Python executable terminates, it runs a sophisticated shutdown +sequence called "finalization". This is what triggers most of the cleanup +behavior you might expect. This includes things like as object destructors, +`atexit` handlers and module finalizers. + +Without finalizing, many libraries can misbehave subtly--including the Python +standard library! + +Some real examples observed include: +- warnings about leaked `multiprocessing` semaphores +- log messages not actually emitted +- `readline` history not written +- files created with `tempfile` aren't deleted +- changes to a `shelve` database aren't persisted +- files and sockets aren't closed +- context managers in generators may not call their exit hooks + +When running under a full OTP supervision tree (i.e. in most apps + IEx), +Pythonx automatically finalizes the interpreter as part of a graceful shutdown. + +Some environments don't run a full supervision tree. Typically that will be +things like exscripts and Mix tasks. In these situations, you can also call +`Pythonx.finalize/0` manually. + +After finalization, the interpreter can be re-initialized with +`Pythonx.uv_init/2` or `Pythonx.init/4`. Objects from the previous interpreter +session are detected via an internal generation counter and rejected with a +clear error message. + +Re-initialization is reliable for pure-Python code, but C extensions with +process-global state (`numpy`, `torch`) may fail or crash on their second +import; see the "Re-initialization limits" section in the `Pythonx.finalize/0` +docs. + ## Python API Pythonx provides a Python module named `pythonx` with extra interoperability diff --git a/c_src/main_thread.hpp b/c_src/main_thread.hpp new file mode 100644 index 0000000..fc17775 --- /dev/null +++ b/c_src/main_thread.hpp @@ -0,0 +1,144 @@ +// A native thread owned by pythonx that acts as CPython's "main thread". +// +// CPython binds the thread that runs Py_InitializeEx to +// runtime->main_tstate, and Py_FinalizeEx must run on that same +// thread: when called from any other thread, CPython 3.13 does not +// swap the main thread state in, frees the calling thread's state as +// a "non-main" one, and then dereferences it while flushing std files. +// NIFs run on whichever dirty scheduler picks them up, and there is no +// way to pin two separate NIF calls to the same scheduler thread, so +// we own one thread for exactly those two calls. Everything else +// (eval, decode, sys.path setup) keeps running on the dirty +// schedulers with their own thread states. +// +// The thread is started in init() and joined in finalize(), so every +// finalize + re-init cycle gets a fresh main thread. It is only ever +// used while init_mutex is held, so a single-slot job handoff suffices. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__linux__) +#include +#endif + +namespace pythonx { + +class MainThread { + std::thread thread; + std::mutex mutex; + std::condition_variable cv; + std::function job; + bool stop = false; + bool has_job = false; + bool done = false; + std::exception_ptr error; + + void loop() { +#if defined(__APPLE__) + pthread_setname_np("pythonx-main"); +#elif defined(__linux__) + pthread_setname_np(pthread_self(), "pythonx-main"); +#endif + + while (true) { + std::function current_job; + + { + auto lock = std::unique_lock(mutex); + cv.wait(lock, [&] { return has_job || stop; }); + + if (!has_job) { + return; + } + + current_job = std::move(job); + has_job = false; + } + + std::exception_ptr current_error; + + try { + current_job(); + } catch (...) { + current_error = std::current_exception(); + } + + { + auto lock = std::lock_guard(mutex); + error = current_error; + done = true; + } + + cv.notify_all(); + } + } + +public: + // Spawns the loop thread. Must not be called while a previous + // thread is still running; finalize() joins before init() starts + // again. + void start() { + if (thread.joinable()) { + throw std::runtime_error("pythonx main thread is already running"); + } + + { + auto lock = std::lock_guard(mutex); + stop = false; + has_job = false; + done = false; + error = nullptr; + } + + thread = std::thread([this] { loop(); }); + } + + // Runs fn on the main thread and blocks until it returns. An + // exception thrown by fn is rethrown on the calling thread. + void run(std::function fn) { + if (!thread.joinable()) { + throw std::runtime_error("pythonx main thread is not running"); + } + + auto lock = std::unique_lock(mutex); + job = std::move(fn); + has_job = true; + done = false; + error = nullptr; + cv.notify_all(); + + cv.wait(lock, [&] { return done; }); + + auto current_error = error; + error = nullptr; + lock.unlock(); + + if (current_error) { + std::rethrow_exception(current_error); + } + } + + // Stops the loop and joins the thread. The object can be started + // again afterwards. + void join() { + { + auto lock = std::lock_guard(mutex); + stop = true; + } + + cv.notify_all(); + + if (thread.joinable()) { + thread.join(); + } + } +}; + +} // namespace pythonx diff --git a/c_src/python.cpp b/c_src/python.cpp index e30139c..3f719bf 100644 --- a/c_src/python.cpp +++ b/c_src/python.cpp @@ -28,6 +28,7 @@ DEF_SYMBOL(PyDict_SetItem) DEF_SYMBOL(PyDict_SetItemString) DEF_SYMBOL(PyDict_Size) DEF_SYMBOL(PyErr_Clear) +DEF_SYMBOL(PyErr_SetString) DEF_SYMBOL(PyErr_Fetch) DEF_SYMBOL(PyErr_Occurred) DEF_SYMBOL(PyEval_GetBuiltins) @@ -74,12 +75,16 @@ DEF_SYMBOL(Py_CompileString) DEF_SYMBOL(Py_DecRef) DEF_SYMBOL(Py_IncRef) DEF_SYMBOL(Py_InitializeEx) +DEF_SYMBOL(Py_FinalizeEx) DEF_SYMBOL(Py_IsFalse) DEF_SYMBOL(Py_IsNone) DEF_SYMBOL(Py_IsTrue) DEF_SYMBOL(Py_SetPythonHome) DEF_SYMBOL(Py_SetProgramName) +// Exception objects +DEF_SYMBOL(PyExc_RuntimeError) + dl::LibraryHandle python_library; void load_python_library(std::string path) { @@ -103,6 +108,7 @@ void load_python_library(std::string path) { LOAD_SYMBOL(python_library, PyDict_SetItemString) LOAD_SYMBOL(python_library, PyDict_Size) LOAD_SYMBOL(python_library, PyErr_Clear) + LOAD_SYMBOL(python_library, PyErr_SetString) LOAD_SYMBOL(python_library, PyErr_Fetch) LOAD_SYMBOL(python_library, PyErr_Occurred) LOAD_SYMBOL(python_library, PyEval_GetBuiltins) @@ -149,11 +155,15 @@ void load_python_library(std::string path) { LOAD_SYMBOL(python_library, Py_DecRef) LOAD_SYMBOL(python_library, Py_IncRef) LOAD_SYMBOL(python_library, Py_InitializeEx) + LOAD_SYMBOL(python_library, Py_FinalizeEx) LOAD_SYMBOL(python_library, Py_IsFalse) LOAD_SYMBOL(python_library, Py_IsNone) LOAD_SYMBOL(python_library, Py_IsTrue) LOAD_SYMBOL(python_library, Py_SetPythonHome) LOAD_SYMBOL(python_library, Py_SetProgramName) + + // Exception objects + LOAD_SYMBOL(python_library, PyExc_RuntimeError) } void unload_python_library() { diff --git a/c_src/python.hpp b/c_src/python.hpp index 70b5d95..6ce39e5 100644 --- a/c_src/python.hpp +++ b/c_src/python.hpp @@ -82,6 +82,7 @@ extern int (*PyDict_SetItem)(PyObjectPtr, PyObjectPtr, PyObjectPtr); extern int (*PyDict_SetItemString)(PyObjectPtr, const char *, PyObjectPtr); extern Py_ssize_t (*PyDict_Size)(PyObjectPtr); extern void (*PyErr_Clear)(); +extern void (*PyErr_SetString)(PyObjectPtr, const char *); extern void (*PyErr_Fetch)(PyObjectPtr *, PyObjectPtr *, PyObjectPtr *); extern PyObjectPtr (*PyErr_Occurred)(); extern PyObjectPtr (*PyEval_GetBuiltins)(); @@ -128,12 +129,16 @@ extern PyObjectPtr (*Py_CompileString)(const char *, const char *, int); extern void (*Py_DecRef)(PyObjectPtr); extern void (*Py_IncRef)(PyObjectPtr); extern void (*Py_InitializeEx)(int); +extern int (*Py_FinalizeEx)(); extern int (*Py_IsFalse)(PyObjectPtr); extern int (*Py_IsNone)(PyObjectPtr); extern int (*Py_IsTrue)(PyObjectPtr); extern void (*Py_SetPythonHome)(const wchar_t *); extern void (*Py_SetProgramName)(const wchar_t *); +// Exception objects (global variables, not functions) +extern PyObjectPtr PyExc_RuntimeError; + // Opens Python dynamic library at the given path and looks up all // relevant symbols. // diff --git a/c_src/pythonx.cpp b/c_src/pythonx.cpp index 8ff48b5..935db92 100644 --- a/c_src/pythonx.cpp +++ b/c_src/pythonx.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -8,9 +10,11 @@ #include #include #include +#include #include #include +#include "main_thread.hpp" #include "python.hpp" extern "C" void pythonx_handle_io_write(const char *message, @@ -28,14 +32,49 @@ using namespace python; // State std::mutex init_mutex; bool is_initialized = false; +// Incremented (set to a steady_clock timestamp) each time the interpreter +// is initialized. Set to 0 during finalization and when not initialized. +// Stamped into PyObjectResource so that objects from a previous generation +// (before finalize + re-init) are detected and rejected. +// Only written while holding init_mutex. +uint64_t init_generation = 0; std::wstring python_home_path_w; std::wstring python_executable_path_w; std::map> compilation_cache; std::mutex compilation_cache_mutex; PyInterpreterStatePtr interpreter_state; +// Thread states for the dirty scheduler threads, created lazily by +// PyGILGuard. The main thread state lives in main_thread_state, not here. std::map thread_states; std::mutex thread_states_mutex; +// Intentionally leaked. If the process exits without finalize() (for +// example with finalization disabled, or via System.halt/1), the +// thread is still joinable, and destroying a joinable std::thread +// during static destruction calls std::terminate, aborting the VM +// on its way out. The thread dies with the process. +MainThread &main_thread = *new MainThread(); +// Thread state created by Py_InitializeEx on main_thread, saved with +// PyEval_SaveThread. Restored on the same thread before Py_FinalizeEx. +PyThreadStatePtr main_thread_state = nullptr; + +// Active thread count for finalize. ensure_initialized() increments +// this under init_mutex (via ActiveThreadGuard) before returning. +// janitor_decref() also increments this (without init_mutex) so that +// finalize() waits for in-flight decref calls that may be blocked on +// the GIL. ActiveThreadGuard's destructor decrements it under +// active_threads_mutex. finalize() holds init_mutex for its entire +// duration and polls this counter to wait for in-flight threads. +std::mutex active_threads_mutex; +int active_threads = 0; + +// Atomic count of live PyObjectResource instances. Incremented by +// make_pyobject, decremented by the resource destructor. Read by the +// finalize NIF (under init_mutex) to report how many resource-backed +// binaries survive finalization — a non-zero count means dangling +// pointers are possible if PYTHONX_FAST_BINARIES is in effect. +std::atomic resource_count{0}; + // Wrapper around the Python Global Interpreter Lock (GIL). // // To acquire the GIL, the caller simply needs to initialize a new @@ -125,12 +164,36 @@ class PyDecRefGuard { } }; -void ensure_initialized() { - auto init_guard = std::lock_guard(init_mutex); +// RAII guard for the active thread count. Returned by ensure_initialized(). +// Its destructor decrements active_threads, signaling finalize() that +// this thread has left Python code. +class ActiveThreadGuard { +public: + ActiveThreadGuard() { + auto guard = std::lock_guard(active_threads_mutex); + active_threads++; + } + + ~ActiveThreadGuard() { + auto guard = std::lock_guard(active_threads_mutex); + active_threads--; + } + ActiveThreadGuard(const ActiveThreadGuard &) = delete; + ActiveThreadGuard &operator=(const ActiveThreadGuard &) = delete; +}; + +// Checks that the interpreter is initialized and returns an +// ActiveThreadGuard that keeps active_threads incremented for the +// duration of the NIF call. The increment happens under init_mutex, +// so finalize() (which holds init_mutex for its entire duration) sees +// a stable count. +ActiveThreadGuard ensure_initialized() { + auto init_guard = std::lock_guard(init_mutex); if (!is_initialized) { throw std::runtime_error("Python interpreter has not been initialized"); } + return ActiveThreadGuard(); } namespace atoms { @@ -154,19 +217,25 @@ auto value = fine::Atom("value"); struct PyObjectResource { PyObjectPtr py_object; + uint64_t generation; - PyObjectResource(PyObjectPtr py_object) : py_object(py_object) {} + PyObjectResource(PyObjectPtr py_object, uint64_t generation) + : py_object(py_object), generation(generation) {} void destructor(ErlNifEnv *env) { + // Always decrement the resource count so finalize can report an + // accurate leftover count. + resource_count.fetch_sub(1, std::memory_order_relaxed); + // Decrementing refcount requires GIL and we should not block in // the destructor, so we send a message to a known process and let // it decrement the refcount for us. Also see [1]. // // [1]:https://erlangforums.com/t/how-to-deal-with-destructors-that-can-take-a-while-to-run-and-possibly-block-the-scheduler/4290 - if (!is_initialized) { - // If we allow multiple initializations, we need to add a counter - // and check that py_object comes from the current initialization + // Skip if the interpreter is not initialized or the object is + // from a previous generation (before finalize + re-init). + if (!is_initialized || generation != init_generation || generation == 0) { return; } @@ -189,6 +258,12 @@ struct PyObjectResource { FINE_RESOURCE(PyObjectResource); +// Factory for creating PyObjectResource with the current init_generation. +fine::ResourcePtr make_pyobject(PyObjectPtr ptr) { + resource_count.fetch_add(1, std::memory_order_relaxed); + return fine::make_resource(ptr, init_generation); +} + // A resource that notifies the given process upon garbage collection. struct GCNotifier { ErlNifPid pid; @@ -246,6 +321,17 @@ struct ExError { static constexpr auto is_exception = true; }; +// Validates that an ExObject's resource is from the current +// interpreter generation. Throws if the object is stale (from a +// previous init, or created during init/finalize when generation is 0). +void validate_generation(const ExObject &obj) { + if (obj.resource->generation != init_generation || init_generation == 0) { + throw std::runtime_error( + "Pythonx object is from a previous interpreter generation " + "and is no longer valid"); + } +} + struct EvalInfo { fine::Term stdout_device; fine::Term stderr_device; @@ -273,9 +359,9 @@ ExError build_py_error_from_current(ErlNifEnv *env) { py_traceback = py_traceback == NULL ? Py_BuildValue("") : py_traceback; auto lines = py_error_lines(env, py_type, py_value, py_traceback); - auto type = fine::make_resource(py_type); - auto value = fine::make_resource(py_value); - auto traceback = fine::make_resource(py_traceback); + auto type = make_pyobject(py_type); + auto value = make_pyobject(py_value); + auto traceback = make_pyobject(py_traceback); return ExError(lines, type, value, traceback); } @@ -302,6 +388,37 @@ void raise_if_failed(ErlNifEnv *env, Py_ssize_t size) { } } +#if defined(PYTHONX_SAFE_BINARIES) + +ERL_NIF_TERM py_str_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { + Py_ssize_t size; + auto buffer = PyUnicode_AsUTF8AndSize(py_object, &size); + raise_if_failed(env, buffer); + + // Safe path: copy the bytes into a new Elixir binary. This avoids + // the dangling-pointer risk after finalization, at the cost of a memcpy. + ERL_NIF_TERM term; + auto *dst = enif_make_new_binary(env, size, &term); + std::memcpy(dst, buffer, size); + return term; +} + +ERL_NIF_TERM py_bytes_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { + Py_ssize_t size; + char *buffer; + auto result = PyBytes_AsStringAndSize(py_object, &buffer, &size); + raise_if_failed(env, result); + + // Safe path: copy the bytes into a new Elixir binary. This avoids + // the dangling-pointer risk after finalization, at the cost of a memcpy. + ERL_NIF_TERM term; + auto *dst = enif_make_new_binary(env, size, &term); + std::memcpy(dst, buffer, size); + return term; +} + +#elif defined(PYTHONX_FAST_BINARIES) + ERL_NIF_TERM py_str_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { Py_ssize_t size; auto buffer = PyUnicode_AsUTF8AndSize(py_object, &size); @@ -309,8 +426,14 @@ ERL_NIF_TERM py_str_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { // The buffer is immutable and lives as long as the Python object, // so we create the term as a resource binary to make it zero-copy. + // This is the fast path (PYTHONX_FAST_BINARIES): the resource binary + // keeps a pointer into the Python object's internal buffer. After + // finalization, that buffer is freed, so any Elixir binary still + // referencing it becomes a dangling pointer. The PYTHONX_SAFE_BINARIES + // macro switches to a copying implementation (enif_make_new_binary) + // that avoids this risk at the cost of a memcpy. Py_IncRef(py_object); - auto ex_object_resource = fine::make_resource(py_object); + auto ex_object_resource = make_pyobject(py_object); return fine::make_resource_binary(env, ex_object_resource, buffer, size); } @@ -322,11 +445,21 @@ ERL_NIF_TERM py_bytes_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { // The buffer is immutable and lives as long as the Python object, // so we create the term as a resource binary to make it zero-copy. + // This is the fast path (PYTHONX_FAST_BINARIES): the resource binary + // keeps a pointer into the Python object's internal buffer. After + // finalization, that buffer is freed, so any Elixir binary still + // referencing it becomes a dangling pointer. The PYTHONX_SAFE_BINARIES + // macro switches to a copying implementation (enif_make_new_binary) + // that avoids this risk at the cost of a memcpy. Py_IncRef(py_object); - auto ex_object_resource = fine::make_resource(py_object); + auto ex_object_resource = make_pyobject(py_object); return fine::make_resource_binary(env, ex_object_resource, buffer, size); } +#else +#error "Either PYTHONX_FAST_BINARIES or PYTHONX_SAFE_BINARIES must be defined." +#endif + std::vector py_error_lines(ErlNifEnv *env, PyObjectPtr py_type, PyObjectPtr py_value, PyObjectPtr py_traceback) { @@ -412,23 +545,30 @@ fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path, Py_SetPythonHome(python_home_path_w.c_str()); Py_SetProgramName(python_executable_path_w.c_str()); - Py_InitializeEx(0); - - interpreter_state = PyInterpreterState_Get(); - + // Initialize the interpreter on the pythonx-owned main thread (see + // MainThread). The thread that runs Py_InitializeEx becomes + // CPython's main thread, and Py_FinalizeEx later has to run on it. + // // In order to use any of the Python C API functions, the calling // thread must hold the GIL. Since every NIF call may run on a // different dirty scheduler thread, we need to acquire the GIL at // the beginning of each NIF and release it afterwards. // - // After initializing the Python interpreter above, the current - // thread automatically holds the GIL, so we explicitly release it. - // See pyo3 [1] for an extra reference. + // After initializing the Python interpreter, the main thread + // automatically holds the GIL, so we explicitly release it and keep + // its thread state for finalize(). See pyo3 [1] for an extra + // reference. // // [1]: https://github.com/PyO3/pyo3/blob/v0.23.3/src/gil.rs#L63-L74 - thread_states[std::this_thread::get_id()] = PyEval_SaveThread(); + main_thread.start(); + main_thread.run([&] { + Py_InitializeEx(0); + interpreter_state = PyInterpreterState_Get(); + main_thread_state = PyEval_SaveThread(); + }); is_initialized = true; + init_generation = std::chrono::steady_clock::now().time_since_epoch().count(); // We still hold the init_mutex, so we can obtain the GIL guard // before any other concurrent NIF. At this point we marked the @@ -436,6 +576,9 @@ fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path, // preparation using Python APIs. If any exception is subsequently // raised, it will propagate as expected, and since the interpreter // is initialized, the exception formatting will also work. + // + // The guard gives this dirty scheduler thread its own thread state, + // as it does for any other NIF call. auto gil_guard = PyGILGuard(); // Add extra paths to sys.path @@ -584,112 +727,251 @@ sys.modules["pythonx"] = pythonx FINE_NIF(init, ERL_NIF_DIRTY_JOB_CPU_BOUND); -fine::Ok<> janitor_decref(ErlNifEnv *env, uint64_t ptr) { +std::tuple finalize(ErlNifEnv *env) { auto init_guard = std::lock_guard(init_mutex); - // If the interpreter is no longer initialized, ignore the call - if (is_initialized) { - auto gil_guard = PyGILGuard(); + if (!is_initialized) { + return std::make_tuple(int64_t(0), int64_t(0)); + } + + // Clear init_generation first, so that in-flight threads calling + // send_tagged_object fail fast (preventing callback deadlock) and + // destructors skip decref. This is the first mutation, before + // is_initialized is set to false. + init_generation = 0; + + // Notify the Janitor to skip decref calls during finalization. + // This is sent from the NIF (rather than the Finalizer GenServer) + // so that it works regardless of whether finalize is called from + // the GenServer or directly by the user. + { + auto janitor_name = fine::encode(env, atoms::ElixirPythonxJanitor); + ErlNifPid janitor_pid; + if (enif_whereis_pid(env, janitor_name, &janitor_pid)) { + auto msg_env = enif_alloc_env(); + auto msg = fine::encode(msg_env, fine::Atom("finalizing")); + enif_send(env, &janitor_pid, msg_env, msg); + enif_free_env(msg_env); + } + } + + // Mark as not initialized so no new NIF calls can enter Python. + is_initialized = false; + + // Wait for all in-flight threads to finish. + // ensure_initialized() happens under init_mutex (which we hold), + // so no new threads can increment. Threads that already passed + // ensure_initialized() hold an ActiveThreadGuard that will + // decrement active_threads when their NIF returns. + // + // This mirrors what Python does before Py_FinalizeEx: join threads. + // If a thread is stuck in an infinite loop, this blocks indefinitely, + // same as sys.exit() in standard Python. + while (true) { + int count; + { + auto guard = std::lock_guard(active_threads_mutex); + count = active_threads; + } + if (count == 0) break; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + // All in-flight threads have released the GIL. No new threads can + // enter (is_initialized is false, init_mutex is held). Safe to + // tear down the interpreter. - auto object = reinterpret_cast(ptr); + // Clear compilation cache (Python objects freed by Py_FinalizeEx) + { + auto guard = std::lock_guard(compilation_cache_mutex); + compilation_cache.clear(); + } - Py_DecRef(object); + // Run full CPython shutdown on the main thread (see MainThread), + // with the GIL held through the thread state Py_InitializeEx + // created there. Py_FinalizeEx runs atexit handlers, module + // finalizers, ResourceTracker.__del__ (stops the daemon) and + // releases the GIL. Returns 0 on success, -1 if finalization had + // errors (e.g., flushing buffered data failed). + // + // We cannot use PyGILGuard because its destructor would call + // PyEval_SaveThread after the interpreter is torn down. + int result = 0; + main_thread.run([&] { + PyEval_RestoreThread(main_thread_state); + result = Py_FinalizeEx(); + }); + + // Nothing else will run on the main thread for this generation. + // Joining here means the next init() starts a fresh one. + main_thread.join(); + + // Py_FinalizeEx has freed all interpreter state and thread states. + // Clear our pointers to prevent use-after-free. + interpreter_state = nullptr; + main_thread_state = nullptr; + { + auto guard = std::lock_guard(thread_states_mutex); + thread_states.clear(); } + // Notify the Janitor that finalization is complete, so it resumes + // normal decref handling. This is sent while init_mutex is still + // held, so no re-init can run before the Janitor receives this. + { + auto janitor_name = fine::encode(env, atoms::ElixirPythonxJanitor); + ErlNifPid janitor_pid; + if (enif_whereis_pid(env, janitor_name, &janitor_pid)) { + auto msg_env = enif_alloc_env(); + auto msg = fine::encode(msg_env, fine::Atom("finalized")); + enif_send(env, &janitor_pid, msg_env, msg); + enif_free_env(msg_env); + } + } + + // Intentionally do NOT call PyEval_SaveThread — the GIL and + // thread state are already destroyed by Py_FinalizeEx. + + // Read resource_count while init_mutex is still held. A non-zero + // count means resource-backed binaries may still reference freed + // Python memory (only relevant under PYTHONX_FAST_BINARIES). + int leftover = resource_count.load(std::memory_order_relaxed); + + return std::make_tuple(int64_t(result), int64_t(leftover)); +} + +FINE_NIF(finalize, ERL_NIF_DIRTY_JOB_CPU_BOUND); + +fine::Ok<> janitor_decref(ErlNifEnv *env, uint64_t ptr) { + // Check is_initialized without acquiring init_mutex. This avoids + // blocking the Janitor process during finalization (which holds + // init_mutex for its entire duration). If is_initialized is false + // or init_generation is 0, the interpreter is being or has been + // finalized and Py_FinalizeEx has freed all objects, so we skip. + // + // The read of is_initialized and init_generation is not protected + // by a mutex, but this is safe: both are written under init_mutex + // and the worst case is a stale read that either skips a decref + // (harmless — Py_FinalizeEx frees everything) or proceeds with one + // (also fine — the object is still alive). The generation check in + // the destructor already prevents stale objects from reaching here. + if (!is_initialized || init_generation == 0) { + return fine::Ok<>(); + } + + // Increment active_threads so that finalize() waits for this call + // to complete before tearing down the interpreter. Without this, + // finalize() could see active_threads == 0, acquire the GIL, and + // call Py_FinalizeEx while we are blocked in PyGILGuard() waiting + // for the GIL — a deadlock, since Py_FinalizeEx destroys the GIL. + auto thread_guard = ActiveThreadGuard(); + + auto gil_guard = PyGILGuard(); + + // Re-check after acquiring the GIL, in case finalization started + // while we were waiting for the GIL. + if (!is_initialized || init_generation == 0) { + return fine::Ok<>(); + } + + auto object = reinterpret_cast(ptr); + + Py_DecRef(object); + return fine::Ok<>(); } FINE_NIF(janitor_decref, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject none_new(ErlNifEnv *env) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); // Note that Limited API has Py_GetConstant, but only since v3.13 auto py_none = Py_BuildValue(""); raise_if_failed(env, py_none); - return ExObject(fine::make_resource(py_none)); + return ExObject(make_pyobject(py_none)); } FINE_NIF(none_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject false_new(ErlNifEnv *env) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_bool = PyBool_FromLong(0); raise_if_failed(env, py_bool); - return ExObject(fine::make_resource(py_bool)); + return ExObject(make_pyobject(py_bool)); } FINE_NIF(false_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject true_new(ErlNifEnv *env) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_bool = PyBool_FromLong(1); raise_if_failed(env, py_bool); - return ExObject(fine::make_resource(py_bool)); + return ExObject(make_pyobject(py_bool)); } FINE_NIF(true_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject long_from_int64(ErlNifEnv *env, int64_t number) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_long = PyLong_FromLongLong(number); raise_if_failed(env, py_long); - return ExObject(fine::make_resource(py_long)); + return ExObject(make_pyobject(py_long)); } FINE_NIF(long_from_int64, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject long_from_string(ErlNifEnv *env, std::string string, int64_t base) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_long = PyLong_FromString(string.c_str(), NULL, static_cast(base)); raise_if_failed(env, py_long); - return ExObject(fine::make_resource(py_long)); + return ExObject(make_pyobject(py_long)); } FINE_NIF(long_from_string, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject float_new(ErlNifEnv *env, double number) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_float = PyFloat_FromDouble(number); raise_if_failed(env, py_float); - return ExObject(fine::make_resource(py_float)); + return ExObject(make_pyobject(py_float)); } FINE_NIF(float_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject bytes_from_binary(ErlNifEnv *env, ErlNifBinary binary) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_object = PyBytes_FromStringAndSize( reinterpret_cast(binary.data), binary.size); raise_if_failed(env, py_object); - return ExObject(fine::make_resource(py_object)); + return ExObject(make_pyobject(py_object)); } FINE_NIF(bytes_from_binary, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject unicode_from_string(ErlNifEnv *env, ErlNifBinary binary) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_object = PyUnicode_FromStringAndSize( @@ -697,13 +979,14 @@ ExObject unicode_from_string(ErlNifEnv *env, ErlNifBinary binary) { raise_if_failed(env, py_object); - return ExObject(fine::make_resource(py_object)); + return ExObject(make_pyobject(py_object)); } FINE_NIF(unicode_from_string, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Term unicode_to_string(ErlNifEnv *env, ExObject ex_object) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); auto gil_guard = PyGILGuard(); return py_str_to_binary_term(env, ex_object.resource->py_object); @@ -712,20 +995,23 @@ fine::Term unicode_to_string(ErlNifEnv *env, ExObject ex_object) { FINE_NIF(unicode_to_string, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject dict_new(ErlNifEnv *env) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_dict = PyDict_New(); raise_if_failed(env, py_dict); - return ExObject(fine::make_resource(py_dict)); + return ExObject(make_pyobject(py_dict)); } FINE_NIF(dict_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Ok<> dict_set_item(ErlNifEnv *env, ExObject ex_object, ExObject ex_key, ExObject ex_value) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); + validate_generation(ex_key); + validate_generation(ex_value); auto gil_guard = PyGILGuard(); auto result = @@ -739,20 +1025,22 @@ fine::Ok<> dict_set_item(ErlNifEnv *env, ExObject ex_object, ExObject ex_key, FINE_NIF(dict_set_item, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject tuple_new(ErlNifEnv *env, uint64_t size) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_tuple = PyTuple_New(size); raise_if_failed(env, py_tuple); - return ExObject(fine::make_resource(py_tuple)); + return ExObject(make_pyobject(py_tuple)); } FINE_NIF(tuple_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Ok<> tuple_set_item(ErlNifEnv *env, ExObject ex_object, uint64_t index, ExObject ex_value) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); + validate_generation(ex_value); auto gil_guard = PyGILGuard(); auto result = PyTuple_SetItem(ex_object.resource->py_object, index, @@ -768,20 +1056,22 @@ fine::Ok<> tuple_set_item(ErlNifEnv *env, ExObject ex_object, uint64_t index, FINE_NIF(tuple_set_item, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject list_new(ErlNifEnv *env, uint64_t size) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_tuple = PyList_New(size); raise_if_failed(env, py_tuple); - return ExObject(fine::make_resource(py_tuple)); + return ExObject(make_pyobject(py_tuple)); } FINE_NIF(list_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Ok<> list_set_item(ErlNifEnv *env, ExObject ex_object, uint64_t index, ExObject ex_value) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); + validate_generation(ex_value); auto gil_guard = PyGILGuard(); auto result = PyList_SetItem(ex_object.resource->py_object, index, @@ -797,19 +1087,21 @@ fine::Ok<> list_set_item(ErlNifEnv *env, ExObject ex_object, uint64_t index, FINE_NIF(list_set_item, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject set_new(ErlNifEnv *env) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_set = PySet_New(NULL); raise_if_failed(env, py_set); - return ExObject(fine::make_resource(py_set)); + return ExObject(make_pyobject(py_set)); } FINE_NIF(set_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Ok<> set_add(ErlNifEnv *env, ExObject ex_object, ExObject ex_key) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); + validate_generation(ex_key); auto gil_guard = PyGILGuard(); auto result = @@ -822,7 +1114,7 @@ fine::Ok<> set_add(ErlNifEnv *env, ExObject ex_object, ExObject ex_key) { FINE_NIF(set_add, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject pid_new(ErlNifEnv *env, ErlNifPid pid) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); // ErlNifPid is self-contained struct, not bound to any env, so it's @@ -847,25 +1139,27 @@ ExObject pid_new(ErlNifEnv *env, ErlNifPid pid) { auto py_pid = PyObject_Call(py_PID, py_PID_args, NULL); raise_if_failed(env, py_pid); - return ExObject(fine::make_resource(py_pid)); + return ExObject(make_pyobject(py_pid)); } FINE_NIF(pid_new, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject object_repr(ErlNifEnv *env, ExObject ex_object) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); auto gil_guard = PyGILGuard(); auto py_repr = PyObject_Repr(ex_object.resource->py_object); raise_if_failed(env, py_repr); - return ExObject(fine::make_resource(py_repr)); + return ExObject(make_pyobject(py_repr)); } FINE_NIF(object_repr, ERL_NIF_DIRTY_JOB_CPU_BOUND); fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); auto gil_guard = PyGILGuard(); auto py_object = ex_object.resource->py_object; @@ -947,7 +1241,7 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) { auto py_item = PyTuple_GetItem(py_object, i); raise_if_failed(env, py_item); Py_IncRef(py_item); - auto ex_item = ExObject(fine::make_resource(py_item)); + auto ex_item = ExObject(make_pyobject(py_item)); terms.push_back(fine::encode(env, ex_item)); } @@ -971,7 +1265,7 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) { auto py_item = PyList_GetItem(py_object, i); raise_if_failed(env, py_item); Py_IncRef(py_item); - auto ex_item = ExObject(fine::make_resource(py_item)); + auto ex_item = ExObject(make_pyobject(py_item)); terms.push_back(fine::encode(env, ex_item)); } @@ -996,10 +1290,10 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) { while (PyDict_Next(py_object, &pos, &py_key, &py_value)) { Py_IncRef(py_key); - auto ex_key = ExObject(fine::make_resource(py_key)); + auto ex_key = ExObject(make_pyobject(py_key)); Py_IncRef(py_value); - auto ex_value = ExObject(fine::make_resource(py_value)); + auto ex_value = ExObject(make_pyobject(py_value)); terms.push_back(fine::encode(env, std::make_tuple(ex_key, ex_value))); } @@ -1048,7 +1342,7 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) { while ((py_item = PyIter_Next(py_iter)) != NULL) { // Note that PyIter_Next already returns a new reference - auto ex_item = ExObject(fine::make_resource(py_item)); + auto ex_item = ExObject(make_pyobject(py_item)); terms.push_back(fine::encode(env, ex_item)); } @@ -1267,7 +1561,12 @@ std::tuple, fine::Term> eval(ErlNifEnv *env, ErlNifBinary code, std::string code_md5, std::vector> globals, fine::Term stdout_device, fine::Term stderr_device) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + + // Validate that all globals objects are from the current generation. + for (const auto &[_key, value] : globals) { + validate_generation(value); + } // Step 1: compile (or get cached result) @@ -1439,7 +1738,7 @@ eval(ErlNifEnv *env, ErlNifBinary code, std::string code_md5, if (py_last_expr_code != nullptr) { auto py_result = PyEval_EvalCode(py_last_expr_code, py_globals, py_globals); raise_if_failed(env, py_result); - result = ExObject(fine::make_resource(py_result)); + result = ExObject(make_pyobject(py_result)); } // Step 4: flat-decode globals @@ -1472,7 +1771,7 @@ eval(ErlNifEnv *env, ErlNifBinary code, std::string code_md5, // Incref before making the resource Py_IncRef(py_value); - auto ex_value = ExObject(fine::make_resource(py_value)); + auto ex_value = ExObject(make_pyobject(py_value)); value_terms.push_back(fine::encode(env, ex_value)); } @@ -1489,7 +1788,8 @@ FINE_NIF(eval, ERL_NIF_DIRTY_JOB_CPU_BOUND); std::variant, fine::Error> dump_object(ErlNifEnv *env, ExObject ex_object) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); + validate_generation(ex_object); auto gil_guard = PyGILGuard(); std::string pickle_module_name; @@ -1530,7 +1830,7 @@ dump_object(ErlNifEnv *env, ExObject ex_object) { FINE_NIF(dump_object, ERL_NIF_DIRTY_JOB_CPU_BOUND); ExObject load_object(ErlNifEnv *env, ErlNifBinary binary) { - ensure_initialized(); + auto thread_guard = ensure_initialized(); auto gil_guard = PyGILGuard(); auto py_pickle = PyImport_ImportModule("pickle"); @@ -1553,7 +1853,7 @@ ExObject load_object(ErlNifEnv *env, ErlNifBinary binary) { auto py_object = PyObject_Call(py_loads, py_loads_args, NULL); raise_if_failed(env, py_object); - return ExObject(fine::make_resource(py_object)); + return ExObject(make_pyobject(py_object)); } FINE_NIF(load_object, ERL_NIF_DIRTY_JOB_CPU_BOUND); @@ -1630,6 +1930,18 @@ extern "C" void pythonx_handle_send_tagged_object(const char *pid_bytes, const char *tag, pythonx::python::PyObjectPtr *py_object, const char *eval_info_bytes) { + // If finalization has started (init_generation == 0), raise a Python + // exception instead of sending. During Py_FinalizeEx's thread joining, + // Python threads spawned by user code may call send_tagged_object. + // The guard raises RuntimeError so the thread unblocks instead of + // touching BEAM APIs (enif_send, enif_alloc_env) with a torn-down + // interpreter, which would crash or deadlock. + if (pythonx::init_generation == 0) { + pythonx::python::PyErr_SetString(pythonx::python::PyExc_RuntimeError, + "Pythonx is finalizing, cannot send tagged object"); + return; + } + auto eval_info = eval_info_from_bytes(eval_info_bytes); auto caller_env = get_caller_env(eval_info); @@ -1642,7 +1954,7 @@ pythonx_handle_send_tagged_object(const char *pid_bytes, const char *tag, env, std::make_tuple( fine::Atom(tag), pythonx::ExObject( - fine::make_resource(py_object)))); + pythonx::make_pyobject(py_object)))); enif_send(caller_env, &pid, env, msg); enif_free_env(env); } diff --git a/lib/pythonx.ex b/lib/pythonx.ex index 8c16fe8..95270fd 100644 --- a/lib/pythonx.ex +++ b/lib/pythonx.ex @@ -8,6 +8,8 @@ defmodule Pythonx do @moduledoc readme_docs + @binaries Application.compile_env(:pythonx, :binaries, :fast) + alias Pythonx.Object @install_env_name "PYTHONX_INIT_STATE" @@ -220,6 +222,141 @@ defmodule Pythonx do Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths]) end + @doc """ + Finalizes the Python interpreter, releasing all resources. + + This runs the full CPython shutdown sequence, including atexit handlers + and module finalizers. After calling this function, the interpreter is + no longer initialized and must be re-initialized with `uv_init/2` or + `init/4` before any Python code can be evaluated. + + When the `:pythonx` application is running under OTP supervision, + finalization happens automatically on shutdown via the + `Pythonx.Finalizer` GenServer. You only need to call this function + manually if you are using Pythonx outside the application lifecycle, + such as in an `.exs` script, a Mix task, or a test. + + In those cases, call `Pythonx.finalize/0` before exiting so that + Python's shutdown sequence runs cleanly (e.g., to release + `multiprocessing` semaphores and stop the resource tracker daemon). + Use `System.stop/0` rather than `System.halt/0` to ensure the + application shutdown callback fires. + + Calling this function while the `:pythonx` application is running + raises an error, as tearing down the interpreter out from under the + supervision tree would leave the Janitor and ObjectTracker in an + inconsistent state. + + It is safe to call even if the interpreter was never initialized + (it is a no-op in that case). + + After finalization, any `Pythonx.Object` structs from the previous + interpreter session are stale and will raise an error if used. This + includes objects passed to `Pythonx.eval/3` as globals or decoded + with `Pythonx.decode/1`. + + If the interpreter is re-initialized after finalization, objects from + the previous session are detected via an internal generation counter + and rejected with a clear error message. + + ## Main thread + + CPython requires `Py_FinalizeEx` to run on the same OS thread that ran + `Py_InitializeEx`. NIFs run on whichever dirty scheduler picks them + up, so Pythonx starts a native thread of its own during + initialization and runs both calls there. That thread is what Python + sees as `threading.main_thread()`; code evaluated with `eval/3` runs + on dirty scheduler threads, which Python reports as `Dummy-N` + threads. Python-level signal handlers (`signal.signal/2`) only run on + the main thread while it executes Python code, and this thread never + does, so they do not run. The thread is joined by `finalize/0`, and a + subsequent initialization starts a fresh one. + + ## Re-initialization limits + + Finalizing and then initializing again is supported for the + interpreter itself and for pure-Python code, including the standard + library: imports, `threading`, `logging` and `atexit` all work in + the new interpreter. CPython documents that some memory is leaked on + every `Py_FinalizeEx` + `Py_InitializeEx` cycle, so a process that + cycles many times will grow. + + C extensions are not guaranteed to survive a cycle. CPython never + unloads an extension's shared library, so any process-global state + the extension set up during its first import is still there when + the new interpreter imports it again, and the extension may not be + prepared for that. Observed with CPython 3.13 on macOS arm64: + + * `numpy` 2.1.2 fails to import a second time with a clean + `RuntimeError: CPU dispatcher tracer already initlized`. + + * `torch` 2.14.0 crashes the VM on the second import, with a + segfault inside `libtorch_python.dylib`'s module initialization. + + Treat `finalize/0` followed by re-initialization as a tool for + processes that only use pure-Python code, or that exit after + finalizing. If an application depends on such extensions, restart + the OS process instead of cycling the interpreter. + + > #### Multiprocessing processes {: .warning} + > + > `Py_FinalizeEx` terminates daemonic child processes and joins + > non-daemonic ones, matching normal Python shutdown behavior. If + > non-daemonic `multiprocessing.Process` instances are still running, + > this call will block until they finish, same as `sys.exit()` in + > standard Python. + + > #### In-flight evaluations {: .warning} + > + > If Python code is being evaluated concurrently (from another + > Elixir process), this function waits for all in-flight evaluations + > to complete before finalizing. If an evaluation is stuck in an + > infinite loop, this call blocks indefinitely, same as `sys.exit()` + > in standard Python. + + > #### Resource cleanup {: .warning} + > + > `Py_FinalizeEx` runs `atexit` handlers and calls `__del__` on + > remaining objects, but CPython destroys modules in random order, + > so a `__del__` that calls into an already-finalized module may + > fail silently. OS-level resources (open file handles, sockets, + > subprocesses) that were not explicitly closed may survive + > finalization. They are reclaimed by the OS on process exit, but + > not on `finalize` + re-init cycles. Use context managers (`with` + > statements) or explicit `close()` calls in Python code to avoid + > this. + """ + @spec finalize() :: :ok | {:error, Pythonx.FinalizeError.t()} + def finalize do + if pythonx_started?() do + raise RuntimeError, + "Pythonx.finalize/0 cannot be called while the :pythonx " <> + "application is running. Finalization is handled " <> + "automatically on application shutdown. If you are using " <> + "Pythonx outside of OTP supervision (e.g., in a script or " <> + "Mix task), make sure the application is not started." + end + + __finalize__() + end + + @doc false + @spec __finalize__() :: :ok | {:error, Pythonx.FinalizeError.t()} + def __finalize__ do + {return_code, resource_count} = Pythonx.NIF.finalize() + + # A non-zero return_code always indicates failure. A non-zero + # resource_count only indicates failure when using :fast binaries, + # because resource-backed binaries may reference freed Python + # memory. With :safe binaries, leftover resources are harmless + # (the bytes were copied). + if return_code != 0 or (@binaries == :fast and resource_count > 0) do + {:error, %Pythonx.FinalizeError{return_code: return_code, resource_count: resource_count}} + else + :ok + end + end + @doc ~S''' Evaluates the Python `code`. diff --git a/lib/pythonx/application.ex b/lib/pythonx/application.ex index 4a01887..fc537a9 100644 --- a/lib/pythonx/application.ex +++ b/lib/pythonx/application.ex @@ -3,14 +3,17 @@ defmodule Pythonx.Application do use Application + @finalization Application.compile_env(:pythonx, :finalization, true) + @impl true def start(_type, _args) do enable_sigchld() - children = [ - Pythonx.Janitor, - Pythonx.ObjectTracker - ] + children = + [ + Pythonx.Janitor, + Pythonx.ObjectTracker + ] ++ if(@finalization, do: [Pythonx.Finalizer], else: []) opts = [strategy: :one_for_one, name: Pythonx.Supervisor] diff --git a/lib/pythonx/finalize_error.ex b/lib/pythonx/finalize_error.ex new file mode 100644 index 0000000..633a59c --- /dev/null +++ b/lib/pythonx/finalize_error.ex @@ -0,0 +1,46 @@ +defmodule Pythonx.FinalizeError do + @moduledoc """ + An exception returned by `Pythonx.finalize/0` when finalization fails. + + Carries both the `return_code` from `Py_FinalizeEx` and the + `resource_count` of `PyObjectResource` instances that survived + finalization. Either being non-zero indicates a problem: + + * A non-zero `return_code` means CPython's shutdown sequence + encountered an error (e.g., flushing buffered data failed). + * A non-zero `resource_count` means resource-backed binaries + may reference freed Python memory. This is only a concern + when `binaries` is configured as `:fast` (the default). + """ + + defexception [:return_code, :resource_count] + + @type t :: %__MODULE__{ + return_code: integer(), + resource_count: integer() + } + + @impl true + def message(%__MODULE__{return_code: return_code, resource_count: resource_count}) do + parts = [] + + parts = + if return_code != 0 do + ["Py_FinalizeEx returned exit code #{return_code}" | parts] + else + parts + end + + parts = + if resource_count > 0 do + ["#{resource_count} PyObjectResource instances survived finalization" | parts] + else + parts + end + + case parts do + [] -> "Pythonx finalization completed" + _ -> "Pythonx finalization failed: " <> Enum.join(parts, ", ") + end + end +end diff --git a/lib/pythonx/finalizer.ex b/lib/pythonx/finalizer.ex new file mode 100644 index 0000000..b2005d3 --- /dev/null +++ b/lib/pythonx/finalizer.ex @@ -0,0 +1,96 @@ +defmodule Pythonx.Finalizer do + @moduledoc false + + # Finalizer is a GenServer that finalizes the Python interpreter when the + # application stops. Placed after Janitor and ObjectTracker in the + # supervision tree, so it is stopped first (reverse order). + # + # Its terminate/3 callback calls Pythonx.__finalize__/0 (the unguarded + # internal function). The finalize NIF sends :finalizing to the Janitor + # before waiting for in-flight threads, and :finalized after Py_FinalizeEx + # completes, before releasing init_mutex. + # + # This way the Janitor coordination works regardless of whether finalize is + # called from this GenServer or directly. + # + # The finalize NIF itself runs on a dirty scheduler, but Py_FinalizeEx does + # not: CPython requires it on the thread that ran Py_InitializeEx, so the + # NIF hands both calls to a native thread pythonx owns (the one Python sees + # as threading.main_thread()) and joins it once finalization is done. This + # GenServer does not need to know which thread it is running on. + + use GenServer + + require Logger + + @name __MODULE__ + + @binaries Application.compile_env(:pythonx, :binaries, :fast) + @finalization Application.compile_env(:pythonx, :finalization, true) + + # Py_FinalizeEx runs atexit handlers and module teardown of arbitrary + # duration (a loaded ML framework can take well over the 5s default). + # If the supervisor gave up early, the NIF would keep running on its + # dirty scheduler while the rest of the tree, including the Janitor, + # is torn down and the VM halts underneath it. Wait for it; bounding + # shutdown time is the job of whatever supervises the OS process. + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + shutdown: :infinity + } + end + + def start_link(_opts) do + GenServer.start_link(__MODULE__, :ok, name: @name) + end + + @impl true + def init(:ok) do + # trap_exit must be true for terminate/3 to be called when the + # supervisor stops this child. + Process.flag(:trap_exit, true) + {:ok, :ok} + end + + @impl true + def terminate(_reason, _state) do + # Tell the Janitor to skip decref calls during finalization. + # Messages are delivered in order, so any decref messages + # already in the mailbox are processed normally before this. + send(Pythonx.Janitor, :finalizing) + + # Ping the Janitor to flush all pending messages (including any + # decref calls) before we finalize. This ensures no decref NIF + # is blocked on the GIL when Py_FinalizeEx destroys it. + Pythonx.Janitor.ping() + + # Finalize the Python interpreter. The NIF sends :finalized to + # the Janitor after Py_FinalizeEx completes, before releasing + # init_mutex. We call __finalize__/0 (the unguarded internal + # function) because the public finalize/0 refuses to run while + # the app is running. + # + # After __finalize__/0, if binaries == :fast and finalization is + # true, check the resource count (returned from the finalize NIF) + # and log a warning if non-zero. + case Pythonx.__finalize__() do + :ok -> + :ok + + {:error, %Pythonx.FinalizeError{return_code: return_code, resource_count: count}} -> + if @binaries == :fast and @finalization and count > 0 do + Logger.warning( + "Pythonx: Py_FinalizeEx (exit code #{return_code}) left #{count} " <> + "PyObjectResource instances alive. With :fast binaries, " <> + "resource-backed binaries may reference freed Python memory." + ) + else + Logger.warning("Pythonx: Py_FinalizeEx returned exit code #{return_code}") + end + + :ok + end + end +end diff --git a/lib/pythonx/janitor.ex b/lib/pythonx/janitor.ex index 9682987..c4d2e1a 100644 --- a/lib/pythonx/janitor.ex +++ b/lib/pythonx/janitor.ex @@ -22,7 +22,7 @@ defmodule Pythonx.Janitor do @impl true def init({}) do - {:ok, {}} + {:ok, %{finalizing: false}} end @impl true @@ -31,19 +31,36 @@ defmodule Pythonx.Janitor do end @impl true + def handle_info(:finalizing, state) do + # Finalization is starting. Skip decref calls from now on — + # Py_FinalizeEx will free all Python objects regardless. + # Output messages are still forwarded normally. + {:noreply, %{state | finalizing: true}} + end + + def handle_info(:finalized, state) do + # Finalization is complete. Resume normal decref handling. + {:noreply, %{state | finalizing: false}} + end + def handle_info({:decref, ptr}, state) do # After %Pythonx.Object{} is garbage collected, the C++ code # sends us a message to decrement refcount of the corresponding # Python object in a separate NIF call. For more details see # ExObjectResource::destructor in the C++ code. - Pythonx.NIF.janitor_decref(ptr) + # + # Skip during finalization — Py_FinalizeEx frees everything. + unless state.finalizing do + Pythonx.NIF.janitor_decref(ptr) + end {:noreply, state} end def handle_info({:output, output, device}, state) do # We send the IO request and continue without waiting for the IO - # reply. + # reply. Output is forwarded even during finalization, since + # Py_FinalizeEx may flush stdout/stderr from module finalizers. send(device, {:io_request, self(), make_ref(), {:put_chars, :unicode, output}}) {:noreply, state} end diff --git a/lib/pythonx/nif.ex b/lib/pythonx/nif.ex index 405f1fd..beca802 100644 --- a/lib/pythonx/nif.ex +++ b/lib/pythonx/nif.ex @@ -13,6 +13,8 @@ defmodule Pythonx.NIF do end def init(_python_dl_path, _python_home_path, _python_executable_path, _sys_paths), do: err!() + # returns {return_code, resource_count} (both ints) + def finalize(), do: err!() def janitor_decref(_ptr), do: err!() def none_new(), do: err!() def false_new(), do: err!() diff --git a/mix.exs b/mix.exs index b9dca84..88ccf99 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,32 @@ defmodule Pythonx.MixProject do compilers: [:elixir_make] ++ Mix.compilers(), docs: docs(), package: package(), - make_env: fn -> %{"FINE_INCLUDE_DIR" => Fine.include_dir()} end, + make_env: fn -> + binaries = Application.get_env(:pythonx, :binaries, :fast) + binaries_str = if(binaries == :safe, do: "safe", else: "fast") + + # Write a stamp file so that changing the :binaries config + # triggers a NIF recompile. The Makefile depends on this file's + # timestamp. Only write if the value changed, to avoid + # unnecessary recompiles on every mix compile. + manifest = Mix.Project.manifest_path() + stamp = Path.join(manifest, "pythonx_binaries.stamp") + + unless File.dir?(manifest) do + File.mkdir_p!(manifest) + end + + existing = if File.exists?(stamp), do: File.read!(stamp), else: nil + + if existing != binaries_str do + File.write!(stamp, binaries_str) + end + + %{ + "FINE_INCLUDE_DIR" => Fine.include_dir(), + "PYTHONX_BINARIES" => binaries_str + } + end, # Precompilation make_precompiler: {:nif, CCPrecompiler}, make_precompiler_url: "#{@github_url}/releases/download/v#{@version}/@{artefact_filename}", diff --git a/test/pythonx/finalize_test.exs b/test/pythonx/finalize_test.exs new file mode 100644 index 0000000..9c97575 --- /dev/null +++ b/test/pythonx/finalize_test.exs @@ -0,0 +1,430 @@ +defmodule Pythonx.FinalizeTest do + # These tests cannot be async because they tear down and re-initialize + # the global Python interpreter. + use ExUnit.Case, async: false + + # The test_helper.exs initializes the interpreter before tests run. + # After each test that calls __finalize__/0, we re-initialize so that + # subsequent tests (including async ones) have a working interpreter. + # We use __finalize__/0 (the unguarded internal function) because + # the public finalize/0 refuses to run while the app is running. + + @pyproject """ + [project] + name = "project" + version = "0.0.0" + requires-python = "==3.13.*" + dependencies = [ + "numpy==2.1.2", + "cloudpickle==3.1.2" + ] + """ + + defp reinit! do + Pythonx.uv_init(@pyproject) + end + + # Helper that mirrors what Pythonx.Finalizer does: flush the Janitor + # before finalizing, so no decref NIF is blocked on the GIL when + # Py_FinalizeEx destroys it. We also force GC so that PyObjectResource + # instances are collected before finalization, keeping resource_count + # at zero (which matters when binaries is :fast). + defp finalize! do + :erlang.garbage_collect() + send(Pythonx.Janitor, :finalizing) + Pythonx.Janitor.ping() + Pythonx.__finalize__() + end + + describe "finalize/0" do + test "refuses to run while the application is running" do + assert_raise RuntimeError, + ~r/cannot be called while the :pythonx application is running/, + fn -> + Pythonx.finalize() + end + end + + test "returns :ok after initialization" do + assert finalize!() == :ok + reinit!() + end + + test "is idempotent (second call is a no-op)" do + finalize!() + assert finalize!() == :ok + reinit!() + end + + test "is a no-op when never initialized" do + # The interpreter is currently initialized (by test_helper). + # Finalize, then finalize again (already not initialized). + finalize!() + assert finalize!() == :ok + reinit!() + end + + test "eval raises after finalize" do + finalize!() + + assert_raise RuntimeError, ~r/Python interpreter has not been initialized/, fn -> + Pythonx.eval("1 + 1", %{}) + end + + reinit!() + end + + test "re-initialization works after finalize" do + finalize!() + reinit!() + + {result, _} = Pythonx.eval("1 + 1", %{}) + assert Pythonx.decode(result) == 2 + end + + test "multiple finalize/reinit cycles work" do + for i <- 1..3 do + finalize!() + reinit!() + + {result, _} = Pythonx.eval("#{i} + #{i}", %{}) + assert Pythonx.decode(result) == i * 2 + end + end + + test "finalizes after evals spread across many dirty schedulers" do + # Regression test for a segfault in Py_FinalizeEx. Concurrent + # evals make several dirty scheduler threads create their own + # Python thread state. Finalize then lands on an arbitrary + # dirty scheduler. CPython requires Py_FinalizeEx to run on the + # thread that ran Py_InitializeEx; when it did not, CPython 3.13 + # freed the calling thread's state and dereferenced it while + # flushing std files. Deterministic on any machine with more + # than one dirty CPU scheduler. + n = System.schedulers_online() * 4 + + for cycle <- 1..3 do + results = + 1..n + |> Task.async_stream( + fn i -> + {result, _} = + Pythonx.eval( + """ + import time + time.sleep(0.05) + #{i} * #{cycle} + """, + %{} + ) + + Pythonx.decode(result) + end, + max_concurrency: n, + timeout: 30_000 + ) + |> Enum.map(fn {:ok, value} -> value end) + + assert results == Enum.map(1..n, &(&1 * cycle)) + + assert finalize!() == :ok + reinit!() + end + + {result, _} = Pythonx.eval("1 + 1", %{}) + assert Pythonx.decode(result) == 2 + end + + test "pure-Python stdlib works after re-initialization" do + # Confirms that core and stdlib re-init is clean under the fresh + # main thread: imports, a real OS thread started from Python, + # and atexit registration observed on the second finalize. + finalize!() + reinit!() + + tmp_path = + System.tmp_dir!() + |> Path.join("pythonx_reinit_test_#{:erlang.unique_integer([:positive])}.txt") + + File.rm(tmp_path) + + {result, _} = + Pythonx.eval( + """ + import atexit + import json + import logging + import threading + + logging.getLogger("pythonx_reinit").info("still works") + + seen = [] + + def work(): + seen.append(threading.current_thread().name) + + thread = threading.Thread(target=work, name="pythonx-reinit-worker") + thread.start() + thread.join() + + atexit.register(lambda: open("#{tmp_path}", "w").write("second finalize")) + + json.dumps({"seen": seen, "main": threading.main_thread().name}) + """, + %{} + ) + + assert Pythonx.decode(result) == + ~s({"seen": ["pythonx-reinit-worker"], "main": "MainThread"}) + + assert finalize!() == :ok + assert File.read!(tmp_path) == "second finalize" + File.rm(tmp_path) + + reinit!() + end + + test "atexit handlers run during finalization" do + # Register an atexit handler that writes to a module-level + # variable. After finalize + re-init, we can't read the old + # module state, so instead we verify via a side effect: + # the handler creates a file that we can check from Elixir. + tmp_path = + System.tmp_dir!() + |> Path.join("pythonx_atexit_test_#{:erlang.unique_integer([:positive])}.txt") + + File.rm(tmp_path) + + Pythonx.eval( + """ + import atexit + atexit.register(lambda: open("#{tmp_path}", "w").write("done")) + """, + %{} + ) + + assert finalize!() == :ok + # Py_FinalizeEx calls atexit handlers, which should create the file. + assert File.exists?(tmp_path) + assert File.read!(tmp_path) == "done" + File.rm(tmp_path) + + reinit!() + end + + test "logging shutdown runs during finalization" do + # logging registers atexit.register(shutdown), which flushes + # and closes all handlers. We verify by writing to a temp file + # via a logging handler, then checking the file content after + # finalization. + tmp_path = + System.tmp_dir!() + |> Path.join("pythonx_logging_test_#{:erlang.unique_integer([:positive])}.log") + + File.rm(tmp_path) + + Pythonx.eval( + """ + import logging + handler = logging.FileHandler("#{tmp_path}") + handler.setFormatter(logging.Formatter("%(message)s")) + logger = logging.getLogger("pythonx_test") + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.info("hello from logging") + """, + %{} + ) + + assert finalize!() == :ok + # The atexit handler for logging should have flushed the message. + assert File.exists?(tmp_path) + assert File.read!(tmp_path) =~ "hello from logging" + File.rm(tmp_path) + + reinit!() + end + end + + describe "generation guard" do + test "prevents stale objects from being decoded after re-init" do + {_result, globals} = Pythonx.eval("x = [1, 2, 3]", %{}) + {result, _} = Pythonx.eval("x", globals) + assert Pythonx.decode(result) == [1, 2, 3] + + finalize!() + reinit!() + + assert_raise RuntimeError, + ~r/Pythonx object is from a previous interpreter generation/, + fn -> Pythonx.decode(result) end + end + + test "prevents eval of stale globals after re-init" do + {_result, globals} = Pythonx.eval("x = [1, 2, 3]", %{}) + + finalize!() + reinit!() + + assert_raise RuntimeError, + ~r/Pythonx object is from a previous interpreter generation/, + fn -> Pythonx.eval("x", globals) end + end + + test "doesn't affect fresh objects after re-init" do + finalize!() + reinit!() + + {result, globals} = Pythonx.eval("x = 42\nx", %{}) + assert Pythonx.decode(result) == 42 + + {result2, _} = Pythonx.eval("x", globals) + assert Pythonx.decode(result2) == 42 + end + + test "prevents repr of stale objects after re-init" do + {result, _} = Pythonx.eval("[1, 2, 3]", %{}) + + finalize!() + reinit!() + + assert_raise RuntimeError, + ~r/Pythonx object is from a previous interpreter generation/, + fn -> Inspect.Pythonx.Object.__repr_string__(result) end + end + + test "prevents dump of stale objects after re-init" do + {result, _} = Pythonx.eval("[1, 2, 3]", %{}) + + finalize!() + reinit!() + + # __dump__ catches exceptions and returns {:error, ...} + assert {:error, %RuntimeError{message: msg}} = Pythonx.__dump__(result) + assert msg =~ "Pythonx object is from a previous interpreter generation" + end + end + + describe "concurrent finalize" do + test "waits for in-flight eval to complete" do + # This test verifies that finalize() waits for an in-flight eval + # to complete before tearing down the interpreter. + # + # The runner evals a 0.3s sleep. After 100ms, we call finalize. + # finalize must wait for the runner's eval NIF to finish (via + # the ActiveThreadGuard) before calling Py_FinalizeEx. + # + # The runner's eval returns a result, but calling decode on it + # after finalization would fail (interpreter is gone), so we + # only check that eval itself completed without error. + runner = + Task.async(fn -> + {result, _} = + Pythonx.eval( + """ + import time + time.sleep(0.3) + 42 + """, + %{} + ) + + # Return the result struct (not decoded — interpreter + # may be torn down by the time we get here) + result + end) + + Process.sleep(100) + + # The runner holds a Pythonx.Object, so resource_count will be + # non-zero. With :fast binaries this triggers a FinalizeError. + # We accept either :ok or an error with return_code 0 — the + # important thing is that finalize waited for the runner. + case finalize!() do + :ok -> :ok + {:error, %Pythonx.FinalizeError{return_code: 0}} -> :ok + end + + # The runner should have completed successfully. The eval NIF + # ran time.sleep(0.3) and returned before finalize destroyed + # the interpreter. The result is a Pythonx.Object struct. + assert Task.await(runner, 5000) + + reinit!() + end + end + + describe "Janitor :finalizing/:finalized" do + test "Janitor skips decref during finalization" do + # Create some Python objects that will be garbage collected + {_result, _globals} = Pythonx.eval("x = [1, 2, 3]", %{}) + + # Tell the Janitor we're finalizing + send(Pythonx.Janitor, :finalizing) + + # Trigger GC — destructors will send decref messages, + # but the Janitor should skip them + :erlang.garbage_collect() + + # Give the Janitor time to process messages + Pythonx.Janitor.ping() + + # Tell the Janitor finalization is done + send(Pythonx.Janitor, :finalized) + + # Janitor should be back to normal + assert Pythonx.Janitor.ping() == :pong + end + + test "Janitor resumes decref after :finalized" do + # Create a Python object, then tell the Janitor we're finalizing. + # GC while finalizing — decref is skipped. Then tell the Janitor + # we're done finalizing. GC again — decref should now work. + {result, _} = Pythonx.eval("[1, 2, 3]", %{}) + + send(Pythonx.Janitor, :finalizing) + :erlang.garbage_collect() + Pythonx.Janitor.ping() + + # Now resume normal operation + send(Pythonx.Janitor, :finalized) + assert Pythonx.Janitor.ping() == :pong + + # Let the object go and trigger GC — the Janitor should + # process the decref normally now (no error, no crash) + _ = result + :erlang.garbage_collect() + Pythonx.Janitor.ping() + assert Pythonx.Janitor.ping() == :pong + end + + test "Janitor forwards output during finalization" do + # Output should be forwarded even when finalizing, since + # Py_FinalizeEx may flush stdout from module finalizers. + test_pid = self() + + # Create a simple IO device that forwards io_request messages + # to the test process so we can assert on them. + io_pid = + spawn(fn -> + receive do + {:io_request, from, reply_as, {:put_chars, :unicode, output}} -> + send(test_pid, {:output_received, output}) + send(from, {:io_reply, reply_as, :ok}) + end + end) + + send(Pythonx.Janitor, :finalizing) + + # Simulate output from a Python finalizer + send(Pythonx.Janitor, {:output, "hello from finalizer", io_pid}) + + Pythonx.Janitor.ping() + + # The output should have been forwarded to the IO device + assert_received {:output_received, "hello from finalizer"} + + send(Pythonx.Janitor, :finalized) + end + end +end