finalize the Python interpreter on shutdown

Makefile

@@ -5,6 +5,17 @@ C_SRC := $(shell pwd)/c_src
 CPPFLAGS := -shared -fPIC -fvisibility=hidden -std=c++17 -Wall -Wextra -Wno-unused-parameter -Wno-comment
 CPPFLAGS += -I$(ERTS_INCLUDE_DIR) -I$(FINE_INCLUDE_DIR)
 
+# PYTHONX_BINARIES is always set by make_env in mix.exs.
+# "safe" copies bytes (no dangling pointer risk after finalization).
+# "fast" uses zero-copy resource binaries (existing behavior).
+ifeq ($(PYTHONX_BINARIES),safe)
+	CPPFLAGS += -DPYTHONX_SAFE_BINARIES
+else ifeq ($(PYTHONX_BINARIES),fast)
+	CPPFLAGS += -DPYTHONX_FAST_BINARIES
+else
+$(error PYTHONX_BINARIES must be "safe" or "fast")
+endif
+
 ifdef DEBUG
 	CPPFLAGS += -g
 else
@@ -22,9 +33,14 @@ endif
 SOURCES := $(wildcard $(C_SRC)/*.cpp)
 HEADERS := $(wildcard $(C_SRC)/*.hpp)
 
+# Stamp file written by make_env (mix.exs) recording the current
+# PYTHONX_BINARIES value. A change in the config triggers a recompile
+# because the NIF target depends on this file's timestamp.
+BINARIES_STAMP := $(MIX_MANIFEST_PATH)/pythonx_binaries.stamp
+
 all: $(NIF_PATH)
 	@ echo > /dev/null # Dummy command to avoid the default output "Nothing to be done"
 
-$(NIF_PATH): $(SOURCES) $(HEADERS)
+$(NIF_PATH): $(SOURCES) $(HEADERS) $(BINARIES_STAMP)
 	@ mkdir -p $(PRIV_DIR)
 	$(CXX) $(CPPFLAGS) $(SOURCES) -o $(NIF_PATH)

README.md

@@ -150,6 +150,42 @@ Note that currently the `~PY` sigil does not work as part of Mix project
 code. This limitation is intentional, since in actual applications it
 is preferable to manage the Python globals explicitly.
 
+### Finalization
+
+When a normal Python executable terminates, it runs a sophisticated shutdown
+sequence called "finalization". This is what triggers most of the cleanup
+behavior you might expect.  This includes things like as object destructors,
+`atexit` handlers and module finalizers.
+
+Without finalizing, many libraries can misbehave subtly--including the Python
+standard library!
+
+Some real examples observed include:
+- warnings about leaked `multiprocessing` semaphores
+- log messages not actually emitted
+- `readline` history not written
+- files created with `tempfile` aren't deleted
+- changes to a `shelve` database aren't persisted
+- files and sockets aren't closed
+- context managers in generators may not call their exit hooks
+
+When running under a full OTP supervision tree (i.e. in most apps + IEx),
+Pythonx automatically finalizes the interpreter as part of a graceful shutdown.
+
+Some environments don't run a full supervision tree.  Typically that will be
+things like exscripts and Mix tasks.  In these situations, you can also call
+`Pythonx.finalize/0` manually.
+
+After finalization, the interpreter can be re-initialized with
+`Pythonx.uv_init/2` or `Pythonx.init/4`. Objects from the previous interpreter
+session are detected via an internal generation counter and rejected with a
+clear error message.
+
+Re-initialization is reliable for pure-Python code, but C extensions with
+process-global state (`numpy`, `torch`) may fail or crash on their second
+import; see the "Re-initialization limits" section in the `Pythonx.finalize/0`
+docs.
+
 ## Python API
 
 Pythonx provides a Python module named `pythonx` with extra interoperability

c_src/main_thread.hpp

@@ -0,0 +1,144 @@
+// A native thread owned by pythonx that acts as CPython's "main thread".
+//
+// CPython binds the thread that runs Py_InitializeEx to
+// runtime->main_tstate, and Py_FinalizeEx must run on that same
+// thread: when called from any other thread, CPython 3.13 does not
+// swap the main thread state in, frees the calling thread's state as
+// a "non-main" one, and then dereferences it while flushing std files.
+// NIFs run on whichever dirty scheduler picks them up, and there is no
+// way to pin two separate NIF calls to the same scheduler thread, so
+// we own one thread for exactly those two calls. Everything else
+// (eval, decode, sys.path setup) keeps running on the dirty
+// schedulers with their own thread states.
+//
+// The thread is started in init() and joined in finalize(), so every
+// finalize + re-init cycle gets a fresh main thread. It is only ever
+// used while init_mutex is held, so a single-slot job handoff suffices.
+
+#pragma once
+
+#include <condition_variable>
+#include <exception>
+#include <functional>
+#include <mutex>
+#include <stdexcept>
+#include <thread>
+
+#if defined(__APPLE__) || defined(__linux__)
+#include <pthread.h>
+#endif
+
+namespace pythonx {
+
+class MainThread {
+  std::thread thread;
+  std::mutex mutex;
+  std::condition_variable cv;
+  std::function<void()> job;
+  bool stop = false;
+  bool has_job = false;
+  bool done = false;
+  std::exception_ptr error;
+
+  void loop() {
+#if defined(__APPLE__)
+    pthread_setname_np("pythonx-main");
+#elif defined(__linux__)
+    pthread_setname_np(pthread_self(), "pythonx-main");
+#endif
+
+    while (true) {
+      std::function<void()> current_job;
+
+      {
+        auto lock = std::unique_lock<std::mutex>(mutex);
+        cv.wait(lock, [&] { return has_job || stop; });
+
+        if (!has_job) {
+          return;
+        }
+
+        current_job = std::move(job);
+        has_job = false;
+      }
+
+      std::exception_ptr current_error;
+
+      try {
+        current_job();
+      } catch (...) {
+        current_error = std::current_exception();
+      }
+
+      {
+        auto lock = std::lock_guard<std::mutex>(mutex);
+        error = current_error;
+        done = true;
+      }
+
+      cv.notify_all();
+    }
+  }
+
+public:
+  // Spawns the loop thread. Must not be called while a previous
+  // thread is still running; finalize() joins before init() starts
+  // again.
+  void start() {
+    if (thread.joinable()) {
+      throw std::runtime_error("pythonx main thread is already running");
+    }
+
+    {
+      auto lock = std::lock_guard<std::mutex>(mutex);
+      stop = false;
+      has_job = false;
+      done = false;
+      error = nullptr;
+    }
+
+    thread = std::thread([this] { loop(); });
+  }
+
+  // Runs fn on the main thread and blocks until it returns. An
+  // exception thrown by fn is rethrown on the calling thread.
+  void run(std::function<void()> fn) {
+    if (!thread.joinable()) {
+      throw std::runtime_error("pythonx main thread is not running");
+    }
+
+    auto lock = std::unique_lock<std::mutex>(mutex);
+    job = std::move(fn);
+    has_job = true;
+    done = false;
+    error = nullptr;
+    cv.notify_all();
+
+    cv.wait(lock, [&] { return done; });
+
+    auto current_error = error;
+    error = nullptr;
+    lock.unlock();
+
+    if (current_error) {
+      std::rethrow_exception(current_error);
+    }
+  }
+
+  // Stops the loop and joins the thread. The object can be started
+  // again afterwards.
+  void join() {
+    {
+      auto lock = std::lock_guard<std::mutex>(mutex);
+      stop = true;
+    }
+
+    cv.notify_all();
+
+    if (thread.joinable()) {
+      thread.join();
+    }
+  }
+};
+
+} // namespace pythonx

c_src/python.cpp

@@ -28,6 +28,7 @@ DEF_SYMBOL(PyDict_SetItem)
 DEF_SYMBOL(PyDict_SetItemString)
 DEF_SYMBOL(PyDict_Size)
 DEF_SYMBOL(PyErr_Clear)
+DEF_SYMBOL(PyErr_SetString)
 DEF_SYMBOL(PyErr_Fetch)
 DEF_SYMBOL(PyErr_Occurred)
 DEF_SYMBOL(PyEval_GetBuiltins)
@@ -74,12 +75,16 @@ DEF_SYMBOL(Py_CompileString)
 DEF_SYMBOL(Py_DecRef)
 DEF_SYMBOL(Py_IncRef)
 DEF_SYMBOL(Py_InitializeEx)
+DEF_SYMBOL(Py_FinalizeEx)
 DEF_SYMBOL(Py_IsFalse)
 DEF_SYMBOL(Py_IsNone)
 DEF_SYMBOL(Py_IsTrue)
 DEF_SYMBOL(Py_SetPythonHome)
 DEF_SYMBOL(Py_SetProgramName)
 
+// Exception objects
+DEF_SYMBOL(PyExc_RuntimeError)
+
 dl::LibraryHandle python_library;
 
 void load_python_library(std::string path) {
@@ -103,6 +108,7 @@ void load_python_library(std::string path) {
   LOAD_SYMBOL(python_library, PyDict_SetItemString)
   LOAD_SYMBOL(python_library, PyDict_Size)
   LOAD_SYMBOL(python_library, PyErr_Clear)
+  LOAD_SYMBOL(python_library, PyErr_SetString)
   LOAD_SYMBOL(python_library, PyErr_Fetch)
   LOAD_SYMBOL(python_library, PyErr_Occurred)
   LOAD_SYMBOL(python_library, PyEval_GetBuiltins)
@@ -149,11 +155,15 @@ void load_python_library(std::string path) {
   LOAD_SYMBOL(python_library, Py_DecRef)
   LOAD_SYMBOL(python_library, Py_IncRef)
   LOAD_SYMBOL(python_library, Py_InitializeEx)
+  LOAD_SYMBOL(python_library, Py_FinalizeEx)
   LOAD_SYMBOL(python_library, Py_IsFalse)
   LOAD_SYMBOL(python_library, Py_IsNone)
   LOAD_SYMBOL(python_library, Py_IsTrue)
   LOAD_SYMBOL(python_library, Py_SetPythonHome)
   LOAD_SYMBOL(python_library, Py_SetProgramName)
+
+  // Exception objects
+  LOAD_SYMBOL(python_library, PyExc_RuntimeError)
 }
 
 void unload_python_library() {

c_src/python.hpp

@@ -82,6 +82,7 @@ extern int (*PyDict_SetItem)(PyObjectPtr, PyObjectPtr, PyObjectPtr);
 extern int (*PyDict_SetItemString)(PyObjectPtr, const char *, PyObjectPtr);
 extern Py_ssize_t (*PyDict_Size)(PyObjectPtr);
 extern void (*PyErr_Clear)();
+extern void (*PyErr_SetString)(PyObjectPtr, const char *);
 extern void (*PyErr_Fetch)(PyObjectPtr *, PyObjectPtr *, PyObjectPtr *);
 extern PyObjectPtr (*PyErr_Occurred)();
 extern PyObjectPtr (*PyEval_GetBuiltins)();
@@ -128,12 +129,16 @@ extern PyObjectPtr (*Py_CompileString)(const char *, const char *, int);
 extern void (*Py_DecRef)(PyObjectPtr);
 extern void (*Py_IncRef)(PyObjectPtr);
 extern void (*Py_InitializeEx)(int);
+extern int (*Py_FinalizeEx)();
 extern int (*Py_IsFalse)(PyObjectPtr);
 extern int (*Py_IsNone)(PyObjectPtr);
 extern int (*Py_IsTrue)(PyObjectPtr);
 extern void (*Py_SetPythonHome)(const wchar_t *);
 extern void (*Py_SetProgramName)(const wchar_t *);
 
+// Exception objects (global variables, not functions)
+extern PyObjectPtr PyExc_RuntimeError;
+
 // Opens Python dynamic library at the given path and looks up all
 // relevant symbols.
 //