From a80580321e73a9a46fdf35173a203ec10748b1ca Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 12 Sep 2026 15:07:18 -0700 Subject: [PATCH] gh-157384: Accept non-ASCII names in PyImport_CreateModuleFromInitfunc() The builtin loader info encoded the module name as ASCII, so a non-ASCII spec name failed with UnicodeEncodeError even for multi-phase init modules, which support such names when loaded dynamically. Fall back to UTF-8 and mark the name as non-ASCII, so that multi-phase init works and single-phase init is rejected with the same error as for dynamically loaded extensions. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HoZE3WdRWQshvinfBZ137N --- Lib/test/test_embed.py | 3 + ...-09-12-18-37-22.gh-issue-157384.uFbdle.rst | 3 + Programs/_testembed.c | 60 ++++++++++++++++++- Python/importdl.c | 17 +++++- 4 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1ff600e30bf4cbd..675ec4849c21709 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -278,6 +278,9 @@ def test_create_module_from_initfunc(self): "my_test_extension.exec_slot_ran='yes'\n" "\n" "embedded_ext.executed='yes'\n" + "ascii(mp.__name__)=\"'m\\\\xf6dul_mp'\" mp.executed='yes'\n" + "SystemError: 'initialization of m\\xf6dul_sp " + "did not return PyModuleDef'\n" ) def test_inittab_submodule_multiphase(self): diff --git a/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst b/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst new file mode 100644 index 000000000000000..0ba43518705c50a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst @@ -0,0 +1,3 @@ +:c:func:`PyImport_CreateModuleFromInitfunc` now accepts non-ASCII module +names for multi-phase init modules, instead of raising +:exc:`UnicodeEncodeError`. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 418609abc5f6b82..e0c274c89f08d22 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2410,6 +2410,43 @@ static int test_repeated_init_and_inittab(void) return 0; } +// Modules with non-ASCII names: multi-phase init is supported, +// single-phase init is not. +static PyModuleDef_Slot cmfi_nonascii_mp_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static PyModuleDef cmfi_nonascii_mp_def = { + PyModuleDef_HEAD_INIT, + .m_name = "nonascii_mp", + .m_size = 0, + .m_slots = cmfi_nonascii_mp_slots, +}; + +static PyObject* +PyInit_cmfi_nonascii_mp(void) +{ + return PyModuleDef_Init(&cmfi_nonascii_mp_def); +} + +static PyModuleDef cmfi_nonascii_sp_def = { + PyModuleDef_HEAD_INIT, + .m_name = "nonascii_sp", + .m_size = -1, +}; + +static PyObject* +PyInit_cmfi_nonascii_sp(void) +{ + return PyModule_Create(&cmfi_nonascii_sp_def); +} + +// "m\xf6dul_mp" and "m\xf6dul_sp" as UTF-8 +#define CMFI_NONASCII_MP_NAME "m\xc3\xb6" "dul_mp" +#define CMFI_NONASCII_SP_NAME "m\xc3\xb6" "dul_sp" + static PyObject* create_module(PyObject* self, PyObject* spec) { @@ -2425,6 +2462,14 @@ create_module(PyObject* self, PyObject* spec) Py_DECREF(name); return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); } + if (PyUnicode_EqualToUTF8(name, CMFI_NONASCII_MP_NAME)) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_mp); + } + if (PyUnicode_EqualToUTF8(name, CMFI_NONASCII_SP_NAME)) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_sp); + } PyErr_Format(PyExc_LookupError, "static module %R not found", name); Py_DECREF(name); return NULL; @@ -2472,6 +2517,11 @@ test_create_module_from_initfunc(void) L"import embedded_ext;" L"print(embedded_ext);" L"print(f'{embedded_ext.executed=}');" + // Non-ASCII names: multi-phase init works, single-phase init doesn't + L"import importlib;" + L"mp = importlib.import_module('m\\xf6dul_mp');" + L"print(f'{ascii(mp.__name__)=} {mp.executed=}');" + L"try_import('m\\xf6dul_sp');" }; PyConfig config; if (PyImport_AppendInittab("create_static_module", @@ -2491,7 +2541,8 @@ test_create_module_from_initfunc(void) " _ORIGIN = \"static-extension\"\n" " @classmethod\n" " def find_spec(cls, fullname, path, target=None):\n" - " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " if fullname in {'my_test_extension', 'embedded_ext',\n" + " 'm\\xf6dul_mp', 'm\\xf6dul_sp'}:\n" " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" " return None\n" " @staticmethod\n" @@ -2502,6 +2553,13 @@ test_create_module_from_initfunc(void) " create_static_module.exec_module(module)\n" " module.executed = 'yes'\n" "sys.meta_path.append(StaticExtensionImporter)\n" + "def try_import(name):\n" + " try:\n" + " importlib.import_module(name)\n" + " except SystemError as exc:\n" + " print(f'SystemError: {ascii(str(exc))}')\n" + " else:\n" + " print(f'no SystemError for {ascii(name)}!')\n" ); if (result < 0) { fprintf(stderr, "PyRun_SimpleString() failed\n"); diff --git a/Python/importdl.c b/Python/importdl.c index 537e8d869dc93ca..0e3ca7d45eb50f3 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -158,9 +158,22 @@ _Py_ext_module_loader_info_init_for_builtin( assert(PyUnicode_Check(name)); assert(PyUnicode_GetLength(name) > 0); + /* The encoded name is only used for error messages, so unlike + * get_encoded_name() we keep the full dotted name. Non-ASCII names + * are allowed, but only for multi-phase init modules; hook_prefixes + * records which case we are in. */ + const struct hook_prefixes *hook_prefixes = &ascii_only_prefixes; PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); if (name_encoded == NULL) { - return -1; + if (!PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + return -1; + } + PyErr_Clear(); + name_encoded = PyUnicode_AsUTF8String(name); + if (name_encoded == NULL) { + return -1; + } + hook_prefixes = &nonascii_prefixes; } *info = (struct _Py_ext_module_loader_info){ @@ -169,7 +182,7 @@ _Py_ext_module_loader_info_init_for_builtin( /* We won't need filename. */ .path=name, .origin=_Py_ext_module_origin_BUILTIN, - .hook_prefixes=&ascii_only_prefixes, + .hook_prefixes=hook_prefixes, .newcontext=NULL, }; return 0;