Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 28 additions & 15 deletions Lib/importlib/_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,7 @@ def _load_unlocked(spec):
del sys.modules[spec.name]
except KeyError:
pass
spec._initializing = False
raise
# Move the module to the end of sys.modules.
# We don't ensure that the import-related module attributes get
Expand All @@ -926,8 +927,10 @@ def _load_unlocked(spec):
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
_verbose_message('import {!r} # {!r}', spec.name, spec.loader)
finally:
except:
# Any other exception bubbles up; clear the flag too.
spec._initializing = False
raise

return module

Expand All @@ -942,7 +945,10 @@ def _load(spec):

"""
with _ModuleLockManager(spec.name):
return _load_unlocked(spec)
try:
return _load_unlocked(spec)
finally:
spec._initializing = False


# Loaders #####################################################################
Expand Down Expand Up @@ -1306,20 +1312,24 @@ def _find_and_load_unlocked(name, import_):
finally:
if parent_spec:
parent_spec._uninitialized_submodules.pop()
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
try:
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
try:
setattr(parent_module, child, module)
except AttributeError:
msg = (f"Cannot set an attribute on {parent!r} for child "
f"module {child!r}")
_warnings.warn(msg, ImportWarning)
# Set attributes to lazy submodules on the module.
try:
setattr(parent_module, child, module)
except AttributeError:
msg = f"Cannot set an attribute on {parent!r} for child module {child!r}"
_imp._set_lazy_attributes(module, name)
except Exception as e:
msg = f"Cannot set lazy attributes on {name!r}: {e!r}"
_warnings.warn(msg, ImportWarning)
# Set attributes to lazy submodules on the module.
try:
_imp._set_lazy_attributes(module, name)
except Exception as e:
msg = f"Cannot set lazy attributes on {name!r}: {e!r}"
_warnings.warn(msg, ImportWarning)
finally:
spec._initializing = False
return module


Expand Down Expand Up @@ -1486,7 +1496,10 @@ def _builtin_from_name(name):
spec = BuiltinImporter.find_spec(name)
if spec is None:
raise ImportError('no built-in module named ' + name)
return _load_unlocked(spec)
try:
return _load_unlocked(spec)
finally:
spec._initializing = False


def _setup(sys_module, _imp_module):
Expand Down
87 changes: 87 additions & 0 deletions Lib/test/test_importlib/test_threaded_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,93 @@ def do_import(name):
errors, [],
f"Import(s) failed on iteration {i}: {errors}")

def test_lazy_submodule_getattr_no_recursion(self):
# Regression test: a package that uses a lazy `__getattr__` doing
# `import pkg.sub as sub` must not recurse to RecursionError when
# another thread first-touches the same submodule during the
# window between `_load_unlocked` clearing `spec._initializing`
# and `_find_and_load_unlocked` running `setattr(parent, child,
# module)`. The race is widened deterministically with a
# threading.Event so the test does not depend on luck.
import importlib._bootstrap as _b

os.makedirs(os.path.join(TESTFN, "lazypkg", "sub"))
self.addCleanup(shutil.rmtree, TESTFN)
with open(os.path.join(TESTFN, "lazypkg", "__init__.py"), "w") as f:
f.write(
"def __getattr__(attr):\n"
" if attr == 'sub':\n"
" import lazypkg.sub as sub\n"
" return sub\n"
" raise AttributeError(attr)\n"
)
with open(os.path.join(TESTFN, "lazypkg", "sub", "__init__.py"), "w") as f:
f.write("X = 42\n")

sys.path.insert(0, TESTFN)
self.addCleanup(sys.path.remove, TESTFN)
for mod in ("lazypkg", "lazypkg.sub"):
self.addCleanup(forget, mod)
importlib.invalidate_caches()

gate = threading.Event()
done = threading.Event()
orig_find_and_load_unlocked = _b._find_and_load_unlocked

def widen(name, import_):
if name != "lazypkg.sub":
return orig_find_and_load_unlocked(name, import_)
parent_module = sys.modules["lazypkg"]
spec = _b._find_spec(name, parent_module.__path__)
parent_module.__spec__._uninitialized_submodules.append("sub")
try:
module = _b._load_unlocked(spec)
finally:
parent_module.__spec__._uninitialized_submodules.pop()
# Pause inside the unsafe window so the observer thread
# can probe the half-initialized state. Without the fix
# in _bootstrap, that probe drives `__getattr__` into
# infinite recursion.
gate.set()
done.wait(timeout=5.0)
setattr(parent_module, "sub", module)
return module

_b._find_and_load_unlocked = widen
self.addCleanup(setattr, _b, "_find_and_load_unlocked",
orig_find_and_load_unlocked)

result = {}

def loader():
__import__("lazypkg").sub

def observer():
gate.wait(timeout=5.0)
try:
result["value"] = __import__("lazypkg").sub
except BaseException as exc:
result["error"] = exc
done.set()

t_loader = threading.Thread(target=loader)
t_observer = threading.Thread(target=observer)
t_observer.start()
t_loader.start()
t_loader.join(timeout=10)
t_observer.join(timeout=10)

self.assertFalse(t_loader.is_alive(), "loader thread deadlocked")
self.assertFalse(t_observer.is_alive(), "observer thread deadlocked")
self.assertNotIn(
"error", result,
f"observer thread raised {type(result.get('error')).__name__}: "
f"{result.get('error')!r}",
)
self.assertIs(
result["value"], sys.modules["lazypkg.sub"],
"observer thread did not get the loaded submodule")


def setUpModule():
thread_info = threading_helper.threading_setup()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fix race in importlib when importing module which defined getattr
Loading