Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions Lib/importlib/util.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Utility code for constructing importers, etc."""

from . import _bootstrap
from ._bootstrap import _resolve_name
from ._bootstrap import _find_spec

Expand Down Expand Up @@ -168,6 +169,33 @@ def override(self):
return -1 if self.disable_check else 1


class _LazyModuleLock:

"""Hold the import system's lock for a module while it loads lazily.

Unlike a plain lock, the module lock detects a deadlock between threads
that are loading each other's modules. On a deadlock, proceed without the
lock as _bootstrap._lock_unlock_module() does for eager imports, so that
the partially initialized module is returned.
"""

def __init__(self, name):
self._lock = _bootstrap._get_module_lock(name)
self._acquired = False

def __enter__(self):
try:
self._lock.acquire()
except _bootstrap._DeadlockError:
pass
else:
self._acquired = True

def __exit__(self, *args):
if self._acquired:
self._lock.release()


class _LazyModule(types.ModuleType):

"""A subclass of the module type which triggers loading upon attribute access."""
Expand All @@ -176,7 +204,7 @@ def __getattribute__(self, attr):
"""Trigger the load of the module and return the attribute."""
__spec__ = object.__getattribute__(self, '__spec__')
loader_state = __spec__.loader_state
with loader_state['lock']:
with _LazyModuleLock(__spec__.name):
# Only the first thread to get the lock should trigger the load
# and reset the module's class. The rest can now getattr().
if object.__getattribute__(self, '__class__') is _LazyModule:
Expand Down Expand Up @@ -259,9 +287,6 @@ def create_module(self, spec):

def exec_module(self, module):
"""Make the module load lazily."""
# Threading is only needed for lazy loading, and importlib.util can
# be pulled in at interpreter startup, so defer until needed.
import threading
module.__spec__.loader = self.loader
module.__loader__ = self.loader
# Don't need to worry about deep-copying as trying to set an attribute
Expand All @@ -271,7 +296,6 @@ def exec_module(self, module):
loader_state = {}
loader_state['__dict__'] = module.__dict__.copy()
loader_state['__class__'] = module.__class__
loader_state['lock'] = threading.RLock()
loader_state['is_loading'] = False
module.__spec__.loader_state = loader_state
module.__class__ = _LazyModule
Expand Down
65 changes: 65 additions & 0 deletions Lib/test/test_importlib/test_lazy.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import importlib
from importlib import abc
from importlib import util
import pathlib
import sys
import tempfile
import time
import threading
import types
Expand Down Expand Up @@ -178,6 +180,69 @@ def access_module():
# Or multiple load attempts
self.assertEqual(loader.load_count, 1)

@threading_helper.requires_working_threading()
def test_lazy_circular_import_no_deadlock(self):
# gh-161962: two threads first-touching two lazily loaded modules
# that import each other must not deadlock. The import system's
# module lock detects the deadlock and one thread proceeds with the
# partially initialized module, as with eager circular imports.
# The barrier makes the timing deterministic: both threads are
# inside a module load before either touches the other module.
sources = {
"sync": "import threading\nbarrier = threading.Barrier(2)\n",
"mod_a": (
"import sys, sync\n"
"A = 1\n"
"sync.barrier.wait()\n"
"FROM_B = sys.modules['mod_b'].B\n"
),
"mod_b": (
"import sys, sync\n"
"B = 2\n"
"sync.barrier.wait()\n"
"FROM_A = sys.modules['mod_a'].A\n"
),
}
with tempfile.TemporaryDirectory() as directory:
for name, source in sources.items():
pathlib.Path(directory, f"{name}.py").write_text(source)
sys.path.insert(0, directory)
try:
with test_util.uncache("sync", "mod_a", "mod_b"):
mod_a = self._lazy_import("mod_a")
mod_b = self._lazy_import("mod_b")
results = {}
threads = [
threading.Thread(
target=lambda i=i, f=f: results.__setitem__(i, f()),
daemon=True,
)
for i, f in enumerate(
[lambda: mod_a.FROM_B, lambda: mod_b.FROM_A]
)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=10)
self.assertFalse(
any(thread.is_alive() for thread in threads),
"threads deadlocked",
)
self.assertEqual(results, {0: 2, 1: 1})
finally:
sys.path.remove(directory)

def _lazy_import(self, name):
# The recipe from the importlib docs.
spec = util.find_spec(name)
loader = util.LazyLoader(spec.loader)
spec.loader = loader
module = util.module_from_spec(spec)
sys.modules[name] = module
loader.exec_module(module)
return module

def test_lazy_self_referential_modules(self):
# Directory modules with submodules that reference the parent can attempt to access
# the parent module during a load. Verify that this common pattern works with lazy loading.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a deadlock in :class:`importlib.util.LazyLoader` when two threads first access two lazily loaded modules that import each other.
Loading