[3.15] gh-153236: Propagate lazy submodule import errors (GH-153237) (#153936)

gh-153236: Propagate lazy submodule import errors (GH-153237)
(cherry picked from commit 0a09dafb03)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>

Include/internal/pycore_global_objects_fini_generated.h

@@ -1544,6 +1544,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_filters));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load_lazy_submodule));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile));

Include/internal/pycore_global_strings.h

@@ -267,6 +267,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_filters)
         STRUCT_FOR_ID(_finalizing)
         STRUCT_FOR_ID(_find_and_load)
+        STRUCT_FOR_ID(_find_and_load_lazy_submodule)
         STRUCT_FOR_ID(_fix_up_module)
         STRUCT_FOR_ID(_flags_)
         STRUCT_FOR_ID(_get_sourcefile)

Include/internal/pycore_import.h

@@ -39,8 +39,13 @@ extern PyObject * _PyImport_GetAbsName(
 // Symbol is exported for the JIT on Windows builds.
 PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate(
     PyThreadState *tstate, PyObject *lazy_import);
-extern PyObject * _PyImport_TryLoadLazySubmodule(
-    PyObject *mod_name, PyObject *attr_name);
+typedef enum {
+    _Py_LAZY_SUBMODULE_ERROR = -1,
+    _Py_LAZY_SUBMODULE_NOT_FOUND = 0,
+    _Py_LAZY_SUBMODULE_LOADED = 1,
+} _PyLazySubmoduleImportResult;
+extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule(
+    PyObject *mod_name, PyObject *attr_name, PyObject **result);
 extern PyObject * _PyImport_LazyImportModuleLevelObject(
     PyThreadState *tstate, PyObject *name, PyObject *builtins,
     PyObject *globals, PyObject *locals, PyObject *fromlist, int level);

Include/internal/pycore_runtime_init_generated.h

@@ -1542,6 +1542,7 @@ extern "C" {
     INIT_ID(_filters), \
     INIT_ID(_finalizing), \
     INIT_ID(_find_and_load), \
+    INIT_ID(_find_and_load_lazy_submodule), \
     INIT_ID(_fix_up_module), \
     INIT_ID(_flags_), \
     INIT_ID(_get_sourcefile), \

Include/internal/pycore_unicodeobject_generated.h

@@ -848,6 +848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));
     assert(PyUnicode_GET_LENGTH(string) != 1);
+    string = &_Py_ID(_find_and_load_lazy_submodule);
+    _PyUnicode_InternStatic(interp, &string);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    assert(PyUnicode_GET_LENGTH(string) != 1);
     string = &_Py_ID(_fix_up_module);
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));

Lib/importlib/_bootstrap.py

@@ -1260,7 +1260,7 @@ def _sanity_check(name, package, level):
 
 _ERR_MSG_PREFIX = 'No module named '
 
-def _find_and_load_unlocked(name, import_):
+def _find_and_load_unlocked(name, import_, *, lazy_submodule=False):
     path = None
     sys.audit(
         "import",
@@ -1283,6 +1283,8 @@ def _find_and_load_unlocked(name, import_):
         try:
             path = parent_module.__path__
         except AttributeError:
+            if lazy_submodule:
+                return None
             msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package'
             raise ModuleNotFoundError(msg, name=name) from None
         parent_spec = parent_module.__spec__
@@ -1295,6 +1297,8 @@ def _find_and_load_unlocked(name, import_):
         child = name.rpartition('.')[2]
     spec = _find_spec(name, path)
     if spec is None:
+        if lazy_submodule:
+            return None
         raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name)
     else:
         if parent_spec:
@@ -1326,7 +1330,7 @@ def _find_and_load_unlocked(name, import_):
 _NEEDS_LOADING = object()
 
 
-def _find_and_load(name, import_):
+def _find_and_load(name, import_, *, lazy_submodule=False):
     """Find and load the module."""
 
     # Optimization: we avoid unneeded module locking if the module
@@ -1343,7 +1347,8 @@ def _find_and_load(name, import_):
         with lock_manager:
             module = sys.modules.get(name, _NEEDS_LOADING)
             if module is _NEEDS_LOADING:
-                return _find_and_load_unlocked(name, import_)
+                return _find_and_load_unlocked(
+                    name, import_, lazy_submodule=lazy_submodule)
 
         # Optimization: only call _bootstrap._lock_unlock_module() if
         # module.__spec__._initializing is True.
@@ -1357,7 +1362,7 @@ def _find_and_load(name, import_):
         # to preserve normal semantics: the caller gets the exception from
         # the actual import failure rather than a synthetic error.
         if sys.modules.get(name) is not module:
-            return _find_and_load(name, import_)
+            return _find_and_load(name, import_, lazy_submodule=lazy_submodule)
 
     if module is None:
         message = f'import of {name} halted; None in sys.modules'
@@ -1366,6 +1371,10 @@ def _find_and_load(name, import_):
     return module
 
 
+def _find_and_load_lazy_submodule(name, import_):
+    return _find_and_load(name, import_, lazy_submodule=True)
+
+
 def _gcd_import(name, package=None, level=0):
     """Import and return the module based on its name, the package the call is
     being made from, and the level adjustment.

Lib/test/test_lazy_import/__init__.py

@@ -684,52 +684,35 @@ class SysLazyImportsAPITests(LazyImportTestCase):
 
 @support.requires_subprocess()
 class ErrorHandlingTests(LazyImportTestCase):
-    """Tests for error handling during lazy import reification.
+    """Tests for error handling during lazy import reification."""
 
-    PEP 810: Errors during reification should show exception chaining with
-    both the lazy import definition location and the access location.
-    """
-
-    def test_import_error_shows_chained_traceback(self):
+    def test_missing_lazy_submodule_raises_attribute_error(self):
         """Accessing a nonexistent lazy submodule via parent attr raises AttributeError."""
         code = textwrap.dedent("""
-            import sys
             lazy import test.test_lazy_import.data.nonexistent_module
 
             try:
-                x = test.test_lazy_import.data.nonexistent_module
-            except AttributeError as e:
-                print("OK")
+                _ = test.test_lazy_import.data.nonexistent_module
+            except AttributeError:
+                pass
+            else:
+                raise AssertionError("AttributeError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
-    def test_attribute_error_on_from_import_shows_chained_traceback(self):
+    def test_missing_lazy_from_import_shows_chained_traceback(self):
         """Accessing missing attribute from lazy from-import should chain errors."""
-        # Tests 'lazy from module import nonexistent' behavior
         code = textwrap.dedent("""
-            import sys
             lazy from test.test_lazy_import.data.basic2 import nonexistent_name
 
             try:
-                x = nonexistent_name
+                _ = nonexistent_name
             except ImportError as e:
-                # PEP 810: Enhanced error reporting through exception chaining
                 assert e.__cause__ is not None, "Expected chained exception"
-                print("OK")
+            else:
+                raise AssertionError("ImportError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
 
     def test_reification_retries_on_failure(self):
         """Failed reification should allow retry on subsequent access.
@@ -739,53 +722,92 @@ class ErrorHandlingTests(LazyImportTestCase):
         """
         code = textwrap.dedent("""
             import sys
-            import types
 
             lazy import test.test_lazy_import.data.broken_module
 
-            # First access - should fail
-            try:
-                x = test.test_lazy_import.data.broken_module
-            except AttributeError:
-                pass
-
-            # The lazy object should still be a lazy proxy (not reified)
-            g = globals()
-            lazy_obj = g['test']
-            # The root 'test' binding should still allow retry
-            # Second access - should also fail (retry the import)
-            try:
-                x = test.test_lazy_import.data.broken_module
-            except AttributeError:
-                print("OK - retry worked")
-        """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
-
-    def test_error_during_module_execution_propagates(self):
-        """Errors in module code during reification should propagate correctly."""
-        code = textwrap.dedent("""
-            import sys
-            lazy import test.test_lazy_import.data.broken_module
-
             try:
                 _ = test.test_lazy_import.data.broken_module
-                print("FAIL - should have raised")
-            except AttributeError:
-                print("OK")
+            except ValueError as exc:
+                assert str(exc) == "This module always fails to import", exc
+            else:
+                raise AssertionError("ValueError was not raised")
+
+            assert "test.test_lazy_import.data.broken_module" not in sys.modules
+
+            try:
+                _ = test.test_lazy_import.data.broken_module
+            except ValueError as exc:
+                assert str(exc) == "This module always fails to import", exc
+            else:
+                raise AssertionError("ValueError was not raised")
         """)
-        result = subprocess.run(
-            [sys.executable, "-c", code],
-            capture_output=True,
-            text=True
-        )
-        self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
-        self.assertIn("OK", result.stdout)
+        assert_python_ok("-c", code)
+
+    def test_lazy_submodule_traceback_hides_importlib_frames(self):
+        code = textwrap.dedent("""
+            import traceback
+
+            lazy import test.test_lazy_import.data.broken_module
+
+            try:
+                _ = test.test_lazy_import.data.broken_module
+            except ValueError as exc:
+                frames = traceback.extract_tb(exc.__traceback__)
+                assert [frame.name for frame in frames] == ["<module>", "<module>"], frames
+                assert frames[0].filename == "<string>", frames
+                assert frames[1].filename.endswith("broken_module.py"), frames
+            else:
+                raise AssertionError("ValueError was not raised")
+        """)
+        assert_python_ok("-c", code)
+
+    def test_module_not_found_during_module_execution_propagates(self):
+        code = textwrap.dedent("""
+            lazy import test.test_lazy_import.data.missing_dependency
+
+            try:
+                _ = test.test_lazy_import.data.missing_dependency
+            except ModuleNotFoundError as exc:
+                assert exc.name == "missing_dependency_for_lazy_import_test", exc.name
+                assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
+        """)
+        assert_python_ok("-c", code)
+
+    def test_self_named_module_not_found_during_module_execution_propagates(self):
+        code = textwrap.dedent("""
+            lazy import test.test_lazy_import.data.self_named_module_not_found
+
+            try:
+                _ = test.test_lazy_import.data.self_named_module_not_found
+            except ModuleNotFoundError as exc:
+                assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name
+                assert str(exc) == "boom", exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
+        """)
+        assert_python_ok("-c", code)
+
+    def test_none_in_sys_modules_during_submodule_resolution_propagates(self):
+        code = textwrap.dedent("""
+            import sys
+
+            sys.modules["test.test_lazy_import.data.blocked_module"] = None
+            lazy import test.test_lazy_import.data.blocked_module
+
+            try:
+                _ = test.test_lazy_import.data.blocked_module
+            except ModuleNotFoundError as exc:
+                assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name
+                assert str(exc) == (
+                    "import of test.test_lazy_import.data.blocked_module "
+                    "halted; None in sys.modules"
+                ), exc
+            else:
+                raise AssertionError("ModuleNotFoundError was not raised")
+        """)
+        assert_python_ok("-c", code)
 
     def test_circular_lazy_import_does_not_crash_for_gh_144727(self):
         with tempfile.TemporaryDirectory() as tmpdir:

Lib/test/test_lazy_import/data/missing_dependency.py

@@ -0,0 +1 @@
+import missing_dependency_for_lazy_import_test

Lib/test/test_lazy_import/data/self_named_module_not_found.py

@@ -0,0 +1 @@
+raise ModuleNotFoundError("boom", name=__name__)

Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst

@@ -0,0 +1,2 @@
+Propagate exceptions raised while importing lazy submodules instead of
+reporting them as missing attributes.

Objects/moduleobject.c

@@ -1299,8 +1299,6 @@ _PyModule_IsPossiblyShadowing(PyObject *origin)
     return result;
 }
 
-// Check if `name` is a lazily pending submodule of module `m`.
-// Returns a new reference on success, or NULL with no error set.
 static PyObject *
 try_load_lazy_submodule(PyModuleObject *m, PyObject *name)
 {
@@ -1313,10 +1311,13 @@ try_load_lazy_submodule(PyModuleObject *m, PyObject *name)
         Py_DECREF(mod_name);
         return NULL;
     }
-    PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name);
+    PyObject *result = NULL;
+    _PyLazySubmoduleImportResult status =
+        _PyImport_TryLoadLazySubmodule(mod_name, name, &result);
     Py_DECREF(mod_name);
-    if (result == NULL) {
-        PyErr_Clear();
+    if (status != _Py_LAZY_SUBMODULE_LOADED) {
+        assert(status == _Py_LAZY_SUBMODULE_ERROR ||
+               status == _Py_LAZY_SUBMODULE_NOT_FOUND);
         return NULL;
     }
     if (PyDict_SetItem(m->md_dict, name, result) < 0) {

Python/import.c

@@ -4100,7 +4100,9 @@ ok:
 }
 
 static PyObject *
-import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
+import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name,
+                               PyObject *find_and_load,
+                               PyObject *not_found)
 {
     PyObject *mod = NULL;
     PyInterpreterState *interp = tstate->interp;
@@ -4127,12 +4129,14 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
     if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED())
         PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name));
 
-    mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load),
+    mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), find_and_load,
                                      abs_name, IMPORT_FUNC(interp), NULL);
 
-    if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED())
+    if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) {
+        int found = mod != NULL && mod != not_found;
         PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name),
-                                       mod != NULL);
+                                       found);
+    }
 
     if (import_time) {
         PyTime_t t2;
@@ -4153,6 +4157,13 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
 #undef accumulated
 }
 
+static PyObject *
+import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
+{
+    return import_find_and_load_with_name(
+        tstate, abs_name, &_Py_ID(_find_and_load), NULL);
+}
+
 static PyObject *
 get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals,
              int level)
@@ -4436,52 +4447,70 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name,
     return res;
 }
 
-PyObject *
-_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name)
+_PyLazySubmoduleImportResult
+_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name,
+                               PyObject **result)
 {
-    PyInterpreterState *interp = _PyInterpreterState_GET();
+    assert(result != NULL);
+    *result = NULL;
+
+    PyThreadState *tstate = _PyThreadState_GET();
+    PyInterpreterState *interp = tstate->interp;
     PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp);
     if (lazy_pending == NULL) {
-        return NULL;
+        return _Py_LAZY_SUBMODULE_NOT_FOUND;
     }
 
     PyObject *pending_set;
     int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set);
-    if (rc <= 0) {
-        return NULL;
+    if (rc < 0) {
+        return _Py_LAZY_SUBMODULE_ERROR;
+    }
+    if (rc == 0) {
+        return _Py_LAZY_SUBMODULE_NOT_FOUND;
     }
 
     int contains = PySet_Contains(pending_set, attr_name);
-    if (contains <= 0) {
+    if (contains < 0) {
         Py_DECREF(pending_set);
-        return NULL;
+        return _Py_LAZY_SUBMODULE_ERROR;
+    }
+    if (contains == 0) {
+        Py_DECREF(pending_set);
+        return _Py_LAZY_SUBMODULE_NOT_FOUND;
     }
 
     PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name);
     if (full_name == NULL) {
         Py_DECREF(pending_set);
-        return NULL;
+        return _Py_LAZY_SUBMODULE_ERROR;
     }
 
-    PyObject *mod = PyImport_ImportModuleLevelObject(
-        full_name, NULL, NULL, NULL, 0);
+    PyObject *mod = import_find_and_load_with_name(
+        tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None);
     if (mod == NULL) {
         Py_DECREF(pending_set);
         Py_DECREF(full_name);
-        return NULL;
+        remove_importlib_frames(tstate);
+        return _Py_LAZY_SUBMODULE_ERROR;
     }
-    Py_DECREF(mod);
-
-    if (PySet_Discard(pending_set, attr_name) < 0) {
+    if (mod == Py_None) {
+        Py_DECREF(mod);
         Py_DECREF(pending_set);
         Py_DECREF(full_name);
-        return NULL;
+        return _Py_LAZY_SUBMODULE_NOT_FOUND;
+    }
+
+    if (PySet_Discard(pending_set, attr_name) < 0) {
+        Py_DECREF(mod);
+        Py_DECREF(pending_set);
+        Py_DECREF(full_name);
+        return _Py_LAZY_SUBMODULE_ERROR;
     }
     Py_DECREF(pending_set);
-
-    PyObject *submod = PyImport_GetModule(full_name);
     Py_DECREF(full_name);
-    return submod;
+    *result = mod;
+    return _Py_LAZY_SUBMODULE_LOADED;
 }
 
 PyObject *