gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover

Doc/library/importlib.rst

@@ -275,6 +275,28 @@ ABC hierarchy::
       .. versionchanged:: 3.4
          Returns ``None`` when called instead of :data:`NotImplemented`.
 
+   .. method:: discover(parent=None)
+
+      An optional method which searches for possible specs with given *parent*
+      module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will
+      search for top-level modules.
+
+      Returns an iterable of possible specs.
+
+      Raises :exc:`ValueError` if *parent* is not a package module.
+
+      .. warning::
+         This method can potentially yield a very large number of objects, and
+         it may carry out IO operations when computing these values.
+
+         Because of this, it will generaly be desirable to compute the result
+         values on-the-fly, as they are needed. As such, the returned object is
+         only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
+         instead of a :class:`list` or other
+         :class:`collection <collections.abc.Collection>` type.
+
+      .. versionadded:: next
+
 
 .. class:: PathEntryFinder
 
@@ -307,6 +329,28 @@ ABC hierarchy::
       :meth:`importlib.machinery.PathFinder.invalidate_caches`
       when invalidating the caches of all cached finders.
 
+   .. method:: discover(parent=None)
+
+      An optional method which searches for possible specs with given *parent*
+      module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will
+      search for top-level modules.
+
+      Returns an iterable of possible specs.
+
+      Raises :exc:`ValueError` if *parent* is not a package module.
+
+      .. warning::
+         This method can potentially yield a very large number of objects, and
+         it may carry out IO operations when computing these values.
+
+         Because of this, it will generaly be desirable to compute the result
+         values on-the-fly, as they are needed. As such, the returned object is
+         only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
+         instead of a :class:`list` or other
+         :class:`collection <collections.abc.Collection>` type.
+
+      .. versionadded:: next
+
 
 .. class:: Loader
 

Lib/importlib/_bootstrap_external.py

@@ -1323,6 +1323,23 @@ def find_spec(cls, fullname, path=None, target=None):
         else:
             return spec
 
+    @classmethod
+    def discover(cls, parent=None):
+        if parent is None:
+            path = sys.path
+        elif parent.submodule_search_locations is None:
+            raise ValueError(f'{parent} is not a package module')
+        else:
+            path = parent.submodule_search_locations
+
+        for entry in set(path):
+            if not isinstance(entry, str):
+                continue
+            if (finder := cls._path_importer_cache(entry)) is None:
+                continue
+            if discover := getattr(finder, 'discover', None):
+                yield from discover(parent)
+
     @staticmethod
     def find_distributions(*args, **kwargs):
         """
@@ -1472,6 +1489,37 @@ def path_hook_for_FileFinder(path):
 
         return path_hook_for_FileFinder
 
+    def _find_children(self):
+        with _os.scandir(self.path) as scan_iterator:
+            while True:
+                try:
+                    entry = next(scan_iterator)
+                    if entry.name == _PYCACHE:
+                        continue
+                    # packages
+                    if entry.is_dir() and '.' not in entry.name:
+                        yield entry.name
+                    # files
+                    if entry.is_file():
+                        yield from {
+                            entry.name.removesuffix(suffix)
+                            for suffix, _ in self._loaders
+                            if entry.name.endswith(suffix)
+                        }
+                except OSError:
+                    pass  # ignore exceptions from next(scan_iterator) and os.DirEntry
+                except StopIteration:
+                    break
+
+    def discover(self, parent=None):
+        if parent and parent.submodule_search_locations is None:
+            raise ValueError(f'{parent} is not a package module')
+
+        module_prefix = f'{parent.name}.' if parent else ''
+        for child_name in self._find_children():
+            if spec := self.find_spec(module_prefix + child_name):
+                yield spec
+
     def __repr__(self):
         return f'FileFinder({self.path!r})'
 

Lib/importlib/abc.py

@@ -45,6 +45,16 @@ def invalidate_caches(self):
         This method is used by importlib.invalidate_caches().
         """
 
+    def discover(self, parent=None):
+        """An optional method which searches for possible specs with given *parent*
+        module spec. If *parent* is *None*, MetaPathFinder.discover will search
+        for top-level modules.
+
+        Returns an iterable of possible specs.
+        """
+        return ()
+
+
 _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
           machinery.PathFinder, machinery.WindowsRegistryFinder)
 
@@ -58,6 +68,15 @@ def invalidate_caches(self):
         This method is used by PathFinder.invalidate_caches().
         """
 
+    def discover(self, parent=None):
+        """An optional method which searches for possible specs with given
+        *parent* module spec. If *parent* is *None*, PathEntryFinder.discover
+        will search for top-level modules.
+
+        Returns an iterable of possible specs.
+        """
+        return ()
+
 _register(PathEntryFinder, machinery.FileFinder)
 
 

Lib/test/test_importlib/test_discover.py

@@ -0,0 +1,121 @@
+from unittest.mock import Mock
+
+from test.test_importlib import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+
+class DiscoverableFinder:
+    def __init__(self, discover=[]):
+        self._discovered_values = discover
+
+    def find_spec(self, fullname, path=None, target=None):
+        raise NotImplemented
+
+    def discover(self, parent=None):
+        yield from self._discovered_values
+
+
+class TestPathFinder:
+    """PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s)
+    registered in sys.path_hooks (and sys.path_importer_cache) to search
+    sys.path or the parent's __path__.
+
+    PathFinder.discover() should redirect to the .discover() method of the
+    PathEntryFinder for each path entry.
+    """
+
+    def test_search_path_hooks_top_level(self):
+        modules = [
+            self.machinery.ModuleSpec(name='example1', loader=None),
+            self.machinery.ModuleSpec(name='example2', loader=None),
+            self.machinery.ModuleSpec(name='example3', loader=None),
+        ]
+
+        with util.import_state(
+            path_importer_cache={
+                'discoverable': DiscoverableFinder(discover=modules),
+            },
+            path=['discoverable'],
+        ):
+            discovered = list(self.machinery.PathFinder.discover())
+
+        self.assertEqual(discovered, modules)
+
+
+    def test_search_path_hooks_parent(self):
+        parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True)
+        parent.submodule_search_locations.append('discoverable')
+
+        children = [
+            self.machinery.ModuleSpec(name='example.child1', loader=None),
+            self.machinery.ModuleSpec(name='example.child2', loader=None),
+            self.machinery.ModuleSpec(name='example.child3', loader=None),
+        ]
+
+        with util.import_state(
+            path_importer_cache={
+                'discoverable': DiscoverableFinder(discover=children)
+            },
+            path=[],
+        ):
+            discovered = list(self.machinery.PathFinder.discover(parent))
+
+        self.assertEqual(discovered, children)
+
+    def test_invalid_parent(self):
+        parent = self.machinery.ModuleSpec(name='example', loader=None)
+        with self.assertRaises(ValueError):
+            list(self.machinery.PathFinder.discover(parent))
+
+
+(
+    Frozen_TestPathFinder,
+    Source_TestPathFinder,
+) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery)
+
+
+class TestFileFinder:
+    """FileFinder implements PathEntryFinder and provides the base finder
+    implementation to search the file system.
+    """
+
+    def get_finder(self, path):
+        loader_details = [
+            (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES),
+            (self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES),
+        ]
+        return self.machinery.FileFinder(path, *loader_details)
+
+    def test_discover_top_level(self):
+        modules = {'example1', 'example2', 'example3'}
+        with util.create_modules(*modules) as mapping:
+            finder = self.get_finder(mapping['.root'])
+            discovered = list(finder.discover())
+        self.assertEqual({spec.name for spec in discovered}, modules)
+
+    def test_discover_parent(self):
+        modules = {
+            'example.child1',
+            'example.child2',
+            'example.child3',
+        }
+        with util.create_modules(*modules) as mapping:
+            example = self.get_finder(mapping['.root']).find_spec('example')
+            finder = self.get_finder(example.submodule_search_locations[0])
+            discovered = list(finder.discover(example))
+        self.assertEqual({spec.name for spec in discovered}, modules)
+
+    def test_invalid_parent(self):
+        with util.create_modules('example') as mapping:
+            finder = self.get_finder(mapping['.root'])
+            example = finder.find_spec('example')
+            with self.assertRaises(ValueError):
+                list(finder.discover(example))
+
+
+(
+    Frozen_TestFileFinder,
+    Source_TestFileFinder,
+) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery)

Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst

@@ -0,0 +1,3 @@
+Introduced :meth:`importlib.abc.MetaPathFinder.discover`
+and :meth:`importlib.abc.PathEntryFinder.discover` to allow module and submodule
+name discovery without assuming the use of traditional filesystem based imports.