17 from . import _adapters, _meta
18 from ._collections import FreezableDefaultDict, Pair
19 from ._compat import (
24 from ._functools import method_cache, pass_none
25 from ._itertools import always_iterable, unique_everseen
26 from ._meta import PackageMetadata, SimplePath
28 from contextlib import suppress
29 from importlib import import_module
30 from importlib.abc import MetaPathFinder
31 from itertools import starmap
32 from typing import List, Mapping, Optional, Union
39 'PackageNotFoundError',
45 'packages_distributions',
51 class PackageNotFoundError(ModuleNotFoundError):
52 """The package was not found."""
55 return f"No package metadata was found for {self.name}"
65 A simple entry point config parser for performance
67 >>> for item in Sectioned.read(Sectioned._sample):
69 Pair(name='sec1', value='# comments ignored')
70 Pair(name='sec1', value='a = 1')
71 Pair(name='sec1', value='b = 2')
72 Pair(name='sec2', value='a = 2')
74 >>> res = Sectioned.section_pairs(Sectioned._sample)
79 Pair(name='a', value='1')
82 Pair(name='b', value='2')
87 Pair(name='a', value='2')
92 _sample = textwrap.dedent(
105 def section_pairs(cls, text):
107 section._replace(value=Pair.parse(section.value))
108 for section in cls.read(text, filter_=cls.valid)
109 if section.name is not None
113 def read(text, filter_=None):
114 lines = filter(filter_, map(str.strip, text.splitlines()))
117 section_match = value.startswith('[') and value.endswith(']')
119 name = value.strip('[]')
121 yield Pair(name, value)
125 return line and not line.startswith('#')
128 class DeprecatedTuple:
130 Provide subscript item access for backward compatibility.
132 >>> recwarn = getfixture('recwarn')
133 >>> ep = EntryPoint(name='name', value='value', group='group')
135 ('name', 'value', 'group')
142 _warn = functools.partial(
144 "EntryPoint tuple interface is deprecated. Access members by name.",
146 stacklevel=pypy_partial(2),
149 def __getitem__(self, item):
151 return self._key()[item]
154 class EntryPoint(DeprecatedTuple):
155 """An entry point as defined by Python packaging conventions.
157 See `the packaging docs on entry points
158 <https://packaging.python.org/specifications/entry-points/>`_
159 for more information.
162 pattern = re.compile(
163 r'(?P<module>[\w.]+)\s*'
164 r'(:\s*(?P<attr>[\w.]+)\s*)?'
165 r'((?P<extras>\[.*\])\s*)?$'
168 A regular expression describing the syntax for an entry point,
169 which might look like:
173 - package.module:attribute
174 - package.module:object.attribute
175 - package.module:attr [extra1, extra2]
177 Other combinations are possible as well.
179 The expression is lenient about whitespace around the ':',
180 following the attr, and following any extras.
183 dist: Optional['Distribution'] = None
185 def __init__(self, name, value, group):
186 vars(self).update(name=name, value=value, group=group)
189 """Load the entry point from its definition. If only a module
190 is indicated by the value, return that module. Otherwise,
191 return the named object.
193 match = self.pattern.match(self.value)
194 module = import_module(match.group('module'))
195 attrs = filter(None, (match.group('attr') or '').split('.'))
196 return functools.reduce(getattr, attrs, module)
200 match = self.pattern.match(self.value)
201 return match.group('module')
205 match = self.pattern.match(self.value)
206 return match.group('attr')
210 match = self.pattern.match(self.value)
211 return list(re.finditer(r'\w+', match.group('extras') or ''))
213 def _for(self, dist):
214 vars(self).update(dist=dist)
219 Supply iter so one may construct dicts of EntryPoints by name.
222 "Construction of dict of EntryPoints is deprecated in "
223 "favor of EntryPoints."
225 warnings.warn(msg, DeprecationWarning)
226 return iter((self.name, self))
228 def matches(self, **params):
229 attrs = (getattr(self, param) for param in params)
230 return all(map(operator.eq, params.values(), attrs))
233 return self.name, self.value, self.group
235 def __lt__(self, other):
236 return self._key() < other._key()
238 def __eq__(self, other):
239 return self._key() == other._key()
241 def __setattr__(self, name, value):
242 raise AttributeError("EntryPoint objects are immutable.")
246 f'EntryPoint(name={self.name!r}, value={self.value!r}, '
247 f'group={self.group!r})'
251 return hash(self._key())
254 class DeprecatedList(list):
256 Allow an otherwise immutable object to implement mutability
259 >>> recwarn = getfixture('recwarn')
260 >>> dl = DeprecatedList(range(3))
278 >>> dl == [0, 1, 2, 5]
280 >>> dl == (0, 1, 2, 5)
288 _warn = functools.partial(
290 "EntryPoints list interface is deprecated. Cast to list if needed.",
292 stacklevel=pypy_partial(2),
295 def _wrap_deprecated_method(method_name: str): # type: ignore
296 def wrapped(self, *args, **kwargs):
298 return getattr(super(), method_name)(*args, **kwargs)
300 return method_name, wrapped
304 _wrap_deprecated_method,
305 '__setitem__ __delitem__ append reverse extend pop remove '
306 '__iadd__ insert sort'.split(),
310 def __add__(self, other):
311 if not isinstance(other, tuple):
314 return self.__class__(tuple(self) + other)
316 def __eq__(self, other):
317 if not isinstance(other, tuple):
321 return tuple(self).__eq__(other)
324 class EntryPoints(DeprecatedList):
326 An immutable collection of selectable EntryPoint objects.
331 def __getitem__(self, name): # -> EntryPoint:
333 Get the EntryPoint in self matching name.
335 if isinstance(name, int):
337 "Accessing entry points by index is deprecated. "
338 "Cast to tuple if needed.",
342 return super().__getitem__(name)
344 return next(iter(self.select(name=name)))
345 except StopIteration:
348 def select(self, **params):
350 Select entry points from self that match the
351 given parameters (typically group and/or name).
353 return EntryPoints(ep for ep in self if ep.matches(**params))
358 Return the set of all names of all entry points.
360 return {ep.name for ep in self}
365 Return the set of all groups of all entry points.
367 For coverage while SelectableGroups is present.
368 >>> EntryPoints().groups
371 return {ep.group for ep in self}
374 def _from_text_for(cls, text, dist):
375 return cls(ep._for(dist) for ep in cls._from_text(text))
378 def _from_text(text):
380 EntryPoint(name=item.value.name, value=item.value.value, group=item.name)
381 for item in Sectioned.section_pairs(text or '')
387 Compatibility add-in for mapping to indicate that
388 mapping behavior is deprecated.
390 >>> recwarn = getfixture('recwarn')
391 >>> class DeprecatedDict(Deprecated, dict): pass
392 >>> dd = DeprecatedDict(foo='bar')
393 >>> dd.get('baz', None)
402 >>> list(dd.values())
408 _warn = functools.partial(
410 "SelectableGroups dict interface is deprecated. Use select.",
412 stacklevel=pypy_partial(2),
415 def __getitem__(self, name):
417 return super().__getitem__(name)
419 def get(self, name, default=None):
421 return super().get(name, default)
425 return super().__iter__()
427 def __contains__(self, *args):
429 return super().__contains__(*args)
433 return super().keys()
437 return super().values()
440 class SelectableGroups(Deprecated, dict):
442 A backward- and forward-compatible result from
443 entry_points that fully implements the dict interface.
448 by_group = operator.attrgetter('group')
449 ordered = sorted(eps, key=by_group)
450 grouped = itertools.groupby(ordered, by_group)
451 return cls((group, EntryPoints(eps)) for group, eps in grouped)
456 Reconstruct a list of all entrypoints from the groups.
458 groups = super(Deprecated, self).values()
459 return EntryPoints(itertools.chain.from_iterable(groups))
463 return self._all.groups
469 >>> SelectableGroups().names
472 return self._all.names
474 def select(self, **params):
477 return self._all.select(**params)
480 class PackagePath(pathlib.PurePosixPath):
481 """A reference to a path in a package"""
483 def read_text(self, encoding='utf-8'):
484 with self.locate().open(encoding=encoding) as stream:
487 def read_binary(self):
488 with self.locate().open('rb') as stream:
492 """Return a path-like object for this path"""
493 return self.dist.locate_file(self)
497 def __init__(self, spec):
498 self.mode, _, self.value = spec.partition('=')
501 return f'<FileHash mode: {self.mode} value: {self.value}>'
505 """A Python distribution package."""
508 def read_text(self, filename):
509 """Attempt to load metadata file given by the name.
511 :param filename: The name of the file in the distribution info.
512 :return: The text if found, otherwise None.
516 def locate_file(self, path):
518 Given a path to a file in this distribution, return a path
523 def from_name(cls, name):
524 """Return the Distribution for the given package name.
526 :param name: The name of the distribution package to search for.
527 :return: The Distribution instance (or subclass thereof) for the named
529 :raises PackageNotFoundError: When the named package's distribution
530 metadata cannot be found.
532 for resolver in cls._discover_resolvers():
533 dists = resolver(DistributionFinder.Context(name=name))
534 dist = next(iter(dists), None)
538 raise PackageNotFoundError(name)
541 def discover(cls, **kwargs):
542 """Return an iterable of Distribution objects for all packages.
544 Pass a ``context`` or pass keyword arguments for constructing
547 :context: A ``DistributionFinder.Context`` object.
548 :return: Iterable of Distribution objects for all packages.
550 context = kwargs.pop('context', None)
551 if context and kwargs:
552 raise ValueError("cannot accept context and kwargs")
553 context = context or DistributionFinder.Context(**kwargs)
554 return itertools.chain.from_iterable(
555 resolver(context) for resolver in cls._discover_resolvers()
560 """Return a Distribution for the indicated metadata path
562 :param path: a string or path-like object
563 :return: a concrete Distribution instance for the path
565 return PathDistribution(pathlib.Path(path))
568 def _discover_resolvers():
569 """Search the meta_path for resolvers."""
571 getattr(finder, 'find_distributions', None) for finder in sys.meta_path
573 return filter(None, declared)
576 def metadata(self) -> _meta.PackageMetadata:
577 """Return the parsed metadata for this Distribution.
579 The returned object will have keys that name the various bits of
580 metadata. See PEP 566 for details.
583 self.read_text('METADATA')
584 or self.read_text('PKG-INFO')
585 # This last clause is here to support old egg-info files. Its
586 # effect is to just end up using the PathDistribution's self._path
587 # (which points to the egg-info file) attribute unchanged.
588 or self.read_text('')
590 return _adapters.Message(email.message_from_string(text))
594 """Return the 'Name' metadata for the distribution package."""
595 return self.metadata['Name']
598 def _normalized_name(self):
599 """Return a normalized version of the name."""
600 return Prepared.normalize(self.name)
604 """Return the 'Version' metadata for the distribution package."""
605 return self.metadata['Version']
608 def entry_points(self):
609 return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self)
613 """Files in this distribution.
615 :return: List of PackagePath for this distribution or None
617 Result is `None` if the metadata file that enumerates files
618 (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
620 Result may be empty if the metadata exists but is empty.
623 def make_file(name, hash=None, size_str=None):
624 result = PackagePath(name)
625 result.hash = FileHash(hash) if hash else None
626 result.size = int(size_str) if size_str else None
631 def make_files(lines):
632 return list(starmap(make_file, csv.reader(lines)))
634 return make_files(self._read_files_distinfo() or self._read_files_egginfo())
636 def _read_files_distinfo(self):
638 Read the lines of RECORD
640 text = self.read_text('RECORD')
641 return text and text.splitlines()
643 def _read_files_egginfo(self):
645 SOURCES.txt might contain literal commas, so wrap each line
648 text = self.read_text('SOURCES.txt')
649 return text and map('"{}"'.format, text.splitlines())
653 """Generated requirements specified for this Distribution"""
654 reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
655 return reqs and list(reqs)
657 def _read_dist_info_reqs(self):
658 return self.metadata.get_all('Requires-Dist')
660 def _read_egg_info_reqs(self):
661 source = self.read_text('requires.txt')
662 return pass_none(self._deps_from_requires_text)(source)
665 def _deps_from_requires_text(cls, source):
666 return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source))
669 def _convert_egg_info_reqs_to_simple_reqs(sections):
671 Historically, setuptools would solicit and store 'extra'
672 requirements, including those with environment markers,
673 in separate sections. More modern tools expect each
674 dependency to be defined separately, with any relevant
675 extras and environment markers attached directly to that
676 requirement. This method converts the former to the
677 latter. See _test_deps_from_requires_text for an example.
680 def make_condition(name):
681 return name and f'extra == "{name}"'
683 def quoted_marker(section):
684 section = section or ''
685 extra, sep, markers = section.partition(':')
686 if extra and markers:
687 markers = f'({markers})'
688 conditions = list(filter(None, [markers, make_condition(extra)]))
689 return '; ' + ' and '.join(conditions) if conditions else ''
691 def url_req_space(req):
693 PEP 508 requires a space between the url_spec and the quoted_marker.
694 Ref python/importlib_metadata#357.
696 # '@' is uniquely indicative of a url_req.
697 return ' ' * ('@' in req)
699 for section in sections:
700 space = url_req_space(section.value)
701 yield section.value + space + quoted_marker(section.name)
704 class DistributionFinder(MetaPathFinder):
706 A MetaPathFinder capable of discovering installed distributions.
711 Keyword arguments presented by the caller to
712 ``distributions()`` or ``Distribution.discover()``
713 to narrow the scope of a search for distributions
714 in all DistributionFinders.
716 Each DistributionFinder may expect any parameters
717 and should attempt to honor the canonical
718 parameters defined below when appropriate.
723 Specific name for which a distribution finder should match.
724 A name of ``None`` matches all distributions.
727 def __init__(self, **kwargs):
728 vars(self).update(kwargs)
733 The sequence of directory path that a distribution finder
736 Typically refers to Python installed package paths such as
737 "site-packages" directories and defaults to ``sys.path``.
739 return vars(self).get('path', sys.path)
742 def find_distributions(self, context=Context()):
746 Return an iterable of all Distribution instances capable of
747 loading the metadata for packages matching the ``context``,
748 a DistributionFinder.Context instance.
754 Micro-optimized class for searching a path for
757 >>> FastPath('').children()
761 @functools.lru_cache() # type: ignore
762 def __new__(cls, root):
763 return super().__new__(cls)
765 def __init__(self, root):
766 self.root = str(root)
768 def joinpath(self, child):
769 return pathlib.Path(self.root, child)
772 with suppress(Exception):
773 return os.listdir(self.root or '.')
774 with suppress(Exception):
775 return self.zip_children()
778 def zip_children(self):
779 zip_path = zipp.Path(self.root)
780 names = zip_path.root.namelist()
781 self.joinpath = zip_path.joinpath
783 return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names)
785 def search(self, name):
786 return self.lookup(self.mtime).search(name)
790 with suppress(OSError):
791 return os.stat(self.root).st_mtime
792 self.lookup.cache_clear()
795 def lookup(self, mtime):
800 def __init__(self, path: FastPath):
801 base = os.path.basename(path.root).lower()
802 base_is_egg = base.endswith(".egg")
803 self.infos = FreezableDefaultDict(list)
804 self.eggs = FreezableDefaultDict(list)
806 for child in path.children():
808 if low.endswith((".dist-info", ".egg-info")):
809 # rpartition is faster than splitext and suitable for this purpose.
810 name = low.rpartition(".")[0].partition("-")[0]
811 normalized = Prepared.normalize(name)
812 self.infos[normalized].append(path.joinpath(child))
813 elif base_is_egg and low == "egg-info":
814 name = base.rpartition(".")[0].partition("-")[0]
815 legacy_normalized = Prepared.legacy_normalize(name)
816 self.eggs[legacy_normalized].append(path.joinpath(child))
821 def search(self, prepared):
823 self.infos[prepared.normalized]
825 else itertools.chain.from_iterable(self.infos.values())
828 self.eggs[prepared.legacy_normalized]
830 else itertools.chain.from_iterable(self.eggs.values())
832 return itertools.chain(infos, eggs)
837 A prepared search for metadata on a possibly-named package.
841 legacy_normalized = None
843 def __init__(self, name):
847 self.normalized = self.normalize(name)
848 self.legacy_normalized = self.legacy_normalize(name)
853 PEP 503 normalization plus dashes as underscores.
855 return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_')
858 def legacy_normalize(name):
860 Normalize the package name as found in the convention in
861 older packaging tools versions and specs.
863 return name.lower().replace('-', '_')
866 return bool(self.name)
870 class MetadataPathFinder(NullFinder, DistributionFinder):
871 """A degenerate finder for distribution packages on the file system.
873 This finder supplies only a find_distributions() method for versions
874 of Python that do not have a PathFinder find_distributions().
877 def find_distributions(self, context=DistributionFinder.Context()):
881 Return an iterable of all Distribution instances capable of
882 loading the metadata for packages matching ``context.name``
883 (or all names if ``None`` indicated) along the paths in the list
884 of directories ``context.path``.
886 found = self._search_paths(context.name, context.path)
887 return map(PathDistribution, found)
890 def _search_paths(cls, name, paths):
891 """Find metadata directories in paths heuristically."""
892 prepared = Prepared(name)
893 return itertools.chain.from_iterable(
894 path.search(prepared) for path in map(FastPath, paths)
897 def invalidate_caches(cls):
898 FastPath.__new__.cache_clear()
901 class PathDistribution(Distribution):
902 def __init__(self, path: SimplePath):
903 """Construct a distribution.
905 :param path: SimplePath indicating the metadata directory.
909 def read_text(self, filename):
917 return self._path.joinpath(filename).read_text(encoding='utf-8')
919 read_text.__doc__ = Distribution.read_text.__doc__
921 def locate_file(self, path):
922 return self._path.parent / path
925 def _normalized_name(self):
927 Performance optimization: where possible, resolve the
928 normalized name from the file system path.
930 stem = os.path.basename(str(self._path))
931 return self._name_from_stem(stem) or super()._normalized_name
933 def _name_from_stem(self, stem):
934 name, ext = os.path.splitext(stem)
935 if ext not in ('.dist-info', '.egg-info'):
937 name, sep, rest = stem.partition('-')
941 def distribution(distribution_name):
942 """Get the ``Distribution`` instance for the named package.
944 :param distribution_name: The name of the distribution package as a string.
945 :return: A ``Distribution`` instance (or subclass thereof).
947 return Distribution.from_name(distribution_name)
950 def distributions(**kwargs):
951 """Get all ``Distribution`` instances in the current environment.
953 :return: An iterable of ``Distribution`` instances.
955 return Distribution.discover(**kwargs)
958 def metadata(distribution_name) -> _meta.PackageMetadata:
959 """Get the metadata for the named package.
961 :param distribution_name: The name of the distribution package to query.
962 :return: A PackageMetadata containing the parsed metadata.
964 return Distribution.from_name(distribution_name).metadata
967 def version(distribution_name):
968 """Get the version string for the named package.
970 :param distribution_name: The name of the distribution package to query.
971 :return: The version string for the package as defined in the package's
972 "Version" metadata key.
974 return distribution(distribution_name).version
977 def entry_points(**params) -> Union[EntryPoints, SelectableGroups]:
978 """Return EntryPoint objects for all installed packages.
980 Pass selection parameters (group or name) to filter the
981 result to entry points matching those properties (see
982 EntryPoints.select()).
984 For compatibility, returns ``SelectableGroups`` object unless
985 selection parameters are supplied. In the future, this function
986 will return ``EntryPoints`` instead of ``SelectableGroups``
987 even when no selection parameters are supplied.
989 For maximum future compatibility, pass selection parameters
990 or invoke ``.select`` with parameters on the result.
992 :return: EntryPoints or SelectableGroups for all installed packages.
994 norm_name = operator.attrgetter('_normalized_name')
995 unique = functools.partial(unique_everseen, key=norm_name)
996 eps = itertools.chain.from_iterable(
997 dist.entry_points for dist in unique(distributions())
999 return SelectableGroups.load(eps).select(**params)
1002 def files(distribution_name):
1003 """Return a list of files for the named package.
1005 :param distribution_name: The name of the distribution package to query.
1006 :return: List of files composing the distribution.
1008 return distribution(distribution_name).files
1011 def requires(distribution_name):
1013 Return a list of requirements for the named package.
1015 :return: An iterator of requirements, suitable for
1016 packaging.requirement.Requirement.
1018 return distribution(distribution_name).requires
1021 def packages_distributions() -> Mapping[str, List[str]]:
1023 Return a mapping of top-level packages to their
1026 >>> import collections.abc
1027 >>> pkgs = packages_distributions()
1028 >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values())
1031 pkg_to_dist = collections.defaultdict(list)
1032 for dist in distributions():
1033 for pkg in _top_level_declared(dist) or _top_level_inferred(dist):
1034 pkg_to_dist[pkg].append(dist.metadata['Name'])
1035 return dict(pkg_to_dist)
1038 def _top_level_declared(dist):
1039 return (dist.read_text('top_level.txt') or '').split()
1042 def _top_level_inferred(dist):
1044 f.parts[0] if len(f.parts) > 1 else f.with_suffix('').name
1045 for f in always_iterable(dist.files)
1046 if f.suffix == ".py"