a4c24b52a1bf4fd055f4a130e80f4401fe06ea6b
[SubU] /
1 import contextlib
2 import functools
3 import logging
4 from typing import (
5     TYPE_CHECKING,
6     Dict,
7     FrozenSet,
8     Iterable,
9     Iterator,
10     List,
11     Mapping,
12     NamedTuple,
13     Optional,
14     Sequence,
15     Set,
16     Tuple,
17     TypeVar,
18     cast,
19 )
20
21 from pip._vendor.packaging.requirements import InvalidRequirement
22 from pip._vendor.packaging.specifiers import SpecifierSet
23 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
24 from pip._vendor.resolvelib import ResolutionImpossible
25
26 from pip._internal.cache import CacheEntry, WheelCache
27 from pip._internal.exceptions import (
28     DistributionNotFound,
29     InstallationError,
30     MetadataInconsistent,
31     UnsupportedPythonVersion,
32     UnsupportedWheel,
33 )
34 from pip._internal.index.package_finder import PackageFinder
35 from pip._internal.metadata import BaseDistribution, get_default_environment
36 from pip._internal.models.link import Link
37 from pip._internal.models.wheel import Wheel
38 from pip._internal.operations.prepare import RequirementPreparer
39 from pip._internal.req.constructors import install_req_from_link_and_ireq
40 from pip._internal.req.req_install import (
41     InstallRequirement,
42     check_invalid_constraint_type,
43 )
44 from pip._internal.resolution.base import InstallRequirementProvider
45 from pip._internal.utils.compatibility_tags import get_supported
46 from pip._internal.utils.hashes import Hashes
47 from pip._internal.utils.packaging import get_requirement
48 from pip._internal.utils.virtualenv import running_under_virtualenv
49
50 from .base import Candidate, CandidateVersion, Constraint, Requirement
51 from .candidates import (
52     AlreadyInstalledCandidate,
53     BaseCandidate,
54     EditableCandidate,
55     ExtrasCandidate,
56     LinkCandidate,
57     RequiresPythonCandidate,
58     as_base_candidate,
59 )
60 from .found_candidates import FoundCandidates, IndexCandidateInfo
61 from .requirements import (
62     ExplicitRequirement,
63     RequiresPythonRequirement,
64     SpecifierRequirement,
65     UnsatisfiableRequirement,
66 )
67
68 if TYPE_CHECKING:
69     from typing import Protocol
70
71     class ConflictCause(Protocol):
72         requirement: RequiresPythonRequirement
73         parent: Candidate
74
75
76 logger = logging.getLogger(__name__)
77
78 C = TypeVar("C")
79 Cache = Dict[Link, C]
80
81
82 class CollectedRootRequirements(NamedTuple):
83     requirements: List[Requirement]
84     constraints: Dict[str, Constraint]
85     user_requested: Dict[str, int]
86
87
88 class Factory:
89     def __init__(
90         self,
91         finder: PackageFinder,
92         preparer: RequirementPreparer,
93         make_install_req: InstallRequirementProvider,
94         wheel_cache: Optional[WheelCache],
95         use_user_site: bool,
96         force_reinstall: bool,
97         ignore_installed: bool,
98         ignore_requires_python: bool,
99         py_version_info: Optional[Tuple[int, ...]] = None,
100     ) -> None:
101         self._finder = finder
102         self.preparer = preparer
103         self._wheel_cache = wheel_cache
104         self._python_candidate = RequiresPythonCandidate(py_version_info)
105         self._make_install_req_from_spec = make_install_req
106         self._use_user_site = use_user_site
107         self._force_reinstall = force_reinstall
108         self._ignore_requires_python = ignore_requires_python
109
110         self._build_failures: Cache[InstallationError] = {}
111         self._link_candidate_cache: Cache[LinkCandidate] = {}
112         self._editable_candidate_cache: Cache[EditableCandidate] = {}
113         self._installed_candidate_cache: Dict[str, AlreadyInstalledCandidate] = {}
114         self._extras_candidate_cache: Dict[
115             Tuple[int, FrozenSet[str]], ExtrasCandidate
116         ] = {}
117
118         if not ignore_installed:
119             env = get_default_environment()
120             self._installed_dists = {
121                 dist.canonical_name: dist
122                 for dist in env.iter_installed_distributions(local_only=False)
123             }
124         else:
125             self._installed_dists = {}
126
127     @property
128     def force_reinstall(self) -> bool:
129         return self._force_reinstall
130
131     def _fail_if_link_is_unsupported_wheel(self, link: Link) -> None:
132         if not link.is_wheel:
133             return
134         wheel = Wheel(link.filename)
135         if wheel.supported(self._finder.target_python.get_tags()):
136             return
137         msg = f"{link.filename} is not a supported wheel on this platform."
138         raise UnsupportedWheel(msg)
139
140     def _make_extras_candidate(
141         self, base: BaseCandidate, extras: FrozenSet[str]
142     ) -> ExtrasCandidate:
143         cache_key = (id(base), extras)
144         try:
145             candidate = self._extras_candidate_cache[cache_key]
146         except KeyError:
147             candidate = ExtrasCandidate(base, extras)
148             self._extras_candidate_cache[cache_key] = candidate
149         return candidate
150
151     def _make_candidate_from_dist(
152         self,
153         dist: BaseDistribution,
154         extras: FrozenSet[str],
155         template: InstallRequirement,
156     ) -> Candidate:
157         try:
158             base = self._installed_candidate_cache[dist.canonical_name]
159         except KeyError:
160             base = AlreadyInstalledCandidate(dist, template, factory=self)
161             self._installed_candidate_cache[dist.canonical_name] = base
162         if not extras:
163             return base
164         return self._make_extras_candidate(base, extras)
165
166     def _make_candidate_from_link(
167         self,
168         link: Link,
169         extras: FrozenSet[str],
170         template: InstallRequirement,
171         name: Optional[NormalizedName],
172         version: Optional[CandidateVersion],
173     ) -> Optional[Candidate]:
174         # TODO: Check already installed candidate, and use it if the link and
175         # editable flag match.
176
177         if link in self._build_failures:
178             # We already tried this candidate before, and it does not build.
179             # Don't bother trying again.
180             return None
181
182         if template.editable:
183             if link not in self._editable_candidate_cache:
184                 try:
185                     self._editable_candidate_cache[link] = EditableCandidate(
186                         link,
187                         template,
188                         factory=self,
189                         name=name,
190                         version=version,
191                     )
192                 except MetadataInconsistent as e:
193                     logger.info(
194                         "Discarding [blue underline]%s[/]: [yellow]%s[reset]",
195                         link,
196                         e,
197                         extra={"markup": True},
198                     )
199                     self._build_failures[link] = e
200                     return None
201
202             base: BaseCandidate = self._editable_candidate_cache[link]
203         else:
204             if link not in self._link_candidate_cache:
205                 try:
206                     self._link_candidate_cache[link] = LinkCandidate(
207                         link,
208                         template,
209                         factory=self,
210                         name=name,
211                         version=version,
212                     )
213                 except MetadataInconsistent as e:
214                     logger.info(
215                         "Discarding [blue underline]%s[/]: [yellow]%s[reset]",
216                         link,
217                         e,
218                         extra={"markup": True},
219                     )
220                     self._build_failures[link] = e
221                     return None
222             base = self._link_candidate_cache[link]
223
224         if not extras:
225             return base
226         return self._make_extras_candidate(base, extras)
227
228     def _iter_found_candidates(
229         self,
230         ireqs: Sequence[InstallRequirement],
231         specifier: SpecifierSet,
232         hashes: Hashes,
233         prefers_installed: bool,
234         incompatible_ids: Set[int],
235     ) -> Iterable[Candidate]:
236         if not ireqs:
237             return ()
238
239         # The InstallRequirement implementation requires us to give it a
240         # "template". Here we just choose the first requirement to represent
241         # all of them.
242         # Hopefully the Project model can correct this mismatch in the future.
243         template = ireqs[0]
244         assert template.req, "Candidates found on index must be PEP 508"
245         name = canonicalize_name(template.req.name)
246
247         extras: FrozenSet[str] = frozenset()
248         for ireq in ireqs:
249             assert ireq.req, "Candidates found on index must be PEP 508"
250             specifier &= ireq.req.specifier
251             hashes &= ireq.hashes(trust_internet=False)
252             extras |= frozenset(ireq.extras)
253
254         def _get_installed_candidate() -> Optional[Candidate]:
255             """Get the candidate for the currently-installed version."""
256             # If --force-reinstall is set, we want the version from the index
257             # instead, so we "pretend" there is nothing installed.
258             if self._force_reinstall:
259                 return None
260             try:
261                 installed_dist = self._installed_dists[name]
262             except KeyError:
263                 return None
264             # Don't use the installed distribution if its version does not fit
265             # the current dependency graph.
266             if not specifier.contains(installed_dist.version, prereleases=True):
267                 return None
268             candidate = self._make_candidate_from_dist(
269                 dist=installed_dist,
270                 extras=extras,
271                 template=template,
272             )
273             # The candidate is a known incompatibility. Don't use it.
274             if id(candidate) in incompatible_ids:
275                 return None
276             return candidate
277
278         def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]:
279             result = self._finder.find_best_candidate(
280                 project_name=name,
281                 specifier=specifier,
282                 hashes=hashes,
283             )
284             icans = list(result.iter_applicable())
285
286             # PEP 592: Yanked releases are ignored unless the specifier
287             # explicitly pins a version (via '==' or '===') that can be
288             # solely satisfied by a yanked release.
289             all_yanked = all(ican.link.is_yanked for ican in icans)
290
291             def is_pinned(specifier: SpecifierSet) -> bool:
292                 for sp in specifier:
293                     if sp.operator == "===":
294                         return True
295                     if sp.operator != "==":
296                         continue
297                     if sp.version.endswith(".*"):
298                         continue
299                     return True
300                 return False
301
302             pinned = is_pinned(specifier)
303
304             # PackageFinder returns earlier versions first, so we reverse.
305             for ican in reversed(icans):
306                 if not (all_yanked and pinned) and ican.link.is_yanked:
307                     continue
308                 func = functools.partial(
309                     self._make_candidate_from_link,
310                     link=ican.link,
311                     extras=extras,
312                     template=template,
313                     name=name,
314                     version=ican.version,
315                 )
316                 yield ican.version, func
317
318         return FoundCandidates(
319             iter_index_candidate_infos,
320             _get_installed_candidate(),
321             prefers_installed,
322             incompatible_ids,
323         )
324
325     def _iter_explicit_candidates_from_base(
326         self,
327         base_requirements: Iterable[Requirement],
328         extras: FrozenSet[str],
329     ) -> Iterator[Candidate]:
330         """Produce explicit candidates from the base given an extra-ed package.
331
332         :param base_requirements: Requirements known to the resolver. The
333             requirements are guaranteed to not have extras.
334         :param extras: The extras to inject into the explicit requirements'
335             candidates.
336         """
337         for req in base_requirements:
338             lookup_cand, _ = req.get_candidate_lookup()
339             if lookup_cand is None:  # Not explicit.
340                 continue
341             # We've stripped extras from the identifier, and should always
342             # get a BaseCandidate here, unless there's a bug elsewhere.
343             base_cand = as_base_candidate(lookup_cand)
344             assert base_cand is not None, "no extras here"
345             yield self._make_extras_candidate(base_cand, extras)
346
347     def _iter_candidates_from_constraints(
348         self,
349         identifier: str,
350         constraint: Constraint,
351         template: InstallRequirement,
352     ) -> Iterator[Candidate]:
353         """Produce explicit candidates from constraints.
354
355         This creates "fake" InstallRequirement objects that are basically clones
356         of what "should" be the template, but with original_link set to link.
357         """
358         for link in constraint.links:
359             self._fail_if_link_is_unsupported_wheel(link)
360             candidate = self._make_candidate_from_link(
361                 link,
362                 extras=frozenset(),
363                 template=install_req_from_link_and_ireq(link, template),
364                 name=canonicalize_name(identifier),
365                 version=None,
366             )
367             if candidate:
368                 yield candidate
369
370     def find_candidates(
371         self,
372         identifier: str,
373         requirements: Mapping[str, Iterable[Requirement]],
374         incompatibilities: Mapping[str, Iterator[Candidate]],
375         constraint: Constraint,
376         prefers_installed: bool,
377     ) -> Iterable[Candidate]:
378         # Collect basic lookup information from the requirements.
379         explicit_candidates: Set[Candidate] = set()
380         ireqs: List[InstallRequirement] = []
381         for req in requirements[identifier]:
382             cand, ireq = req.get_candidate_lookup()
383             if cand is not None:
384                 explicit_candidates.add(cand)
385             if ireq is not None:
386                 ireqs.append(ireq)
387
388         # If the current identifier contains extras, add explicit candidates
389         # from entries from extra-less identifier.
390         with contextlib.suppress(InvalidRequirement):
391             parsed_requirement = get_requirement(identifier)
392             explicit_candidates.update(
393                 self._iter_explicit_candidates_from_base(
394                     requirements.get(parsed_requirement.name, ()),
395                     frozenset(parsed_requirement.extras),
396                 ),
397             )
398
399         # Add explicit candidates from constraints. We only do this if there are
400         # known ireqs, which represent requirements not already explicit. If
401         # there are no ireqs, we're constraining already-explicit requirements,
402         # which is handled later when we return the explicit candidates.
403         if ireqs:
404             try:
405                 explicit_candidates.update(
406                     self._iter_candidates_from_constraints(
407                         identifier,
408                         constraint,
409                         template=ireqs[0],
410                     ),
411                 )
412             except UnsupportedWheel:
413                 # If we're constrained to install a wheel incompatible with the
414                 # target architecture, no candidates will ever be valid.
415                 return ()
416
417         # Since we cache all the candidates, incompatibility identification
418         # can be made quicker by comparing only the id() values.
419         incompat_ids = {id(c) for c in incompatibilities.get(identifier, ())}
420
421         # If none of the requirements want an explicit candidate, we can ask
422         # the finder for candidates.
423         if not explicit_candidates:
424             return self._iter_found_candidates(
425                 ireqs,
426                 constraint.specifier,
427                 constraint.hashes,
428                 prefers_installed,
429                 incompat_ids,
430             )
431
432         return (
433             c
434             for c in explicit_candidates
435             if id(c) not in incompat_ids
436             and constraint.is_satisfied_by(c)
437             and all(req.is_satisfied_by(c) for req in requirements[identifier])
438         )
439
440     def _make_requirement_from_install_req(
441         self, ireq: InstallRequirement, requested_extras: Iterable[str]
442     ) -> Optional[Requirement]:
443         if not ireq.match_markers(requested_extras):
444             logger.info(
445                 "Ignoring %s: markers '%s' don't match your environment",
446                 ireq.name,
447                 ireq.markers,
448             )
449             return None
450         if not ireq.link:
451             return SpecifierRequirement(ireq)
452         self._fail_if_link_is_unsupported_wheel(ireq.link)
453         cand = self._make_candidate_from_link(
454             ireq.link,
455             extras=frozenset(ireq.extras),
456             template=ireq,
457             name=canonicalize_name(ireq.name) if ireq.name else None,
458             version=None,
459         )
460         if cand is None:
461             # There's no way we can satisfy a URL requirement if the underlying
462             # candidate fails to build. An unnamed URL must be user-supplied, so
463             # we fail eagerly. If the URL is named, an unsatisfiable requirement
464             # can make the resolver do the right thing, either backtrack (and
465             # maybe find some other requirement that's buildable) or raise a
466             # ResolutionImpossible eventually.
467             if not ireq.name:
468                 raise self._build_failures[ireq.link]
469             return UnsatisfiableRequirement(canonicalize_name(ireq.name))
470         return self.make_requirement_from_candidate(cand)
471
472     def collect_root_requirements(
473         self, root_ireqs: List[InstallRequirement]
474     ) -> CollectedRootRequirements:
475         collected = CollectedRootRequirements([], {}, {})
476         for i, ireq in enumerate(root_ireqs):
477             if ireq.constraint:
478                 # Ensure we only accept valid constraints
479                 problem = check_invalid_constraint_type(ireq)
480                 if problem:
481                     raise InstallationError(problem)
482                 if not ireq.match_markers():
483                     continue
484                 assert ireq.name, "Constraint must be named"
485                 name = canonicalize_name(ireq.name)
486                 if name in collected.constraints:
487                     collected.constraints[name] &= ireq
488                 else:
489                     collected.constraints[name] = Constraint.from_ireq(ireq)
490             else:
491                 req = self._make_requirement_from_install_req(
492                     ireq,
493                     requested_extras=(),
494                 )
495                 if req is None:
496                     continue
497                 if ireq.user_supplied and req.name not in collected.user_requested:
498                     collected.user_requested[req.name] = i
499                 collected.requirements.append(req)
500         return collected
501
502     def make_requirement_from_candidate(
503         self, candidate: Candidate
504     ) -> ExplicitRequirement:
505         return ExplicitRequirement(candidate)
506
507     def make_requirement_from_spec(
508         self,
509         specifier: str,
510         comes_from: Optional[InstallRequirement],
511         requested_extras: Iterable[str] = (),
512     ) -> Optional[Requirement]:
513         ireq = self._make_install_req_from_spec(specifier, comes_from)
514         return self._make_requirement_from_install_req(ireq, requested_extras)
515
516     def make_requires_python_requirement(
517         self,
518         specifier: SpecifierSet,
519     ) -> Optional[Requirement]:
520         if self._ignore_requires_python:
521             return None
522         # Don't bother creating a dependency for an empty Requires-Python.
523         if not str(specifier):
524             return None
525         return RequiresPythonRequirement(specifier, self._python_candidate)
526
527     def get_wheel_cache_entry(
528         self, link: Link, name: Optional[str]
529     ) -> Optional[CacheEntry]:
530         """Look up the link in the wheel cache.
531
532         If ``preparer.require_hashes`` is True, don't use the wheel cache,
533         because cached wheels, always built locally, have different hashes
534         than the files downloaded from the index server and thus throw false
535         hash mismatches. Furthermore, cached wheels at present have
536         nondeterministic contents due to file modification times.
537         """
538         if self._wheel_cache is None or self.preparer.require_hashes:
539             return None
540         return self._wheel_cache.get_cache_entry(
541             link=link,
542             package_name=name,
543             supported_tags=get_supported(),
544         )
545
546     def get_dist_to_uninstall(self, candidate: Candidate) -> Optional[BaseDistribution]:
547         # TODO: Are there more cases this needs to return True? Editable?
548         dist = self._installed_dists.get(candidate.project_name)
549         if dist is None:  # Not installed, no uninstallation required.
550             return None
551
552         # We're installing into global site. The current installation must
553         # be uninstalled, no matter it's in global or user site, because the
554         # user site installation has precedence over global.
555         if not self._use_user_site:
556             return dist
557
558         # We're installing into user site. Remove the user site installation.
559         if dist.in_usersite:
560             return dist
561
562         # We're installing into user site, but the installed incompatible
563         # package is in global site. We can't uninstall that, and would let
564         # the new user installation to "shadow" it. But shadowing won't work
565         # in virtual environments, so we error out.
566         if running_under_virtualenv() and dist.in_site_packages:
567             message = (
568                 f"Will not install to the user site because it will lack "
569                 f"sys.path precedence to {dist.raw_name} in {dist.location}"
570             )
571             raise InstallationError(message)
572         return None
573
574     def _report_requires_python_error(
575         self, causes: Sequence["ConflictCause"]
576     ) -> UnsupportedPythonVersion:
577         assert causes, "Requires-Python error reported with no cause"
578
579         version = self._python_candidate.version
580
581         if len(causes) == 1:
582             specifier = str(causes[0].requirement.specifier)
583             message = (
584                 f"Package {causes[0].parent.name!r} requires a different "
585                 f"Python: {version} not in {specifier!r}"
586             )
587             return UnsupportedPythonVersion(message)
588
589         message = f"Packages require a different Python. {version} not in:"
590         for cause in causes:
591             package = cause.parent.format_for_error()
592             specifier = str(cause.requirement.specifier)
593             message += f"\n{specifier!r} (required by {package})"
594         return UnsupportedPythonVersion(message)
595
596     def _report_single_requirement_conflict(
597         self, req: Requirement, parent: Optional[Candidate]
598     ) -> DistributionNotFound:
599         if parent is None:
600             req_disp = str(req)
601         else:
602             req_disp = f"{req} (from {parent.name})"
603
604         cands = self._finder.find_all_candidates(req.project_name)
605         skipped_by_requires_python = self._finder.requires_python_skipped_reasons()
606         versions = [str(v) for v in sorted({c.version for c in cands})]
607
608         if skipped_by_requires_python:
609             logger.critical(
610                 "Ignored the following versions that require a different python "
611                 "version: %s",
612                 "; ".join(skipped_by_requires_python) or "none",
613             )
614         logger.critical(
615             "Could not find a version that satisfies the requirement %s "
616             "(from versions: %s)",
617             req_disp,
618             ", ".join(versions) or "none",
619         )
620         if str(req) == "requirements.txt":
621             logger.info(
622                 "HINT: You are attempting to install a package literally "
623                 'named "requirements.txt" (which cannot exist). Consider '
624                 "using the '-r' flag to install the packages listed in "
625                 "requirements.txt"
626             )
627
628         return DistributionNotFound(f"No matching distribution found for {req}")
629
630     def get_installation_error(
631         self,
632         e: "ResolutionImpossible[Requirement, Candidate]",
633         constraints: Dict[str, Constraint],
634     ) -> InstallationError:
635
636         assert e.causes, "Installation error reported with no cause"
637
638         # If one of the things we can't solve is "we need Python X.Y",
639         # that is what we report.
640         requires_python_causes = [
641             cause
642             for cause in e.causes
643             if isinstance(cause.requirement, RequiresPythonRequirement)
644             and not cause.requirement.is_satisfied_by(self._python_candidate)
645         ]
646         if requires_python_causes:
647             # The comprehension above makes sure all Requirement instances are
648             # RequiresPythonRequirement, so let's cast for convenience.
649             return self._report_requires_python_error(
650                 cast("Sequence[ConflictCause]", requires_python_causes),
651             )
652
653         # Otherwise, we have a set of causes which can't all be satisfied
654         # at once.
655
656         # The simplest case is when we have *one* cause that can't be
657         # satisfied. We just report that case.
658         if len(e.causes) == 1:
659             req, parent = e.causes[0]
660             if req.name not in constraints:
661                 return self._report_single_requirement_conflict(req, parent)
662
663         # OK, we now have a list of requirements that can't all be
664         # satisfied at once.
665
666         # A couple of formatting helpers
667         def text_join(parts: List[str]) -> str:
668             if len(parts) == 1:
669                 return parts[0]
670
671             return ", ".join(parts[:-1]) + " and " + parts[-1]
672
673         def describe_trigger(parent: Candidate) -> str:
674             ireq = parent.get_install_requirement()
675             if not ireq or not ireq.comes_from:
676                 return f"{parent.name}=={parent.version}"
677             if isinstance(ireq.comes_from, InstallRequirement):
678                 return str(ireq.comes_from.name)
679             return str(ireq.comes_from)
680
681         triggers = set()
682         for req, parent in e.causes:
683             if parent is None:
684                 # This is a root requirement, so we can report it directly
685                 trigger = req.format_for_error()
686             else:
687                 trigger = describe_trigger(parent)
688             triggers.add(trigger)
689
690         if triggers:
691             info = text_join(sorted(triggers))
692         else:
693             info = "the requested packages"
694
695         msg = (
696             "Cannot install {} because these package versions "
697             "have conflicting dependencies.".format(info)
698         )
699         logger.critical(msg)
700         msg = "\nThe conflict is caused by:"
701
702         relevant_constraints = set()
703         for req, parent in e.causes:
704             if req.name in constraints:
705                 relevant_constraints.add(req.name)
706             msg = msg + "\n    "
707             if parent:
708                 msg = msg + f"{parent.name} {parent.version} depends on "
709             else:
710                 msg = msg + "The user requested "
711             msg = msg + req.format_for_error()
712         for key in relevant_constraints:
713             spec = constraints[key].specifier
714             msg += f"\n    The user requested (constraint) {key}{spec}"
715
716         msg = (
717             msg
718             + "\n\n"
719             + "To fix this you could try to:\n"
720             + "1. loosen the range of package versions you've specified\n"
721             + "2. remove package versions to allow pip attempt to solve "
722             + "the dependency conflict\n"
723         )
724
725         logger.info(msg)
726
727         return DistributionNotFound(
728             "ResolutionImpossible: for help visit "
729             "https://pip.pypa.io/en/latest/topics/dependency-resolution/"
730             "#dealing-with-dependency-conflicts"
731         )