From 24c210503d1ba6407a527a9f41f77d2e177aa031 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Thu, 20 Jul 2023 20:31:51 +0200 Subject: [PATCH 1/3] Update types using fresh dependencies --- .pre-commit-config.yaml | 8 +++----- piptools/__main__.py | 2 +- piptools/_compat/pip_compat.py | 15 ++++++++------- piptools/repositories/local.py | 14 +++++++------- piptools/repositories/pypi.py | 28 ++++++++++++++++++---------- piptools/resolver.py | 31 +++++++++++++++++++++---------- piptools/sync.py | 11 +++++------ piptools/utils.py | 29 +++++++++++++++++------------ pyproject.toml | 5 ++++- tests/conftest.py | 6 +++--- 10 files changed, 87 insertions(+), 62 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 551140d68..0a7f54aa8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,11 +28,9 @@ repos: # Keep exclude in sync with mypy own excludes exclude: ^tests/test_data/ additional_dependencies: - - click==8.0.1 - - pep517==0.10.0 - - toml==0.10.2 - - pip==20.3.4 - - build==0.9.0 + - click==8.1.6 + - pip==23.2 + - build==0.10.0 - repo: https://github.com/PyCQA/bandit rev: 1.7.5 hooks: diff --git a/piptools/__main__.py b/piptools/__main__.py index 9cd2bab3e..e63c2a421 100644 --- a/piptools/__main__.py +++ b/piptools/__main__.py @@ -5,7 +5,7 @@ from piptools.scripts import compile, sync -@click.group() +@click.group def cli() -> None: pass diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 6409fbc2a..fa230459b 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,12 +2,13 @@ import optparse from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast +from typing import Iterable, Iterator, Set, cast import pip from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution +from pip._internal.metadata.importlib import Distribution as _ImportLibDist from pip._internal.metadata.pkg_resources import Distribution as _PkgResourcesDist from pip._internal.models.direct_url import DirectUrl from pip._internal.network.session import PipSession @@ -23,8 +24,6 @@ # importlib.metadata, so this compat layer allows for a consistent access # pattern. In pip 22.1, importlib.metadata became the default on Python 3.11 # (and later), but is overridable. `select_backend` returns what's being used. -if TYPE_CHECKING: - from pip._internal.metadata.importlib import Distribution as _ImportLibDist @dataclass(frozen=True) @@ -40,8 +39,10 @@ def from_pip_distribution(cls, dist: BaseDistribution) -> Distribution: # instead of specializing by type. if isinstance(dist, _PkgResourcesDist): return cls._from_pkg_resources(dist) - else: + elif isinstance(dist, _ImportLibDist): return cls._from_importlib(dist) + else: + raise NotImplementedError @classmethod def _from_pkg_resources(cls, dist: _PkgResourcesDist) -> Distribution: @@ -78,15 +79,15 @@ def parse_requirements( def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: - kwargs: dict[str, str | None] = {"cache_dir": cache_dir} + kwargs: dict[str, str] = {"cache_dir": cache_dir} if PIP_VERSION[:2] <= (23, 0): - kwargs["format_control"] = format_control + kwargs["format_control"] = format_control # type: ignore return WheelCache(**kwargs) def get_dev_pkgs() -> set[str]: if PIP_VERSION[:2] <= (23, 1): - from pip._internal.commands.freeze import DEV_PKGS + from pip._internal.commands.freeze import DEV_PKGS # type: ignore return cast(Set[str], DEV_PKGS) diff --git a/piptools/repositories/local.py b/piptools/repositories/local.py index 76cbea8fc..6d94dc7c2 100644 --- a/piptools/repositories/local.py +++ b/piptools/repositories/local.py @@ -2,11 +2,10 @@ import optparse from contextlib import contextmanager -from typing import Iterator, Mapping, cast +from typing import Iterator, Mapping from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder -from pip._internal.models.candidate import InstallationCandidate from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.utils.hashes import FAVORITE_HASH @@ -18,17 +17,18 @@ def ireq_satisfied_by_existing_pin( - ireq: InstallRequirement, existing_pin: InstallationCandidate + ireq: InstallRequirement, existing_pin: InstallRequirement ) -> bool: """ Return True if the given InstallationRequirement is satisfied by the previously encountered version pin. """ + assert ireq.req is not None + assert existing_pin.req is not None version = next(iter(existing_pin.req.specifier)).version - result = ireq.req.specifier.contains( + return ireq.req.specifier.contains( version, prereleases=existing_pin.req.specifier.prereleases ) - return cast(bool, result) class LocalRequirementsRepository(BaseRepository): @@ -44,7 +44,7 @@ class LocalRequirementsRepository(BaseRepository): def __init__( self, - existing_pins: Mapping[str, InstallationCandidate], + existing_pins: Mapping[str, InstallRequirement], proxied_repository: PyPIRepository, reuse_hashes: bool = True, ): @@ -74,7 +74,7 @@ def clear_caches(self) -> None: def find_best_match( self, ireq: InstallRequirement, prereleases: bool | None = None - ) -> InstallationCandidate: + ) -> InstallRequirement: key = key_from_ireq(ireq) existing_pin = self.existing_pins.get(key) if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 5a4448e3f..93aaa5f3f 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -7,7 +7,7 @@ import os from contextlib import contextmanager from shutil import rmtree -from typing import Any, BinaryIO, ContextManager, Iterator, NamedTuple +from typing import Any, BinaryIO, ContextManager, Iterable, Iterator, NamedTuple from click import progressbar from pip._internal.cache import WheelCache @@ -21,6 +21,7 @@ from pip._internal.network.session import PipSession from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.req import InstallRequirement, RequirementSet +from pip._internal.resolution.legacy.resolver import Resolver from pip._internal.utils.hashes import FAVORITE_HASH from pip._internal.utils.logging import indent_log, setup_logging from pip._internal.utils.misc import normalize_path @@ -64,7 +65,9 @@ def __init__(self, pip_args: list[str], cache_dir: str): # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are deferred to pip. - self._command: InstallCommand = create_command("install") + command = create_command("install") + assert isinstance(command, InstallCommand) + self._command = command options, _ = self.command.parse_args(pip_args) if options.cache_dir: @@ -136,6 +139,7 @@ def find_best_match( if ireq.editable or is_url_requirement(ireq): return ireq # return itself as the best match + assert ireq.name is not None all_candidates = self.find_all_candidates(ireq.name) candidates_by_version = lookup_table(all_candidates, key=candidate_version) matching_versions = ireq.specifier.filter( @@ -155,6 +159,7 @@ def find_best_match( best_candidate = best_candidate_result.best_candidate # Turn the candidate into a pinned InstallRequirement + assert best_candidate is not None return make_install_requirement( best_candidate.name, best_candidate.version, @@ -166,11 +171,11 @@ def resolve_reqs( download_dir: str | None, ireq: InstallRequirement, wheel_cache: WheelCache, - ) -> set[InstallationCandidate]: + ) -> set[InstallRequirement]: with get_build_tracker() as build_tracker, TempDirectory( kind="resolver" ) as temp_dir, indent_log(): - preparer_kwargs = { + preparer_kwargs: dict[str, Any] = { "temp_build_dir": temp_dir, "options": self.options, "session": self.session, @@ -199,6 +204,7 @@ def resolve_reqs( force_reinstall=False, upgrade_strategy="to-satisfy-only", ) + assert isinstance(resolver, Resolver) results = resolver._resolve_one(reqset, ireq) if not ireq.prepared: # If still not prepared, e.g. a constraint, do enough to assign @@ -323,6 +329,7 @@ def get_hashes(self, ireq: InstallRequirement) -> set[str]: if not is_pinned_requirement(ireq): raise TypeError(f"Expected pinned requirement, got {ireq}") + assert ireq.name is not None log.debug(ireq.name) with log.indentation(): @@ -385,6 +392,7 @@ def _get_matching_candidates( # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. + assert ireq.name is not None all_candidates = self.find_all_candidates(ireq.name) candidates_by_version = lookup_table(all_candidates, key=candidate_version) matching_versions = list( @@ -431,11 +439,11 @@ def allow_all_wheels(self) -> Iterator[None]: the previous non-patched calls will interfere. """ - def _wheel_supported(self: Wheel, tags: list[Tag]) -> bool: + def _wheel_supported(self: Wheel, tags: Iterable[Tag]) -> bool: # Ignore current platform. Support everything. return True - def _wheel_support_index_min(self: Wheel, tags: list[Tag]) -> int: + def _wheel_support_index_min(self: Wheel, tags: Iterable[Tag]) -> int: # All wheels are equal priority for sorting. return 0 @@ -443,8 +451,8 @@ def _wheel_support_index_min(self: Wheel, tags: list[Tag]) -> int: original_support_index_min = Wheel.support_index_min original_cache = self._available_candidates_cache - Wheel.supported = _wheel_supported - Wheel.support_index_min = _wheel_support_index_min + Wheel.supported = _wheel_supported # type: ignore[method-assign] + Wheel.support_index_min = _wheel_support_index_min # type: ignore[method-assign] self._available_candidates_cache = {} # If we don't clear this cache then it can contain results from an @@ -454,8 +462,8 @@ def _wheel_support_index_min(self: Wheel, tags: list[Tag]) -> int: try: yield finally: - Wheel.supported = original_wheel_supported - Wheel.support_index_min = original_support_index_min + Wheel.supported = original_wheel_supported # type: ignore[method-assign] + Wheel.support_index_min = original_support_index_min # type: ignore[method-assign] self._available_candidates_cache = original_cache diff --git a/piptools/resolver.py b/piptools/resolver.py index b6d2e1378..850faa5f6 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -108,7 +108,7 @@ def combine_install_requirements( req.specifier &= ireq.req.specifier constraint &= ireq.constraint - extras |= ireq.extras + extras |= set(ireq.extras) if req is not None: req.extras = set(extras) @@ -141,11 +141,16 @@ def combine_install_requirements( extras=extras, **link_attrs, ) - combined_ireq._source_ireqs = source_ireqs + combined_ireq._source_ireqs = source_ireqs # type: ignore[attr-defined] return combined_ireq +def key_from_req_summary(req_summary: RequirementSummary) -> str: + assert req_summary.req is not None + return key_from_req(req_summary.req) + + class BaseResolver(metaclass=ABCMeta): repository: BaseRepository unsafe_constraints: set[InstallRequirement] @@ -358,11 +363,11 @@ def _resolve_one_round(self) -> tuple[bool, set[InstallRequirement]]: log.debug("") log.debug("New dependencies found in this round:") with log.indentation(): - for new_dependency in sorted(diff, key=key_from_ireq): + for new_dependency in sorted(diff, key=key_from_req_summary): log.debug(f"adding {new_dependency}") log.debug("Removed dependencies in this round:") with log.indentation(): - for removed_dependency in sorted(removed, key=key_from_ireq): + for removed_dependency in sorted(removed, key=key_from_req_summary): log.debug(f"removing {removed_dependency}") # Store the last round's results in the their_constraints @@ -409,7 +414,7 @@ def get_best_match(self, ireq: InstallRequirement) -> InstallRequirement: ) best_match.comes_from = ireq.comes_from if hasattr(ireq, "_source_ireqs"): - best_match._source_ireqs = ireq._source_ireqs + best_match._source_ireqs = ireq._source_ireqs # type: ignore[attr-defined] return best_match def _iter_dependencies( @@ -564,7 +569,7 @@ def resolve(self, max_rounds: int = 10) -> set[InstallRequirement]: globally_managed=True, ) - preparer_kwargs = { + preparer_kwargs: dict[str, Any] = { "temp_build_dir": temp_dir, "options": self.options, "session": self.session, @@ -586,6 +591,7 @@ def resolve(self, max_rounds: int = 10) -> set[InstallRequirement]: use_pep517=self.options.use_pep517, upgrade_strategy="to-satisfy-only", ) + assert isinstance(resolver, Resolver) self.command.trace_basic_info(self.finder) @@ -702,7 +708,9 @@ def _get_install_requirements( for extras_candidate in extras_candidates: project_name = canonicalize_name(extras_candidate.project_name) ireq = result_ireqs[project_name] - ireq.extras |= extras_candidate.extras + ireq.extras = set(ireq.extras) | extras_candidate.extras + + assert ireq.req is not None ireq.req.extras |= extras_candidate.extras return set(result_ireqs.values()) @@ -759,6 +767,7 @@ def _get_install_requirement_from_candidate( # Canonicalize name assert ireq.name is not None + assert pinned_ireq.req is not None pinned_ireq.req.name = canonicalize_name(ireq.name) # Pin requirement to a resolved version @@ -768,16 +777,18 @@ def _get_install_requirement_from_candidate( # Save reverse dependencies for annotation ireq_key = key_from_ireq(ireq) - pinned_ireq._required_by = reverse_dependencies.get(ireq_key, set()) + required_by = reverse_dependencies.get(ireq_key, set()) + pinned_ireq._required_by = required_by # type: ignore[attr-defined] # Save sources for annotation constraint_ireq = self._constraints_map.get(ireq_key) if constraint_ireq is not None: if hasattr(constraint_ireq, "_source_ireqs"): # If the constraint is combined (has _source_ireqs), use those - pinned_ireq._source_ireqs = constraint_ireq._source_ireqs + source_ireqs = constraint_ireq._source_ireqs else: # Otherwise (the constraint is not combined) it is the source - pinned_ireq._source_ireqs = [constraint_ireq] + source_ireqs = [constraint_ireq] + pinned_ireq._source_ireqs = source_ireqs # type: ignore[attr-defined] return pinned_ireq diff --git a/piptools/sync.py b/piptools/sync.py index c1d690aee..5ba9f736b 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -131,11 +131,10 @@ def diff_key_from_ireq(ireq: InstallRequirement) -> str: if the contents at the URL have changed but the version has not. """ if is_url_requirement(ireq): - if getattr(ireq.req, "name", None) and ireq.link.has_hash: - return str( - direct_url_as_pep440_direct_reference( - direct_url_from_link(ireq.link), ireq.req.name - ) + req_name = getattr(ireq.req, "name", None) + if req_name is not None and ireq.link is not None and ireq.link.has_hash: + return direct_url_as_pep440_direct_reference( + direct_url_from_link(ireq.link), req_name ) # TODO: Also support VCS and editable installs. return str(ireq.link) @@ -189,7 +188,7 @@ def diff( def sync( to_install: Iterable[InstallRequirement], - to_uninstall: Iterable[InstallRequirement], + to_uninstall: Iterable[str], dry_run: bool = False, install_flags: list[str] | None = None, ask: bool = False, diff --git a/piptools/utils.py b/piptools/utils.py index 68a46f51d..7c2069144 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -22,12 +22,12 @@ from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line from pip._internal.utils.misc import redact_auth_from_url -from pip._internal.vcs import is_url +from pip._internal.vcs import is_url # type: ignore[attr-defined] from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import LegacyVersion, Version from pip._vendor.pkg_resources import get_distribution from piptools._compat import PIP_VERSION @@ -64,11 +64,13 @@ def key_from_ireq(ireq: InstallRequirement) -> str: if ireq.req is None and ireq.link is not None: return str(ireq.link) else: + assert ireq.req is not None return key_from_req(ireq.req) def key_from_req(req: InstallRequirement | Requirement) -> str: """Get an all-lowercase version of the requirement's name.""" + assert req.name is not None return str(canonicalize_name(req.name)) @@ -77,7 +79,7 @@ def comment(text: str) -> str: def make_install_requirement( - name: str, version: str | Version, ireq: InstallRequirement + name: str, version: str | Version | LegacyVersion, ireq: InstallRequirement ) -> InstallRequirement: # If no extras are specified, the extras string is blank extras_string = "" @@ -117,12 +119,14 @@ def format_requirement( in a less verbose way than using its `__str__` method. """ if ireq.editable: + assert ireq.link is not None line = f"-e {ireq.link.url}" elif is_url_requirement(ireq): line = _build_direct_reference_best_efforts(ireq) else: # Canonicalize the requirement name # https://packaging.pypa.io/en/latest/utils.html#packaging.utils.canonicalize_name + assert ireq.req is not None req = copy.copy(ireq.req) req.name = canonicalize_name(req.name) line = str(req) @@ -142,13 +146,15 @@ def _build_direct_reference_best_efforts(ireq: InstallRequirement) -> str: Returns a string of a direct reference URI, whenever possible. See https://www.python.org/dev/peps/pep-0508/ """ + assert ireq.link is not None + # If the requirement has no name then we cannot build a direct reference. if not ireq.name: - return cast(str, ireq.link.url) + return ireq.link.url # Look for a relative file path, the direct reference currently does not work with it. if ireq.link.is_file and not ireq.link.path.startswith("/"): - return cast(str, ireq.link.url) + return ireq.link.url # If we get here then we have a requirement that supports direct reference. # We need to remove the egg if it exists and keep the rest of the fragments. @@ -176,10 +182,8 @@ def format_specifier(ireq: InstallRequirement) -> str: """ # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier if ireq.req is not None else SpecifierSet() - # FIXME: remove ignore type marker once the following issue get fixed - # https://github.com/python/mypy/issues/9656 - specs = sorted(specs, key=lambda x: x.version) - return ",".join(str(s) for s in specs) or "" + sorted_specs = sorted(specs, key=lambda x: x.version) + return ",".join(str(s) for s in sorted_specs) or "" def is_pinned_requirement(ireq: InstallRequirement) -> bool: @@ -422,7 +426,7 @@ def get_required_pip_specification() -> SpecifierSet: assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" - return requirement.specifier + return cast(SpecifierSet, requirement.specifier) def get_pip_version_for_python_executable(python_executable: str) -> Version: @@ -467,7 +471,7 @@ def copy_install_requirement( ) -> InstallRequirement: """Make a copy of a template ``InstallRequirement`` with extra kwargs.""" # Prepare install requirement kwargs. - kwargs = { + kwargs: dict[str, Any] = { "comes_from": template.comes_from, "editable": template.editable, "link": template.link, @@ -483,7 +487,7 @@ def copy_install_requirement( kwargs.update(extra_kwargs) if PIP_VERSION[:2] <= (23, 0): - kwargs["install_options"] = template.install_options + kwargs["install_options"] = template.install_options # type: ignore[attr-defined] # Original link does not belong to install requirements constructor, # pop it now to update later. @@ -517,6 +521,7 @@ def parse_requirements_from_wheel_metadata( for req in metadata.get_all("Requires-Dist") or []: parts = parse_req_from_line(req, comes_from) + assert parts.requirement is not None if parts.requirement.name == package_name: package_dir = os.path.dirname(os.path.abspath(src_file)) # Replace package name with package directory in the requirement diff --git a/pyproject.toml b/pyproject.toml index b10386e31..8ed87ed57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,6 @@ disallow_incomplete_defs = true disallow_subclassing_any = true disallow_untyped_calls = true disallow_untyped_decorators = true -ignore_missing_imports = true no_implicit_optional = true no_implicit_reexport = true strict_equality = true @@ -95,6 +94,10 @@ exclude = "^tests/test_data/" module = ["tests.*"] disallow_untyped_defs = false +[[tool.mypy.overrides]] +module = ["pip._vendor.requests.*", "pip._vendor.pkg_resources.*"] +follow_imports = "skip" + [tool.pytest.ini_options] addopts = [ # `pytest-xdist`: diff --git a/tests/conftest.py b/tests/conftest.py index dedf366e4..64055231c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -115,15 +115,15 @@ def options(self): @property def session(self) -> PipSession: - """Not used""" + raise NotImplementedError("not used") @property def finder(self) -> PackageFinder: - """Not used""" + raise NotImplementedError("not used") @property def command(self) -> InstallCommand: - """Not used""" + raise NotImplementedError("not used") def pytest_collection_modifyitems(config, items): From 13473942ba582339e517f3228703f4003e81b6d9 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Thu, 20 Jul 2023 20:57:14 +0200 Subject: [PATCH 2/3] Revert NotImplementedError raising --- tests/conftest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 64055231c..0ea9b5d05 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,16 +114,16 @@ def options(self): return self._options @property - def session(self) -> PipSession: - raise NotImplementedError("not used") + def session(self) -> PipSession: # type: ignore + """not used""" @property - def finder(self) -> PackageFinder: - raise NotImplementedError("not used") + def finder(self) -> PackageFinder: # type: ignore + """not used""" @property - def command(self) -> InstallCommand: - raise NotImplementedError("not used") + def command(self) -> InstallCommand: # type: ignore + """not used""" def pytest_collection_modifyitems(config, items): From 1ae64ffb2db3b2f80971b865bf6cef0fb14ee773 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Thu, 20 Jul 2023 21:00:40 +0200 Subject: [PATCH 3/3] Formatting --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8ed87ed57..73d138717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,10 @@ module = ["tests.*"] disallow_untyped_defs = false [[tool.mypy.overrides]] -module = ["pip._vendor.requests.*", "pip._vendor.pkg_resources.*"] +module = [ + "pip._vendor.pkg_resources.*", + "pip._vendor.requests.*", +] follow_imports = "skip" [tool.pytest.ini_options]