From 417bdb9ac960d46de2ed3355839b2ada568d141e Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 12 Mar 2024 16:34:47 +0100 Subject: [PATCH] Refactor and merge requirements parsing (#11581) --- tests/check_consistent.py | 39 ++++++++++++++++++++------------------- tests/utils.py | 27 +++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index bf02d848c..e5673be2c 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -17,7 +17,15 @@ from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from parse_metadata import read_metadata -from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments +from utils import ( + REQS_FILE, + VERSIONS_RE, + get_all_testcase_directories, + get_gitignore_spec, + parse_requirements, + spec_matches_path, + strip_comments, +) extension_descriptions = {".pyi": "stub", ".py": ".py"} @@ -135,13 +143,6 @@ def check_metadata() -> None: read_metadata(distribution) -def get_txt_requirements() -> dict[str, SpecifierSet]: - with open("requirements-tests.txt", encoding="UTF-8") as requirements_file: - stripped_lines = map(strip_comments, requirements_file) - requirements = map(Requirement, filter(None, stripped_lines)) - return {requirement.name: requirement.specifier for requirement in requirements} - - class PreCommitConfigRepos(TypedDict): hooks: list[dict[str, str]] repo: str @@ -172,30 +173,30 @@ def get_precommit_requirements() -> dict[str, SpecifierSet]: def check_requirement_pins() -> None: """Check that type checkers and linters are pinned to an exact version.""" - requirements = get_txt_requirements() + requirements = parse_requirements() for package in linters: - assert package in requirements, f"type checker/linter '{package}' not found in requirements-tests.txt" - spec = requirements[package] - assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in requirements-tests.txt" - msg = f"type checker/linter '{package}' is not pinned to an exact version in requirements-tests.txt" + assert package in requirements, f"type checker/linter '{package}' not found in {REQS_FILE}" + spec = requirements[package].specifier + assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQS_FILE}" + msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQS_FILE}" assert str(spec).startswith("=="), msg def check_precommit_requirements() -> None: - """Check that the requirements in requirements-tests.txt and .pre-commit-config.yaml match.""" - requirements_txt_requirements = get_txt_requirements() + """Check that the requirements in the requirements file and .pre-commit-config.yaml match.""" + requirements_txt_requirements = parse_requirements() precommit_requirements = get_precommit_requirements() - no_txt_entry_msg = "All pre-commit requirements must also be listed in `requirements-tests.txt` (missing {requirement!r})" + no_txt_entry_msg = f"All pre-commit requirements must also be listed in `{REQS_FILE}` (missing {{requirement!r}})" for requirement, specifier in precommit_requirements.items(): - # annoying: the Ruff and Black repos for pre-commit are different to the names in requirements-tests.txt + # annoying: the Ruff and Black repos for pre-commit are different to the names in the requirements file if requirement in {"ruff-pre-commit", "black-pre-commit-mirror"}: requirement = requirement.split("-")[0] assert requirement in requirements_txt_requirements, no_txt_entry_msg.format(requirement=requirement) specifier_mismatch = ( f'Specifier "{specifier}" for {requirement!r} in `.pre-commit-config.yaml` ' - f'does not match specifier "{requirements_txt_requirements[requirement]}" in `requirements-tests.txt`' + f'does not match specifier "{requirements_txt_requirements[requirement].specifier}" in `{REQS_FILE}`' ) - assert specifier == requirements_txt_requirements[requirement], specifier_mismatch + assert specifier == requirements_txt_requirements[requirement].specifier, specifier_mismatch if __name__ == "__main__": diff --git a/tests/utils.py b/tests/utils.py index 18d97cc6a..1501f9c4a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,11 +5,13 @@ from __future__ import annotations import os import re import sys +from collections.abc import Mapping from functools import lru_cache from pathlib import Path from typing import Any, Final, NamedTuple import pathspec +from packaging.requirements import Requirement try: from termcolor import colored as colored # pyright: ignore[reportAssignmentType] @@ -31,6 +33,11 @@ def strip_comments(text: str) -> str: return text.split("#")[0].strip() +# ==================================================================== +# Printing utilities +# ==================================================================== + + def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None: error_split = error.split("\n") old, new = fix_path @@ -55,10 +62,26 @@ def venv_python(venv_dir: Path) -> Path: return venv_dir / "bin" / "python" +# ==================================================================== +# Parsing the requirements file +# ==================================================================== + + +REQS_FILE: Final = "requirements-tests.txt" + + @cache +def parse_requirements() -> Mapping[str, Requirement]: + """Return a dictionary of requirements from the requirements file.""" + + with open(REQS_FILE, encoding="UTF-8") as requirements_file: + stripped_lines = map(strip_comments, requirements_file) + requirements = map(Requirement, filter(None, stripped_lines)) + return {requirement.name: requirement for requirement in requirements} + + def get_mypy_req() -> str: - with open("requirements-tests.txt", encoding="UTF-8") as requirements_file: - return next(strip_comments(line) for line in requirements_file if "mypy" in line) + return str(parse_requirements()["mypy"]) # ====================================================================