From 2d0f6d8277527bc48871cc04e1cf875b13a6f992 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Thu, 17 Oct 2024 08:16:10 +0200 Subject: [PATCH] Bundle path handling in ts_utils.paths (#12805) --- lib/ts_utils/metadata.py | 13 ++++------- lib/ts_utils/paths.py | 39 +++++++++++++++++++++++++++++++ lib/ts_utils/utils.py | 35 ++++----------------------- scripts/stubsabot.py | 7 +++--- tests/check_typeshed_structure.py | 18 ++++++-------- tests/mypy_test.py | 25 ++++++++------------ tests/regr_test.py | 8 +++---- tests/runtests.py | 3 ++- tests/stubtest_stdlib.py | 5 ++-- tests/stubtest_third_party.py | 8 +++---- 10 files changed, 81 insertions(+), 80 deletions(-) create mode 100644 lib/ts_utils/paths.py diff --git a/lib/ts_utils/metadata.py b/lib/ts_utils/metadata.py index 113cbc1d0..7725aa1a5 100644 --- a/lib/ts_utils/metadata.py +++ b/lib/ts_utils/metadata.py @@ -5,7 +5,6 @@ from __future__ import annotations -import os import re import urllib.parse from collections.abc import Mapping @@ -18,6 +17,7 @@ import tomli from packaging.requirements import Requirement from packaging.specifiers import Specifier +from .paths import PYPROJECT_PATH, STUBS_PATH, distribution_path from .utils import cache __all__ = [ @@ -43,20 +43,15 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]: @cache def _get_oldest_supported_python() -> str: - with open("pyproject.toml", "rb") as config: + with PYPROJECT_PATH.open("rb") as config: val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"] assert type(val) is str return val -def stubs_path(distribution: str) -> Path: - """Return the path to the directory of a third-party distribution.""" - return Path("stubs", distribution) - - def metadata_path(distribution: str) -> Path: """Return the path to the METADATA.toml file of a third-party distribution.""" - return stubs_path(distribution) / "METADATA.toml" + return distribution_path(distribution) / "METADATA.toml" @final @@ -317,7 +312,7 @@ class PackageDependencies(NamedTuple): @cache def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]: - return {read_metadata(typeshed_name).stub_distribution: typeshed_name for typeshed_name in os.listdir("stubs")} + return {read_metadata(dir.name).stub_distribution: dir.name for dir in STUBS_PATH.iterdir()} @cache diff --git a/lib/ts_utils/paths.py b/lib/ts_utils/paths.py new file mode 100644 index 000000000..631192317 --- /dev/null +++ b/lib/ts_utils/paths.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Final + +# TODO: Use base path relative to this file. Currently, ts_utils gets +# installed into the user's virtual env, so we can't determine the path +# to typeshed. Installing ts_utils editable would solve that, see +# https://github.com/python/typeshed/pull/12806. +TS_BASE_PATH: Final = Path("") +STDLIB_PATH: Final = TS_BASE_PATH / "stdlib" +STUBS_PATH: Final = TS_BASE_PATH / "stubs" + +PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml" +REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt" + +TESTS_DIR: Final = "@tests" +TEST_CASES_DIR: Final = "test_cases" + + +def distribution_path(distribution_name: str) -> Path: + """Return the path to the directory of a third-party distribution.""" + return STUBS_PATH / distribution_name + + +def tests_path(distribution_name: str) -> Path: + if distribution_name == "stdlib": + return STDLIB_PATH / TESTS_DIR + else: + return STUBS_PATH / distribution_name / TESTS_DIR + + +def test_cases_path(distribution_name: str) -> Path: + return tests_path(distribution_name) / TEST_CASES_DIR + + +def allowlists_path(distribution_name: str) -> Path: + if distribution_name == "stdlib": + return tests_path("stdlib") / "stubtest_allowlists" + else: + return tests_path(distribution_name) diff --git a/lib/ts_utils/utils.py b/lib/ts_utils/utils.py index 7381707ed..4dfe05488 100644 --- a/lib/ts_utils/utils.py +++ b/lib/ts_utils/utils.py @@ -21,10 +21,9 @@ except ImportError: return text -PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" +from .paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path -STDLIB_PATH = Path("stdlib") -STUBS_PATH = Path("stubs") +PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" # A backport of functools.cache for Python <3.9 @@ -90,16 +89,14 @@ def venv_python(venv_dir: Path) -> Path: # ==================================================================== -REQS_FILE: Final = "requirements-tests.txt" - - @cache def parse_requirements() -> Mapping[str, Requirement]: """Return a dictionary of requirements from the requirements file.""" - with open(REQS_FILE, encoding="UTF-8") as requirements_file: + with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file: stripped_lines = map(strip_comments, requirements_file) - requirements = map(Requirement, filter(None, stripped_lines)) + stripped_more = [li for li in stripped_lines if not li.startswith("-")] + requirements = map(Requirement, filter(None, stripped_more)) return {requirement.name: requirement for requirement in requirements} @@ -155,10 +152,6 @@ def _parse_version(v_str: str) -> tuple[int, int]: # ==================================================================== -TESTS_DIR: Final = "@tests" -TEST_CASES_DIR: Final = "test_cases" - - class DistributionTests(NamedTuple): name: str test_cases_path: Path @@ -179,17 +172,6 @@ def distribution_info(distribution_name: str) -> DistributionTests: raise RuntimeError(f"No test cases found for {distribution_name!r}!") -def tests_path(distribution_name: str) -> Path: - if distribution_name == "stdlib": - return STDLIB_PATH / TESTS_DIR - else: - return STUBS_PATH / distribution_name / TESTS_DIR - - -def test_cases_path(distribution_name: str) -> Path: - return tests_path(distribution_name) / TEST_CASES_DIR - - def get_all_testcase_directories() -> list[DistributionTests]: testcase_directories: list[DistributionTests] = [] for distribution_path in STUBS_PATH.iterdir(): @@ -201,13 +183,6 @@ def get_all_testcase_directories() -> list[DistributionTests]: return [distribution_info("stdlib"), *sorted(testcase_directories)] -def allowlists_path(distribution_name: str) -> Path: - if distribution_name == "stdlib": - return tests_path("stdlib") / "stubtest_allowlists" - else: - return tests_path(distribution_name) - - def allowlists(distribution_name: str) -> list[str]: prefix = "" if distribution_name == "stdlib" else "stubtest_allowlist_" version_id = f"py{sys.version_info.major}{sys.version_info.minor}" diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 2e1f76fd0..18985f5bc 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -29,7 +29,8 @@ import tomlkit from packaging.specifiers import Specifier from termcolor import colored -from ts_utils.metadata import StubMetadata, metadata_path, read_metadata, stubs_path +from ts_utils.metadata import StubMetadata, metadata_path, read_metadata +from ts_utils.paths import STUBS_PATH, distribution_path TYPESHED_OWNER = "python" TYPESHED_API_URL = f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed" @@ -463,7 +464,7 @@ async def analyze_diff( assert isinstance(json_resp, dict) # https://docs.github.com/en/rest/commits/commits#compare-two-commits py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"] - stub_path = stubs_path(distribution) + stub_path = distribution_path(distribution) files_in_typeshed = set(stub_path.rglob("*.pyi")) py_files_stubbed_in_typeshed = [file for file in py_files if (stub_path / f"{file['filename']}i") in files_in_typeshed] return DiffAnalysis(py_files=py_files, py_files_stubbed_in_typeshed=py_files_stubbed_in_typeshed) @@ -759,7 +760,7 @@ async def main() -> None: if args.distributions: dists_to_update = args.distributions else: - dists_to_update = [path.name for path in Path("stubs").iterdir()] + dists_to_update = [path.name for path in STUBS_PATH.iterdir()] if args.action_level > ActionLevel.nothing: subprocess.run(["git", "update-index", "--refresh"], capture_output=True) diff --git a/tests/check_typeshed_structure.py b/tests/check_typeshed_structure.py index 56e35f38f..d83fa1706 100755 --- a/tests/check_typeshed_structure.py +++ b/tests/check_typeshed_structure.py @@ -13,17 +13,13 @@ import sys from pathlib import Path from ts_utils.metadata import read_metadata +from ts_utils.paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path from ts_utils.utils import ( - REQS_FILE, - STDLIB_PATH, - TEST_CASES_DIR, - TESTS_DIR, get_all_testcase_directories, get_gitignore_spec, parse_requirements, parse_stdlib_versions_file, spec_matches_path, - tests_path, ) extension_descriptions = {".pyi": "stub", ".py": ".py"} @@ -66,7 +62,7 @@ def check_stdlib() -> None: def check_stubs() -> None: """Check that the stubs directory contains only the correct files.""" gitignore_spec = get_gitignore_spec() - for dist in Path("stubs").iterdir(): + for dist in STUBS_PATH.iterdir(): if spec_matches_path(gitignore_spec, dist): continue assert dist.is_dir(), f"Only directories allowed in stubs, got {dist}" @@ -97,8 +93,8 @@ def check_distutils() -> None: def all_relative_paths_in_directory(path: Path) -> set[Path]: return {pyi.relative_to(path) for pyi in path.rglob("*.pyi")} - all_setuptools_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "setuptools", "_distutils")) - all_distutils_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "distutils")) + all_setuptools_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "setuptools" / "_distutils") + all_distutils_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "distutils") assert all_setuptools_files and all_distutils_files, "Looks like this test might be out of date!" extra_files = all_setuptools_files - all_distutils_files joined = "\n".join(f" * {f}" for f in extra_files) @@ -164,10 +160,10 @@ def check_requirement_pins() -> None: """Check that type checkers and linters are pinned to an exact version.""" requirements = parse_requirements() for package in linters: - assert package in requirements, f"type checker/linter '{package}' not found in {REQS_FILE}" + assert package in requirements, f"type checker/linter '{package}' not found in {REQUIREMENTS_PATH.name}" spec = requirements[package].specifier - assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQS_FILE}" - msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQS_FILE}" + assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQUIREMENTS_PATH.name}" + msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQUIREMENTS_PATH.name}" assert str(spec).startswith("=="), msg diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 5d841c558..b1ad54735 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -22,11 +22,10 @@ from typing_extensions import Annotated, TypeAlias import tomli from packaging.requirements import Requirement -from ts_utils.metadata import PackageDependencies, get_recursive_requirements, read_metadata +from ts_utils.metadata import PackageDependencies, get_recursive_requirements, metadata_path, read_metadata +from ts_utils.paths import STDLIB_PATH, STUBS_PATH, TESTS_DIR, TS_BASE_PATH, distribution_path from ts_utils.utils import ( PYTHON_VERSION, - STDLIB_PATH, - TESTS_DIR, colored, get_gitignore_spec, get_mypy_req, @@ -47,7 +46,7 @@ except ImportError: SUPPORTED_VERSIONS = ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") -DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")] +DIRECTORIES_TO_TEST = [STDLIB_PATH, STUBS_PATH] VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"] Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"] @@ -170,7 +169,7 @@ class MypyDistConf(NamedTuple): def add_configuration(configurations: list[MypyDistConf], distribution: str) -> None: - with Path("stubs", distribution, "METADATA.toml").open("rb") as f: + with metadata_path(distribution).open("rb") as f: data = tomli.load(f) # TODO: This could be added to ts_utils.metadata, but is currently unused @@ -229,7 +228,7 @@ def run_mypy( "--platform", args.platform, "--custom-typeshed-dir", - str(Path(__file__).parent.parent), + str(TS_BASE_PATH), "--strict", # Stub completion is checked by pyright (--allow-*-defs) "--allow-untyped-defs", @@ -283,7 +282,7 @@ def add_third_party_files( return seen_dists.add(distribution) seen_dists.update(r.name for r in typeshed_reqs) - root = Path("stubs", distribution) + root = distribution_path(distribution) for name in os.listdir(root): if name.startswith("."): continue @@ -319,7 +318,7 @@ def test_third_party_distribution( print_error("no files found") sys.exit(1) - mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) + mypypath = os.pathsep.join(str(distribution_path(dist)) for dist in seen_dists) if args.verbose: print(colored(f"\nMYPYPATH={mypypath}", "blue")) result = run_mypy( @@ -516,16 +515,12 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: distributions_to_check: dict[str, PackageDependencies] = {} for distribution in sorted(os.listdir("stubs")): - distribution_path = Path("stubs", distribution) + dist_path = distribution_path(distribution) - if spec_matches_path(gitignore_spec, distribution_path): + if spec_matches_path(gitignore_spec, dist_path): continue - if ( - distribution_path in args.filter - or Path("stubs") in args.filter - or any(distribution_path in path.parents for path in args.filter) - ): + if dist_path in args.filter or STUBS_PATH in args.filter or any(dist_path in path.parents for path in args.filter): metadata = read_metadata(distribution) if not metadata.requires_python.contains(PYTHON_VERSION): msg = ( diff --git a/tests/regr_test.py b/tests/regr_test.py index 87e29c65a..0889b3fb6 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -22,9 +22,9 @@ from pathlib import Path from typing_extensions import TypeAlias from ts_utils.metadata import get_recursive_requirements, read_metadata +from ts_utils.paths import STDLIB_PATH, TEST_CASES_DIR, TS_BASE_PATH, distribution_path from ts_utils.utils import ( PYTHON_VERSION, - TEST_CASES_DIR, DistributionTests, colored, distribution_info, @@ -134,14 +134,14 @@ def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Ver # that has only the required stubs copied over. new_typeshed = tempdir / TYPESHED new_typeshed.mkdir() - shutil.copytree(Path("stdlib"), new_typeshed / "stdlib") + shutil.copytree(STDLIB_PATH, new_typeshed / "stdlib") requirements = get_recursive_requirements(package.name) # mypy refuses to consider a directory a "valid typeshed directory" # unless there's a stubs/mypy-extensions path inside it, # so add that to the list of stubs to copy over to the new directory typeshed_requirements = [r.name for r in requirements.typeshed_pkgs] for requirement in {package.name, *typeshed_requirements, "mypy-extensions"}: - shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement) + shutil.copytree(distribution_path(requirement), new_typeshed / "stubs" / requirement) if requirements.external_pkgs: venv_location = str(tempdir / VENV_DIR) @@ -190,7 +190,7 @@ def run_testcases( if package.is_stdlib: python_exe = sys.executable - custom_typeshed = Path(__file__).parent.parent + custom_typeshed = TS_BASE_PATH flags.append("--no-site-packages") else: custom_typeshed = tempdir / TYPESHED diff --git a/tests/runtests.py b/tests/runtests.py index 0caff10e5..1c5ba0c18 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -10,7 +10,8 @@ import sys from importlib.util import find_spec from pathlib import Path -from ts_utils.utils import TEST_CASES_DIR, colored, test_cases_path +from ts_utils.paths import TEST_CASES_DIR, test_cases_path +from ts_utils.utils import colored _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json" _TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json" diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py index 9d356aa54..5134f8270 100755 --- a/tests/stubtest_stdlib.py +++ b/tests/stubtest_stdlib.py @@ -13,7 +13,8 @@ import subprocess import sys from pathlib import Path -from ts_utils.utils import allowlist_stubtest_arguments, allowlists_path +from ts_utils.paths import TS_BASE_PATH, allowlists_path +from ts_utils.utils import allowlist_stubtest_arguments def run_stubtest(typeshed_dir: Path) -> int: @@ -57,4 +58,4 @@ def run_stubtest(typeshed_dir: Path) -> int: if __name__ == "__main__": - sys.exit(run_stubtest(typeshed_dir=Path("."))) + sys.exit(run_stubtest(typeshed_dir=TS_BASE_PATH)) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 0619f015c..e951901f1 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -14,17 +14,16 @@ from textwrap import dedent from typing import NoReturn from ts_utils.metadata import NoSuchStubError, get_recursive_requirements, read_metadata +from ts_utils.paths import STUBS_PATH, allowlists_path, tests_path from ts_utils.utils import ( PYTHON_VERSION, allowlist_stubtest_arguments, - allowlists_path, colored, get_mypy_req, print_divider, print_error, print_info, print_success_msg, - tests_path, ) @@ -386,11 +385,10 @@ def main() -> NoReturn: parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE) args = parser.parse_args() - typeshed_dir = Path(".").resolve() if len(args.dists) == 0: - dists = sorted((typeshed_dir / "stubs").iterdir()) + dists = sorted(STUBS_PATH.iterdir()) else: - dists = [typeshed_dir / "stubs" / d for d in args.dists] + dists = [STUBS_PATH / d for d in args.dists] result = 0 for i, dist in enumerate(dists):