From c5dde1e7208c85cddd6cc90e8c4af733d34d1cba Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 24 Sep 2023 19:39:12 +0200 Subject: [PATCH] Add optional `requires_python` field to third-party stubs metadata (#10724) Co-authored-by: Alex Waygood --- pyproject.toml | 1 + tests/mypy_test.py | 74 +++++++++++++++++++++++++++-------- tests/parse_metadata.py | 26 ++++++++++++ tests/regr_test.py | 44 ++++++++++++++++----- tests/stubtest_third_party.py | 6 ++- tests/utils.py | 5 ++- 6 files changed, 127 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 841e30634..894c9b234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,4 @@ select = [ [tool.typeshed] pyright_version = "1.1.328" +oldest_supported_python = "3.7" diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 3d10ad689..0ff0cd66c 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -25,8 +25,9 @@ from typing_extensions import Annotated, TypeAlias import tomli -from parse_metadata import PackageDependencies, get_recursive_requirements +from parse_metadata import PackageDependencies, get_recursive_requirements, read_metadata from utils import ( + PYTHON_VERSION, VERSIONS_RE as VERSION_LINE_RE, VenvInfo, colored, @@ -307,6 +308,7 @@ def add_third_party_files( class TestResults(NamedTuple): exit_code: int files_checked: int + packages_skipped: int = 0 def test_third_party_distribution( @@ -393,6 +395,9 @@ def install_requirements_for_venv(venv_info: VenvInfo, args: TestConfig, externa def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None: """Logic necessary for testing stubs with non-types dependencies in isolated environments.""" + if not distributions: + return # hooray! Nothing to do + # STAGE 1: Determine which (if any) stubs packages require virtual environments. # Group stubs packages according to their external-requirements sets @@ -471,6 +476,7 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: print("Testing third-party packages...") files_checked = 0 + packages_skipped = 0 gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -480,6 +486,21 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe if spec_matches_path(gitignore_spec, distribution_path): continue + metadata = read_metadata(distribution) + if not metadata.requires_python.contains(PYTHON_VERSION): + msg = ( + f"skipping {distribution!r} (requires Python {metadata.requires_python}; " + f"test is being run using Python {PYTHON_VERSION})" + ) + print(colored(msg, "yellow")) + packages_skipped += 1 + continue + if not metadata.requires_python.contains(args.version): + msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" + print(colored(msg, "yellow")) + packages_skipped += 1 + continue + if ( distribution_path in args.filter or Path("stubs") in args.filter @@ -487,40 +508,51 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe ): distributions_to_check[distribution] = get_recursive_requirements(distribution) - # If it's the first time test_third_party_stubs() has been called during this session, - # setup the necessary virtual environments for testing the third-party stubs. - # It should only be necessary to call setup_virtual_environments() once per session. - if not _DISTRIBUTION_TO_VENV_MAPPING: - setup_virtual_environments(distributions_to_check, args, tempdir) + # Setup the necessary virtual environments for testing the third-party stubs. + # Note that some stubs may not be tested on all Python versions + # (due to version incompatibilities), + # so we can't guarantee that setup_virtual_environments() + # will only be called once per session. + distributions_without_venv = { + distribution: requirements + for distribution, requirements in distributions_to_check.items() + if distribution not in _DISTRIBUTION_TO_VENV_MAPPING + } + setup_virtual_environments(distributions_without_venv, args, tempdir) - assert _DISTRIBUTION_TO_VENV_MAPPING.keys() == distributions_to_check.keys() + # Check that there is a venv for every distribution we're testing. + # Some venvs may exist from previous runs but are skipped in this run. + assert _DISTRIBUTION_TO_VENV_MAPPING.keys() >= distributions_to_check.keys() - for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): + for distribution in distributions_to_check: + venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable - this_code, checked = test_third_party_distribution( + this_code, checked, _ = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) code = max(code, this_code) files_checked += checked - return TestResults(code, files_checked) + return TestResults(code, files_checked, packages_skipped) def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: print(f"*** Testing Python {args.version} on {args.platform}") files_checked_this_version = 0 + packages_skipped_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, stdlib_files_checked = test_stdlib(code, args) + code, stdlib_files_checked, _ = test_stdlib(code, args) files_checked_this_version += stdlib_files_checked print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - code, third_party_files_checked = test_third_party_stubs(code, args, tempdir) + code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir) files_checked_this_version += third_party_files_checked + packages_skipped_this_version = third_party_packages_skipped print() - return TestResults(code, files_checked_this_version) + return TestResults(code, files_checked_this_version, packages_skipped_this_version) def main() -> None: @@ -531,19 +563,27 @@ def main() -> None: exclude = args.exclude or [] code = 0 total_files_checked = 0 + total_packages_skipped = 0 with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) - code, files_checked_this_version = test_typeshed(code, args=config, tempdir=td_path) + code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path) total_files_checked += files_checked_this_version + total_packages_skipped += packages_skipped_this_version if code: - print_error(f"--- exit status {code}, {total_files_checked} files checked ---") + plural = "" if total_files_checked == 1 else "s" + print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---") sys.exit(code) - if not total_files_checked: + if total_packages_skipped: + plural = "" if total_packages_skipped == 1 else "s" + print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) + if total_files_checked: + plural = "" if total_files_checked == 1 else "s" + print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green")) + else: print_error("--- nothing to do; exit 1 ---") sys.exit(1) - print(colored(f"--- success, {total_files_checked} files checked ---", "green")) if __name__ == "__main__": diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index f6148e6e3..dfb266ad0 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -15,6 +15,7 @@ from typing_extensions import Annotated, Final, TypeGuard, final import tomli from packaging.requirements import Requirement +from packaging.specifiers import Specifier from packaging.version import Version from utils import cache @@ -40,6 +41,14 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]: return isinstance(obj, list) and all(isinstance(item, str) for item in obj) +@cache +def _get_oldest_supported_python() -> str: + with open("pyproject.toml", "rb") as config: + val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"] + assert type(val) is str + return val + + @final @dataclass(frozen=True) class StubtestSettings: @@ -130,6 +139,7 @@ class StubMetadata: uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"] partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."] stubtest_settings: StubtestSettings + requires_python: Annotated[Specifier, "Versions of Python supported by the stub package"] _KNOWN_METADATA_FIELDS: Final = frozenset( @@ -144,6 +154,7 @@ _KNOWN_METADATA_FIELDS: Final = frozenset( "upload", "tool", "partial_stub", + "requires_python", } ) _KNOWN_METADATA_TOOL_FIELDS: Final = { @@ -240,6 +251,20 @@ def read_metadata(distribution: str) -> StubMetadata: assert type(uploaded_to_pypi) is bool partial_stub: object = data.get("partial_stub", True) assert type(partial_stub) is bool + requires_python_str: object = data.get("requires_python") + oldest_supported_python = _get_oldest_supported_python() + oldest_supported_python_specifier = Specifier(f">={oldest_supported_python}") + if requires_python_str is None: + requires_python = oldest_supported_python_specifier + else: + assert type(requires_python_str) is str + requires_python = Specifier(requires_python_str) + assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant' + # Check minimum Python version is not less than the oldest version of Python supported by typeshed + assert oldest_supported_python_specifier.contains( + requires_python.version + ), f"'requires_python' contains versions lower than typeshed's oldest supported Python ({oldest_supported_python})" + assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'" empty_tools: dict[object, object] = {} tools_settings: object = data.get("tool", empty_tools) @@ -262,6 +287,7 @@ def read_metadata(distribution: str) -> StubMetadata: uploaded_to_pypi=uploaded_to_pypi, partial_stub=partial_stub, stubtest_settings=read_stubtest_settings(distribution), + requires_python=requires_python, ) diff --git a/tests/regr_test.py b/tests/regr_test.py index f7a2c2140..aaface2fc 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -13,15 +13,17 @@ import subprocess import sys import tempfile import threading +from collections.abc import Callable from contextlib import ExitStack, suppress from dataclasses import dataclass from enum import IntEnum -from itertools import product +from functools import partial from pathlib import Path from typing_extensions import TypeAlias -from parse_metadata import get_recursive_requirements +from parse_metadata import get_recursive_requirements, read_metadata from utils import ( + PYTHON_VERSION, PackageInfo, VenvInfo, colored, @@ -274,6 +276,30 @@ def concurrently_run_testcases( packageinfo_to_tempdir = { package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories } + to_do: list[Callable[[], Result]] = [] + for testcase_dir, tempdir in packageinfo_to_tempdir.items(): + pkg = testcase_dir.name + requires_python = None + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master + requires_python = read_metadata(pkg).requires_python + if not requires_python.contains(PYTHON_VERSION): + msg = f"skipping {pkg!r} (requires Python {requires_python}; test is being run using Python {PYTHON_VERSION})" + print(colored(msg, "yellow")) + continue + for version in versions_to_test: + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master + assert requires_python is not None + if not requires_python.contains(version): + msg = f"skipping {pkg!r} for target Python {version} (requires Python {requires_python})" + print(colored(msg, "yellow")) + continue + to_do.extend( + partial(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) + for platform in platforms_to_test + ) + + if not to_do: + return [] event = threading.Event() printer_thread = threading.Thread(target=print_queued_messages, args=(event,)) @@ -289,12 +315,7 @@ def concurrently_run_testcases( ] concurrent.futures.wait(testcase_futures) - mypy_futures = [ - executor.submit(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) - for (testcase_dir, tempdir), platform, version in product( - packageinfo_to_tempdir.items(), platforms_to_test, versions_to_test - ) - ] + mypy_futures = [executor.submit(task) for task in to_do] results = [future.result() for future in mypy_futures] event.set() @@ -315,15 +336,18 @@ def main() -> ReturnCode: platforms_to_test, versions_to_test = SUPPORTED_PLATFORMS, SUPPORTED_VERSIONS else: platforms_to_test = args.platforms_to_test or [sys.platform] - versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"] + versions_to_test = args.versions_to_test or [PYTHON_VERSION] - code = 0 results: list[Result] | None = None with ExitStack() as stack: results = concurrently_run_testcases(stack, testcase_directories, verbosity, platforms_to_test, versions_to_test) assert results is not None + if not results: + print_error("All tests were skipped!") + return 1 + print() for result in results: diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index af393c35f..9b4e301d0 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -13,7 +13,7 @@ from textwrap import dedent from typing import NoReturn from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata -from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg +from utils import PYTHON_VERSION, colored, get_mypy_req, make_venv, print_error, print_success_msg def run_stubtest( @@ -37,6 +37,10 @@ def run_stubtest( return True print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) + if not metadata.requires_python.contains(PYTHON_VERSION): + print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow")) + return True + with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) try: diff --git a/tests/utils.py b/tests/utils.py index dd5243a59..2fd376314 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,7 @@ import sys import venv from functools import lru_cache from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, Final, NamedTuple from typing_extensions import Annotated import pathspec @@ -22,6 +22,9 @@ except ImportError: return text +PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" + + # A backport of functools.cache for Python <3.9 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI cache = lru_cache(None)