From 008fe2e76479d8980387f7d83c69f81c4c8a4f6b Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 22 Sep 2022 08:31:24 +0100 Subject: [PATCH] `mypy_test.py`: rework `filter`, `exclude` and `--dir` arguments (#8711) --- tests/check_consistent.py | 11 +-- tests/mypy_test.py | 147 ++++++++++++++++++-------------------- tests/utils.py | 14 ++++ 3 files changed, 85 insertions(+), 87 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 32f1c3c70..77601bc9b 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -15,7 +15,7 @@ import yaml from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.version import Version -from utils import get_all_testcase_directories +from utils import VERSIONS_RE, get_all_testcase_directories, strip_comments metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"} tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}} @@ -85,13 +85,6 @@ def check_no_symlinks() -> None: raise ValueError(no_symlink.format(file)) -_VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): [23]\.\d{1,2}-(?:[23]\.\d{1,2})?$") - - -def strip_comments(text: str) -> str: - return text.split("#")[0].strip() - - def check_versions() -> None: versions = set() with open("stdlib/VERSIONS") as f: @@ -100,7 +93,7 @@ def check_versions() -> None: line = strip_comments(line) if line == "": continue - m = _VERSIONS_RE.match(line) + m = VERSIONS_RE.match(line) if not m: raise AssertionError(f"Bad line in VERSIONS: {line}") module = m.group(1) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index b648b2b48..8c113863f 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -21,53 +21,55 @@ if TYPE_CHECKING: from typing_extensions import Annotated, TypeAlias import tomli -from utils import colored, print_error, print_success_msg, read_dependencies +from utils import VERSIONS_RE as VERSION_LINE_RE, colored, print_error, print_success_msg, read_dependencies, strip_comments -SUPPORTED_VERSIONS = [(3, 11), (3, 10), (3, 9), (3, 8), (3, 7)] +SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"] SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") -TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs"}) +DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")] ReturnCode: TypeAlias = int -MajorVersion: TypeAlias = int -MinorVersion: TypeAlias = int -MinVersion: TypeAlias = tuple[MajorVersion, MinorVersion] -MaxVersion: TypeAlias = tuple[MajorVersion, MinorVersion] +VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"] +VersionTuple: TypeAlias = tuple[int, int] Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"] -Directory: TypeAlias = Annotated[str, "Must be one of the entries in TYPESHED_DIRECTORIES"] - - -def python_version(arg: str) -> tuple[MajorVersion, MinorVersion]: - version = tuple(map(int, arg.split("."))) # This will naturally raise TypeError if it's not in the form "{major}.{minor}" - if version not in SUPPORTED_VERSIONS: - raise ValueError - # mypy infers the return type as tuple[int, ...] - return version # type: ignore[return-value] class CommandLineArgs(argparse.Namespace): verbose: int - exclude: list[str] | None - python_version: list[tuple[MajorVersion, MinorVersion]] | None - dir: list[Directory] | None + filter: list[Path] + exclude: list[Path] | None + python_version: list[VersionString] | None platform: list[Platform] | None - filter: list[str] + + +def valid_path(cmd_arg: str) -> Path: + """Helper function for argument-parsing""" + path = Path(cmd_arg) + if not path.exists(): + raise argparse.ArgumentTypeError(f'"{path}" does not exist in typeshed!') + if not (path in DIRECTORIES_TO_TEST or any(directory in path.parents for directory in DIRECTORIES_TO_TEST)): + raise argparse.ArgumentTypeError('mypy_test.py only tests the stubs found in the "stdlib" and "stubs" directories') + return path parser = argparse.ArgumentParser( description="Typecheck typeshed's stubs with mypy. Patterns are unanchored regexps on the full path." ) -parser.add_argument("-v", "--verbose", action="count", default=0, help="More output") -parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern") parser.add_argument( - "-p", "--python-version", type=python_version, nargs="*", action="extend", help="These versions only (major[.minor])" + "filter", + type=valid_path, + nargs="*", + help='Test these files and directories (defaults to all files in the "stdlib" and "stubs" directories)', ) +parser.add_argument("-x", "--exclude", type=valid_path, nargs="*", help="Exclude these files and directories") +parser.add_argument("-v", "--verbose", action="count", default=0, help="More output") parser.add_argument( - "-d", - "--dir", - choices=TYPESHED_DIRECTORIES, + "-p", + "--python-version", + type=str, + choices=SUPPORTED_VERSIONS, nargs="*", action="extend", - help="Test only these top-level typeshed directories (defaults to all typeshed directories)", + help="These versions only (major[.minor])", ) parser.add_argument( "--platform", @@ -76,7 +78,6 @@ parser.add_argument( action="extend", help="Run mypy for certain OS platforms (defaults to sys.platform only)", ) -parser.add_argument("filter", type=str, nargs="*", help="Include pattern (default all)") @dataclass @@ -84,12 +85,10 @@ class TestConfig: """Configuration settings for a single run of the `test_typeshed` function.""" verbose: int - exclude: list[str] | None - major: MajorVersion - minor: MinorVersion - directories: frozenset[Directory] + filter: list[Path] + exclude: list[Path] + version: VersionString platform: Platform - filter: list[str] def log(args: TestConfig, *varargs: object) -> None: @@ -98,39 +97,35 @@ def log(args: TestConfig, *varargs: object) -> None: def match(path: Path, args: TestConfig) -> bool: - fn = str(path) - if not args.filter and not args.exclude: - log(args, fn, "accept by default") - return True - if args.exclude: - for f in args.exclude: - if re.search(f, fn): - log(args, fn, "excluded by pattern", f) - return False - if args.filter: - for f in args.filter: - if re.search(f, fn): - log(args, fn, "accepted by pattern", f) - return True - if args.filter: - log(args, fn, "rejected (no pattern matches)") - return False - log(args, fn, "accepted (no exclude pattern matches)") - return True + for excluded_path in args.exclude: + if path == excluded_path: + log(args, path, "explicitly excluded") + return False + if excluded_path in path.parents: + log(args, path, f'is in an explicitly excluded directory "{excluded_path}"') + return False + for included_path in args.filter: + if path == included_path: + log(args, path, "was explicitly included") + return True + if included_path in path.parents: + log(args, path, f'is in an explicitly included directory "{included_path}"') + return True + log_msg = ( + f'is implicitly excluded: was not in any of the directories or paths specified on the command line: "{args.filter!r}"' + ) + log(args, path, log_msg) + return False -_VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$") - - -def parse_versions(fname: StrPath) -> dict[str, tuple[MinVersion, MaxVersion]]: +def parse_versions(fname: StrPath) -> dict[str, tuple[VersionTuple, VersionTuple]]: result = {} with open(fname) as f: for line in f: - # Allow having some comments or empty lines. - line = line.split("#")[0].strip() + line = strip_comments(line) if line == "": continue - m = _VERSION_LINE_RE.match(line) + m = VERSION_LINE_RE.match(line) assert m, f"invalid VERSIONS line: {line}" mod: str = m.group(1) min_version = parse_version(m.group(2)) @@ -151,10 +146,11 @@ def parse_version(v_str: str) -> tuple[int, int]: def add_files(files: list[Path], seen: set[str], module: Path, args: TestConfig) -> None: """Add all files in package or module represented by 'name' located in 'root'.""" if module.is_file() and module.suffix == ".pyi": - files.append(module) - seen.add(module.stem) + if match(module, args): + files.append(module) + seen.add(module.stem) else: - to_add = sorted(module.rglob("*.pyi")) + to_add = sorted(file for file in module.rglob("*.pyi") if match(file, args)) files.extend(to_add) seen.update(path.stem for path in to_add) @@ -239,7 +235,7 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[P def get_mypy_flags(args: TestConfig, temp_name: str) -> list[str]: return [ "--python-version", - f"{args.major}.{args.minor}", + args.version, "--show-traceback", "--warn-incomplete-stub", "--show-error-codes", @@ -323,7 +319,8 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: if name == "VERSIONS" or name.startswith("."): continue module = Path(name).stem - if supported_versions[module][0] <= (args.major, args.minor) <= supported_versions[module][1]: + module_min_version, module_max_version = supported_versions[module] + if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version: add_files(files, seen, (stdlib / name), args) if files: @@ -354,14 +351,15 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults: def test_typeshed(code: int, args: TestConfig) -> TestResults: - print(f"*** Testing Python {args.major}.{args.minor} on {args.platform}") + print(f"*** Testing Python {args.version} on {args.platform}") files_checked_this_version = 0 - if "stdlib" in args.directories: + stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") + if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): code, stdlib_files_checked = test_stdlib(code, args) files_checked_this_version += stdlib_files_checked print() - if "stubs" in args.directories: + if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): code, third_party_files_checked = test_third_party_stubs(code, args) files_checked_this_version += third_party_files_checked print() @@ -373,19 +371,12 @@ def main() -> None: args = parser.parse_args(namespace=CommandLineArgs()) versions = args.python_version or SUPPORTED_VERSIONS platforms = args.platform or [sys.platform] - tested_directories = frozenset(args.dir) if args.dir else TYPESHED_DIRECTORIES + filter = args.filter or DIRECTORIES_TO_TEST + exclude = args.exclude or [] code = 0 total_files_checked = 0 - for (major, minor), platform in product(versions, platforms): - config = TestConfig( - verbose=args.verbose, - exclude=args.exclude, - major=major, - minor=minor, - directories=tested_directories, - platform=platform, - filter=args.filter, - ) + for version, platform in product(versions, platforms): + config = TestConfig(args.verbose, filter, exclude, version, platform) code, files_checked_this_version = test_typeshed(code, args=config) total_files_checked += files_checked_this_version if code: diff --git a/tests/utils.py b/tests/utils.py index e560acb34..4237ef6e1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,12 +1,18 @@ """Utilities that are imported by multiple scripts in the tests directory.""" import os +import re from functools import cache from pathlib import Path from typing import NamedTuple import tomli + +def strip_comments(text: str) -> str: + return text.split("#")[0].strip() + + try: from termcolor import colored as colored except ImportError: @@ -46,6 +52,14 @@ def read_dependencies(distribution: str) -> tuple[str, ...]: return tuple(dependencies) +# ==================================================================== +# Parsing the stdlib/VERSIONS file +# ==================================================================== + + +VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$") + + # ==================================================================== # Getting test-case directories from package names # ====================================================================