mypy_test.py: rework filter, exclude and --dir arguments (#8711)

tests/check_consistent.py

@@ -15,7 +15,7 @@ import yaml
 from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
-from utils import get_all_testcase_directories
+from utils import VERSIONS_RE, get_all_testcase_directories, strip_comments
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
@@ -85,13 +85,6 @@ def check_no_symlinks() -> None:
             raise ValueError(no_symlink.format(file))
 
 
-_VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): [23]\.\d{1,2}-(?:[23]\.\d{1,2})?$")
-
-
-def strip_comments(text: str) -> str:
-    return text.split("#")[0].strip()
-
-
 def check_versions() -> None:
     versions = set()
     with open("stdlib/VERSIONS") as f:
@@ -100,7 +93,7 @@ def check_versions() -> None:
         line = strip_comments(line)
         if line == "":
             continue
-        m = _VERSIONS_RE.match(line)
+        m = VERSIONS_RE.match(line)
         if not m:
             raise AssertionError(f"Bad line in VERSIONS: {line}")
         module = m.group(1)

tests/mypy_test.py

@@ -21,53 +21,55 @@ if TYPE_CHECKING:
 from typing_extensions import Annotated, TypeAlias
 
 import tomli
-from utils import colored, print_error, print_success_msg, read_dependencies
+from utils import VERSIONS_RE as VERSION_LINE_RE, colored, print_error, print_success_msg, read_dependencies, strip_comments
 
-SUPPORTED_VERSIONS = [(3, 11), (3, 10), (3, 9), (3, 8), (3, 7)]
+SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
-TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs"})
+DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
 
 ReturnCode: TypeAlias = int
-MajorVersion: TypeAlias = int
-MinorVersion: TypeAlias = int
-MinVersion: TypeAlias = tuple[MajorVersion, MinorVersion]
-MaxVersion: TypeAlias = tuple[MajorVersion, MinorVersion]
+VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
+VersionTuple: TypeAlias = tuple[int, int]
 Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
-Directory: TypeAlias = Annotated[str, "Must be one of the entries in TYPESHED_DIRECTORIES"]
-
-
-def python_version(arg: str) -> tuple[MajorVersion, MinorVersion]:
-    version = tuple(map(int, arg.split(".")))  # This will naturally raise TypeError if it's not in the form "{major}.{minor}"
-    if version not in SUPPORTED_VERSIONS:
-        raise ValueError
-    # mypy infers the return type as tuple[int, ...]
-    return version  # type: ignore[return-value]
 
 
 class CommandLineArgs(argparse.Namespace):
     verbose: int
-    exclude: list[str] | None
-    python_version: list[tuple[MajorVersion, MinorVersion]] | None
-    dir: list[Directory] | None
+    filter: list[Path]
+    exclude: list[Path] | None
+    python_version: list[VersionString] | None
     platform: list[Platform] | None
-    filter: list[str]
+
+
+def valid_path(cmd_arg: str) -> Path:
+    """Helper function for argument-parsing"""
+    path = Path(cmd_arg)
+    if not path.exists():
+        raise argparse.ArgumentTypeError(f'"{path}" does not exist in typeshed!')
+    if not (path in DIRECTORIES_TO_TEST or any(directory in path.parents for directory in DIRECTORIES_TO_TEST)):
+        raise argparse.ArgumentTypeError('mypy_test.py only tests the stubs found in the "stdlib" and "stubs" directories')
+    return path
 
 
 parser = argparse.ArgumentParser(
     description="Typecheck typeshed's stubs with mypy. Patterns are unanchored regexps on the full path."
 )
-parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
-parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern")
 parser.add_argument(
-    "-p", "--python-version", type=python_version, nargs="*", action="extend", help="These versions only (major[.minor])"
+    "filter",
+    type=valid_path,
+    nargs="*",
+    help='Test these files and directories (defaults to all files in the "stdlib" and "stubs" directories)',
 )
+parser.add_argument("-x", "--exclude", type=valid_path, nargs="*", help="Exclude these files and directories")
+parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
 parser.add_argument(
-    "-d",
-    "--dir",
-    choices=TYPESHED_DIRECTORIES,
+    "-p",
+    "--python-version",
+    type=str,
+    choices=SUPPORTED_VERSIONS,
     nargs="*",
     action="extend",
-    help="Test only these top-level typeshed directories (defaults to all typeshed directories)",
+    help="These versions only (major[.minor])",
 )
 parser.add_argument(
     "--platform",
@@ -76,7 +78,6 @@ parser.add_argument(
     action="extend",
     help="Run mypy for certain OS platforms (defaults to sys.platform only)",
 )
-parser.add_argument("filter", type=str, nargs="*", help="Include pattern (default all)")
 
 
 @dataclass
@@ -84,12 +85,10 @@ class TestConfig:
     """Configuration settings for a single run of the `test_typeshed` function."""
 
     verbose: int
-    exclude: list[str] | None
-    major: MajorVersion
-    minor: MinorVersion
-    directories: frozenset[Directory]
+    filter: list[Path]
+    exclude: list[Path]
+    version: VersionString
     platform: Platform
-    filter: list[str]
 
 
 def log(args: TestConfig, *varargs: object) -> None:
@@ -98,39 +97,35 @@ def log(args: TestConfig, *varargs: object) -> None:
 
 
 def match(path: Path, args: TestConfig) -> bool:
-    fn = str(path)
-    if not args.filter and not args.exclude:
-        log(args, fn, "accept by default")
-        return True
-    if args.exclude:
-        for f in args.exclude:
-            if re.search(f, fn):
-                log(args, fn, "excluded by pattern", f)
-                return False
-    if args.filter:
-        for f in args.filter:
-            if re.search(f, fn):
-                log(args, fn, "accepted by pattern", f)
-                return True
-    if args.filter:
-        log(args, fn, "rejected (no pattern matches)")
-        return False
-    log(args, fn, "accepted (no exclude pattern matches)")
-    return True
+    for excluded_path in args.exclude:
+        if path == excluded_path:
+            log(args, path, "explicitly excluded")
+            return False
+        if excluded_path in path.parents:
+            log(args, path, f'is in an explicitly excluded directory "{excluded_path}"')
+            return False
+    for included_path in args.filter:
+        if path == included_path:
+            log(args, path, "was explicitly included")
+            return True
+        if included_path in path.parents:
+            log(args, path, f'is in an explicitly included directory "{included_path}"')
+            return True
+    log_msg = (
+        f'is implicitly excluded: was not in any of the directories or paths specified on the command line: "{args.filter!r}"'
+    )
+    log(args, path, log_msg)
+    return False
 
 
-_VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$")
-
-
-def parse_versions(fname: StrPath) -> dict[str, tuple[MinVersion, MaxVersion]]:
+def parse_versions(fname: StrPath) -> dict[str, tuple[VersionTuple, VersionTuple]]:
     result = {}
     with open(fname) as f:
         for line in f:
-            # Allow having some comments or empty lines.
-            line = line.split("#")[0].strip()
+            line = strip_comments(line)
             if line == "":
                 continue
-            m = _VERSION_LINE_RE.match(line)
+            m = VERSION_LINE_RE.match(line)
             assert m, f"invalid VERSIONS line: {line}"
             mod: str = m.group(1)
             min_version = parse_version(m.group(2))
@@ -151,10 +146,11 @@ def parse_version(v_str: str) -> tuple[int, int]:
 def add_files(files: list[Path], seen: set[str], module: Path, args: TestConfig) -> None:
     """Add all files in package or module represented by 'name' located in 'root'."""
     if module.is_file() and module.suffix == ".pyi":
-        files.append(module)
-        seen.add(module.stem)
+        if match(module, args):
+            files.append(module)
+            seen.add(module.stem)
     else:
-        to_add = sorted(module.rglob("*.pyi"))
+        to_add = sorted(file for file in module.rglob("*.pyi") if match(file, args))
         files.extend(to_add)
         seen.update(path.stem for path in to_add)
 
@@ -239,7 +235,7 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[P
 def get_mypy_flags(args: TestConfig, temp_name: str) -> list[str]:
     return [
         "--python-version",
-        f"{args.major}.{args.minor}",
+        args.version,
         "--show-traceback",
         "--warn-incomplete-stub",
         "--show-error-codes",
@@ -323,7 +319,8 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults:
         if name == "VERSIONS" or name.startswith("."):
             continue
         module = Path(name).stem
-        if supported_versions[module][0] <= (args.major, args.minor) <= supported_versions[module][1]:
+        module_min_version, module_max_version = supported_versions[module]
+        if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version:
             add_files(files, seen, (stdlib / name), args)
 
     if files:
@@ -354,14 +351,15 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults:
 
 
 def test_typeshed(code: int, args: TestConfig) -> TestResults:
-    print(f"*** Testing Python {args.major}.{args.minor} on {args.platform}")
+    print(f"*** Testing Python {args.version} on {args.platform}")
     files_checked_this_version = 0
-    if "stdlib" in args.directories:
+    stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs")
+    if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter):
         code, stdlib_files_checked = test_stdlib(code, args)
         files_checked_this_version += stdlib_files_checked
         print()
 
-    if "stubs" in args.directories:
+    if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter):
         code, third_party_files_checked = test_third_party_stubs(code, args)
         files_checked_this_version += third_party_files_checked
         print()
@@ -373,19 +371,12 @@ def main() -> None:
     args = parser.parse_args(namespace=CommandLineArgs())
     versions = args.python_version or SUPPORTED_VERSIONS
     platforms = args.platform or [sys.platform]
-    tested_directories = frozenset(args.dir) if args.dir else TYPESHED_DIRECTORIES
+    filter = args.filter or DIRECTORIES_TO_TEST
+    exclude = args.exclude or []
     code = 0
     total_files_checked = 0
-    for (major, minor), platform in product(versions, platforms):
-        config = TestConfig(
-            verbose=args.verbose,
-            exclude=args.exclude,
-            major=major,
-            minor=minor,
-            directories=tested_directories,
-            platform=platform,
-            filter=args.filter,
-        )
+    for version, platform in product(versions, platforms):
+        config = TestConfig(args.verbose, filter, exclude, version, platform)
         code, files_checked_this_version = test_typeshed(code, args=config)
         total_files_checked += files_checked_this_version
     if code:

tests/utils.py

@@ -1,12 +1,18 @@
 """Utilities that are imported by multiple scripts in the tests directory."""
 
 import os
+import re
 from functools import cache
 from pathlib import Path
 from typing import NamedTuple
 
 import tomli
 
+
+def strip_comments(text: str) -> str:
+    return text.split("#")[0].strip()
+
+
 try:
     from termcolor import colored as colored
 except ImportError:
@@ -46,6 +52,14 @@ def read_dependencies(distribution: str) -> tuple[str, ...]:
     return tuple(dependencies)
 
 
+# ====================================================================
+# Parsing the stdlib/VERSIONS file
+# ====================================================================
+
+
+VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$")
+
+
 # ====================================================================
 # Getting test-case directories from package names
 # ====================================================================