Run mypy on the test cases and test scripts (#7746)

tests/check_new_syntax.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from __future__ import annotations
+
 import ast
 import sys
 from itertools import chain

tests/mypy_test.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import argparse
 import os
 import re
+import subprocess
 import sys
 import tempfile
 from pathlib import Path
@@ -187,14 +188,18 @@ def run_mypy(
 
 
 def get_mypy_flags(
-    args: argparse.Namespace, major: int, minor: int, temp_name: str, *, custom_typeshed: bool = False
+    args: argparse.Namespace,
+    major: int,
+    minor: int,
+    temp_name: str | None,
+    *,
+    custom_typeshed: bool = False,
+    strict: bool = False,
+    test_suite_run: bool = False,
 ) -> list[str]:
     flags = [
         "--python-version",
         "%d.%d" % (major, minor),
-        "--config-file",
-        temp_name,
-        "--no-site-packages",
         "--show-traceback",
         "--no-implicit-optional",
         "--disallow-untyped-decorators",
@@ -206,12 +211,21 @@ def get_mypy_flags(
         "ignore-without-code",
         "--strict-equality",
     ]
+    if temp_name is not None:
+        flags.extend(["--config-file", temp_name])
     if custom_typeshed:
         # Setting custom typeshed dir prevents mypy from falling back to its bundled
         # typeshed in case of stub deletions
         flags.extend(["--custom-typeshed-dir", os.path.dirname(os.path.dirname(__file__))])
     if args.platform:
         flags.extend(["--platform", args.platform])
+    if strict:
+        flags.append("--strict")
+    if test_suite_run:
+        if sys.platform == "win32" or args.platform == "win32":
+            flags.extend(["--exclude", "tests/pytype_test.py"])
+    else:
+        flags.append("--no-site-packages")
     return flags
 
 
@@ -280,6 +294,116 @@ def is_probably_stubs_folder(distribution: str, distribution_path: Path) -> bool
     return distribution != ".mypy_cache" and distribution_path.is_dir()
 
 
+class TestResults(NamedTuple):
+    exit_code: int
+    files_checked: int
+
+
+def test_stdlib(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults:
+    seen = {"__builtin__", "builtins", "typing"}  # Always ignore these.
+
+    files: list[str] = []
+    if major == 2:
+        root = os.path.join("stdlib", "@python2")
+        for name in os.listdir(root):
+            mod, _ = os.path.splitext(name)
+            if mod in seen or mod.startswith("."):
+                continue
+            add_files(files, seen, root, name, args)
+    else:
+        supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS"))
+        root = "stdlib"
+        for name in os.listdir(root):
+            if name == "@python2" or name == "VERSIONS" or name.startswith("."):
+                continue
+            mod, _ = os.path.splitext(name)
+            if supported_versions[mod][0] <= (major, minor) <= supported_versions[mod][1]:
+                add_files(files, seen, root, name, args)
+
+    if files:
+        print(f"Testing stdlib ({len(files)} files)...")
+        print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...", custom_typeshed=True)))
+        this_code = run_mypy(args, [], major, minor, files, custom_typeshed=True)
+        code = max(code, this_code)
+
+    return TestResults(code, len(files))
+
+
+def test_third_party_stubs(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults:
+    print("Testing third-party packages...")
+    print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...")))
+    files_checked = 0
+
+    for distribution in sorted(os.listdir("stubs")):
+        if distribution == "SQLAlchemy":
+            continue  # Crashes
+
+        distribution_path = Path("stubs", distribution)
+
+        if not is_probably_stubs_folder(distribution, distribution_path):
+            continue
+
+        this_code, checked = test_third_party_distribution(distribution, major, minor, args)
+        code = max(code, this_code)
+        files_checked += checked
+
+    return TestResults(code, files_checked)
+
+
+def test_the_test_scripts(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults:
+    files_to_test = list(Path("tests").rglob("*.py"))
+    if sys.platform == "win32":
+        files_to_test.remove(Path("tests/pytype_test.py"))
+    num_test_files_to_test = len(files_to_test)
+    flags = get_mypy_flags(args, major, minor, None, strict=True, test_suite_run=True)
+    print(f"Testing the test suite ({num_test_files_to_test} files)...")
+    print("Running mypy " + " ".join(flags))
+    this_code = subprocess.run([sys.executable, "-m", "mypy", "tests", *flags]).returncode
+    code = max(code, this_code)
+    return TestResults(code, num_test_files_to_test)
+
+
+def test_the_test_cases(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults:
+    test_case_files = list(map(str, Path("test_cases").rglob("*.py")))
+    num_test_case_files = len(test_case_files)
+    flags = get_mypy_flags(args, major, minor, None, strict=True, custom_typeshed=True)
+    print(f"Running mypy on the test_cases directory ({num_test_case_files} files)...")
+    print("Running mypy " + " ".join(flags))
+    this_code = subprocess.run([sys.executable, "-m", "mypy", "test_cases", *flags]).returncode
+    code = max(code, this_code)
+    return TestResults(code, num_test_case_files)
+
+
+def test_typeshed(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults:
+    print(f"*** Testing Python {major}.{minor}")
+    files_checked_this_version = 0
+    code, stdlib_files_checked = test_stdlib(code, major, minor, args)
+    files_checked_this_version += stdlib_files_checked
+    print()
+
+    if major == 2:
+        return TestResults(code, files_checked_this_version)
+
+    code, third_party_files_checked = test_third_party_stubs(code, major, minor, args)
+    files_checked_this_version += third_party_files_checked
+    print()
+
+    if minor >= 9:
+        # Run mypy against our own test suite
+        #
+        # Skip this on earlier Python versions,
+        # as we're using new syntax and new functions in some test files
+        code, test_script_files_checked = test_the_test_scripts(code, major, minor, args)
+        files_checked_this_version += test_script_files_checked
+        print()
+
+    code, test_case_files_checked = test_the_test_cases(code, major, minor, args)
+    files_checked_this_version += test_case_files_checked
+    print()
+
+    return TestResults(code, files_checked_this_version)
+
+
 def main() -> None:
     args = parser.parse_args()
 
@@ -291,63 +415,17 @@ def main() -> None:
             sys.exit(1)
 
     code = 0
-    files_checked = 0
+    total_files_checked = 0
     for major, minor in versions:
-        print(f"*** Testing Python {major}.{minor}")
-
-        seen = {"__builtin__", "builtins", "typing"}  # Always ignore these.
-
-        # Test standard library files.
-        files: list[str] = []
-        if major == 2:
-            root = os.path.join("stdlib", "@python2")
-            for name in os.listdir(root):
-                mod, _ = os.path.splitext(name)
-                if mod in seen or mod.startswith("."):
-                    continue
-                add_files(files, seen, root, name, args)
-        else:
-            supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS"))
-            root = "stdlib"
-            for name in os.listdir(root):
-                if name == "@python2" or name == "VERSIONS" or name.startswith("."):
-                    continue
-                mod, _ = os.path.splitext(name)
-                if supported_versions[mod][0] <= (major, minor) <= supported_versions[mod][1]:
-                    add_files(files, seen, root, name, args)
-
-        if files:
-            print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...", custom_typeshed=True)))
-            print(f"testing stdlib ({len(files)} files)...")
-            this_code = run_mypy(args, [], major, minor, files, custom_typeshed=True)
-            code = max(code, this_code)
-            files_checked += len(files)
-
-        # Test files of all third party distributions.
-        if major != 2:
-            print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...")))
-            for distribution in sorted(os.listdir("stubs")):
-                if distribution == "SQLAlchemy":
-                    continue  # Crashes
-
-                distribution_path = Path("stubs", distribution)
-
-                if not is_probably_stubs_folder(distribution, distribution_path):
-                    continue
-
-                this_code, checked = test_third_party_distribution(distribution, major, minor, args)
-                code = max(code, this_code)
-                files_checked += checked
-
-        print()
-
+        code, files_checked_this_version = test_typeshed(code, major, minor, args)
+        total_files_checked += files_checked_this_version
     if code:
-        print(f"--- exit status {code}, {files_checked} files checked ---")
+        print(f"--- exit status {code}, {total_files_checked} files checked ---")
         sys.exit(code)
-    if not files_checked:
+    if not total_files_checked:
         print("--- nothing to do; exit 1 ---")
         sys.exit(1)
-    print(f"--- success, {files_checked} files checked ---")
+    print(f"--- success, {total_files_checked} files checked ---")
 
 
 if __name__ == "__main__":

tests/pytype_test.py

@@ -19,8 +19,8 @@ import sys
 import traceback
 from collections.abc import Sequence
 
-from pytype import config as pytype_config, load_pytd
-from pytype.pytd import typeshed
+from pytype import config as pytype_config, load_pytd  # type: ignore[import]
+from pytype.pytd import typeshed  # type: ignore[import]
 
 TYPESHED_SUBDIRS = ["stdlib", "stubs"]
 TYPESHED_HOME = "TYPESHED_HOME"