Fix mypy_test if mypy returns negative exit code (#10866)

tests/mypy_test.py

@@ -13,6 +13,7 @@ import tempfile
 import time
 from collections import defaultdict
 from dataclasses import dataclass
+from enum import Enum
 from itertools import product
 from pathlib import Path
 from threading import Lock
@@ -51,7 +52,6 @@ SUPPORTED_VERSIONS = ["3.12", "3.11", "3.10", "3.9", "3.8"]
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
 DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
 
-ReturnCode: TypeAlias = int
 VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
 VersionTuple: TypeAlias = Tuple[int, int]
 Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
@@ -222,6 +222,12 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) ->
         configurations.append(MypyDistConf(module_name, values.copy()))
 
 
+class MypyResult(Enum):
+    SUCCESS = 0
+    FAILURE = 1
+    CRASH = 2
+
+
 def run_mypy(
     args: TestConfig,
     configurations: list[MypyDistConf],
@@ -231,7 +237,7 @@ def run_mypy(
     non_types_dependencies: bool,
     venv_info: VenvInfo,
     mypypath: str | None = None,
-) -> ReturnCode:
+) -> MypyResult:
     env_vars = dict(os.environ)
     if mypypath is not None:
         env_vars["MYPYPATH"] = mypypath
@@ -276,7 +282,7 @@ def run_mypy(
             print(colored(f"running {' '.join(mypy_command)}", "blue"))
         result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
         if result.returncode:
-            print_error("failure\n")
+            print_error(f"failure (exit code {result.returncode})\n")
             if result.stdout:
                 print_error(result.stdout)
             if result.stderr:
@@ -287,7 +293,12 @@ def run_mypy(
                 print()
         else:
             print_success_msg()
-        return result.returncode
+        if result.returncode == 0:
+            return MypyResult.SUCCESS
+        elif result.returncode == 1:
+            return MypyResult.FAILURE
+        else:
+            return MypyResult.CRASH
 
 
 def add_third_party_files(
@@ -305,15 +316,14 @@ def add_third_party_files(
         add_configuration(configurations, distribution)
 
 
-class TestResults(NamedTuple):
-    exit_code: int
+class TestResult(NamedTuple):
+    mypy_result: MypyResult
     files_checked: int
-    packages_skipped: int = 0
 
 
 def test_third_party_distribution(
     distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool
-) -> TestResults:
+) -> TestResult:
     """Test the stubs of a third-party distribution.
 
     Return a tuple, where the first element indicates mypy's return code
@@ -326,7 +336,7 @@ def test_third_party_distribution(
     add_third_party_files(distribution, files, args, configurations, seen_dists)
 
     if not files and args.filter:
-        return TestResults(0, 0)
+        return TestResult(MypyResult.SUCCESS, 0)
 
     print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True)
 
@@ -337,7 +347,7 @@ def test_third_party_distribution(
     mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists)
     if args.verbose:
         print(colored(f"\nMYPYPATH={mypypath}", "blue"))
-    code = run_mypy(
+    result = run_mypy(
         args,
         configurations,
         files,
@@ -346,10 +356,10 @@ def test_third_party_distribution(
         testing_stdlib=False,
         non_types_dependencies=non_types_dependencies,
     )
-    return TestResults(code, len(files))
+    return TestResult(result, len(files))
 
 
-def test_stdlib(code: int, args: TestConfig) -> TestResults:
+def test_stdlib(args: TestConfig) -> TestResult:
     files: list[Path] = []
     stdlib = Path("stdlib")
     supported_versions = parse_versions(stdlib / "VERSIONS")
@@ -361,14 +371,39 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults:
         if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version:
             add_files(files, (stdlib / name), args)
 
-    if files:
-        print(f"Testing stdlib ({len(files)} files)...", end="", flush=True)
-        # We don't actually need pip for the stdlib testing
-        venv_info = VenvInfo(pip_exe="", python_exe=sys.executable)
-        this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False)
-        code = max(code, this_code)
+    if not files:
+        return TestResult(MypyResult.SUCCESS, 0)
 
-    return TestResults(code, len(files))
+    print(f"Testing stdlib ({len(files)} files)...", end="", flush=True)
+    # We don't actually need pip for the stdlib testing
+    venv_info = VenvInfo(pip_exe="", python_exe=sys.executable)
+    result = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False)
+    return TestResult(result, len(files))
+
+
+@dataclass
+class TestSummary:
+    mypy_result: MypyResult = MypyResult.SUCCESS
+    files_checked: int = 0
+    packages_skipped: int = 0
+    packages_with_errors: int = 0
+
+    def register_result(self, mypy_result: MypyResult, files_checked: int) -> None:
+        if mypy_result.value > self.mypy_result.value:
+            self.mypy_result = mypy_result
+        if mypy_result != MypyResult.SUCCESS:
+            self.packages_with_errors += 1
+        self.files_checked += files_checked
+
+    def skip_package(self) -> None:
+        self.packages_skipped += 1
+
+    def merge(self, other: TestSummary) -> None:
+        if other.mypy_result.value > self.mypy_result.value:
+            self.mypy_result = other.mypy_result
+        self.files_checked += other.files_checked
+        self.packages_skipped += other.packages_skipped
+        self.packages_with_errors += other.packages_with_errors
 
 
 _PRINT_LOCK = Lock()
@@ -473,10 +508,9 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar
         _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use))
 
 
-def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults:
+def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary:
     print("Testing third-party packages...")
-    files_checked = 0
-    packages_skipped = 0
+    summary = TestSummary()
     gitignore_spec = get_gitignore_spec()
     distributions_to_check: dict[str, PackageDependencies] = {}
 
@@ -493,12 +527,12 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe
                 f"test is being run using Python {PYTHON_VERSION})"
             )
             print(colored(msg, "yellow"))
-            packages_skipped += 1
+            summary.skip_package()
             continue
         if not metadata.requires_python.contains(args.version):
             msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})"
             print(colored(msg, "yellow"))
-            packages_skipped += 1
+            summary.skip_package()
             continue
 
         if (
@@ -527,32 +561,30 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe
     for distribution in distributions_to_check:
         venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution]
         non_types_dependencies = venv_info.python_exe != sys.executable
-        this_code, checked, _ = test_third_party_distribution(
+        mypy_result, files_checked = test_third_party_distribution(
             distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies
         )
-        code = max(code, this_code)
-        files_checked += checked
+        summary.register_result(mypy_result, files_checked)
 
-    return TestResults(code, files_checked, packages_skipped)
+    return summary
 
 
-def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults:
+def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary:
     print(f"*** Testing Python {args.version} on {args.platform}")
-    files_checked_this_version = 0
-    packages_skipped_this_version = 0
     stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs")
+    summary = TestSummary()
+
     if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter):
-        code, stdlib_files_checked, _ = test_stdlib(code, args)
-        files_checked_this_version += stdlib_files_checked
+        mypy_result, files_checked = test_stdlib(args)
+        summary.register_result(mypy_result, files_checked)
         print()
 
     if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter):
-        code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir)
-        files_checked_this_version += third_party_files_checked
-        packages_skipped_this_version = third_party_packages_skipped
+        tp_results = test_third_party_stubs(args, tempdir)
+        summary.merge(tp_results)
         print()
 
-    return TestResults(code, files_checked_this_version, packages_skipped_this_version)
+    return summary
 
 
 def main() -> None:
@@ -561,26 +593,31 @@ def main() -> None:
     platforms = args.platform or [sys.platform]
     filter = args.filter or DIRECTORIES_TO_TEST
     exclude = args.exclude or []
-    code = 0
-    total_files_checked = 0
-    total_packages_skipped = 0
+    summary = TestSummary()
     with tempfile.TemporaryDirectory() as td:
         td_path = Path(td)
         for version, platform in product(versions, platforms):
             config = TestConfig(args.verbose, filter, exclude, version, platform)
-            code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path)
-            total_files_checked += files_checked_this_version
-            total_packages_skipped += packages_skipped_this_version
-    if code:
-        plural = "" if total_files_checked == 1 else "s"
-        print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---")
-        sys.exit(code)
-    if total_packages_skipped:
-        plural = "" if total_packages_skipped == 1 else "s"
-        print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow"))
-    if total_files_checked:
-        plural = "" if total_files_checked == 1 else "s"
-        print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green"))
+            version_summary = test_typeshed(args=config, tempdir=td_path)
+            summary.merge(version_summary)
+
+    if summary.mypy_result == MypyResult.FAILURE:
+        plural1 = "" if summary.packages_with_errors == 1 else "s"
+        plural2 = "" if summary.files_checked == 1 else "s"
+        print_error(
+            f"--- {summary.packages_with_errors} package{plural1} with errors, {summary.files_checked} file{plural2} checked ---"
+        )
+        sys.exit(1)
+    if summary.mypy_result == MypyResult.CRASH:
+        plural = "" if summary.files_checked == 1 else "s"
+        print_error(f"--- mypy crashed, {summary.files_checked} file{plural} checked ---")
+        sys.exit(2)
+    if summary.packages_skipped:
+        plural = "" if summary.packages_skipped == 1 else "s"
+        print(colored(f"--- {summary.packages_skipped} package{plural} skipped ---", "yellow"))
+    if summary.files_checked:
+        plural = "" if summary.files_checked == 1 else "s"
+        print(colored(f"--- success, {summary.files_checked} file{plural} checked ---", "green"))
     else:
         print_error("--- nothing to do; exit 1 ---")
         sys.exit(1)