diff --git a/tests/check_new_syntax.py b/tests/check_new_syntax.py index 56e3bfc43..41878fd2a 100755 --- a/tests/check_new_syntax.py +++ b/tests/check_new_syntax.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from __future__ import annotations + import ast import sys from itertools import chain diff --git a/tests/mypy_test.py b/tests/mypy_test.py index c72958d90..dfa89f0b2 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -14,6 +14,7 @@ from __future__ import annotations import argparse import os import re +import subprocess import sys import tempfile from pathlib import Path @@ -187,14 +188,18 @@ def run_mypy( def get_mypy_flags( - args: argparse.Namespace, major: int, minor: int, temp_name: str, *, custom_typeshed: bool = False + args: argparse.Namespace, + major: int, + minor: int, + temp_name: str | None, + *, + custom_typeshed: bool = False, + strict: bool = False, + test_suite_run: bool = False, ) -> list[str]: flags = [ "--python-version", "%d.%d" % (major, minor), - "--config-file", - temp_name, - "--no-site-packages", "--show-traceback", "--no-implicit-optional", "--disallow-untyped-decorators", @@ -206,12 +211,21 @@ def get_mypy_flags( "ignore-without-code", "--strict-equality", ] + if temp_name is not None: + flags.extend(["--config-file", temp_name]) if custom_typeshed: # Setting custom typeshed dir prevents mypy from falling back to its bundled # typeshed in case of stub deletions flags.extend(["--custom-typeshed-dir", os.path.dirname(os.path.dirname(__file__))]) if args.platform: flags.extend(["--platform", args.platform]) + if strict: + flags.append("--strict") + if test_suite_run: + if sys.platform == "win32" or args.platform == "win32": + flags.extend(["--exclude", "tests/pytype_test.py"]) + else: + flags.append("--no-site-packages") return flags @@ -280,6 +294,116 @@ def is_probably_stubs_folder(distribution: str, distribution_path: Path) -> bool return distribution != ".mypy_cache" and distribution_path.is_dir() +class TestResults(NamedTuple): + exit_code: int + files_checked: int + + +def test_stdlib(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults: + seen = {"__builtin__", "builtins", "typing"} # Always ignore these. + + files: list[str] = [] + if major == 2: + root = os.path.join("stdlib", "@python2") + for name in os.listdir(root): + mod, _ = os.path.splitext(name) + if mod in seen or mod.startswith("."): + continue + add_files(files, seen, root, name, args) + else: + supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS")) + root = "stdlib" + for name in os.listdir(root): + if name == "@python2" or name == "VERSIONS" or name.startswith("."): + continue + mod, _ = os.path.splitext(name) + if supported_versions[mod][0] <= (major, minor) <= supported_versions[mod][1]: + add_files(files, seen, root, name, args) + + if files: + print(f"Testing stdlib ({len(files)} files)...") + print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...", custom_typeshed=True))) + this_code = run_mypy(args, [], major, minor, files, custom_typeshed=True) + code = max(code, this_code) + + return TestResults(code, len(files)) + + +def test_third_party_stubs(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults: + print("Testing third-party packages...") + print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/..."))) + files_checked = 0 + + for distribution in sorted(os.listdir("stubs")): + if distribution == "SQLAlchemy": + continue # Crashes + + distribution_path = Path("stubs", distribution) + + if not is_probably_stubs_folder(distribution, distribution_path): + continue + + this_code, checked = test_third_party_distribution(distribution, major, minor, args) + code = max(code, this_code) + files_checked += checked + + return TestResults(code, files_checked) + + +def test_the_test_scripts(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults: + files_to_test = list(Path("tests").rglob("*.py")) + if sys.platform == "win32": + files_to_test.remove(Path("tests/pytype_test.py")) + num_test_files_to_test = len(files_to_test) + flags = get_mypy_flags(args, major, minor, None, strict=True, test_suite_run=True) + print(f"Testing the test suite ({num_test_files_to_test} files)...") + print("Running mypy " + " ".join(flags)) + this_code = subprocess.run([sys.executable, "-m", "mypy", "tests", *flags]).returncode + code = max(code, this_code) + return TestResults(code, num_test_files_to_test) + + +def test_the_test_cases(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults: + test_case_files = list(map(str, Path("test_cases").rglob("*.py"))) + num_test_case_files = len(test_case_files) + flags = get_mypy_flags(args, major, minor, None, strict=True, custom_typeshed=True) + print(f"Running mypy on the test_cases directory ({num_test_case_files} files)...") + print("Running mypy " + " ".join(flags)) + this_code = subprocess.run([sys.executable, "-m", "mypy", "test_cases", *flags]).returncode + code = max(code, this_code) + return TestResults(code, num_test_case_files) + + +def test_typeshed(code: int, major: int, minor: int, args: argparse.Namespace) -> TestResults: + print(f"*** Testing Python {major}.{minor}") + files_checked_this_version = 0 + code, stdlib_files_checked = test_stdlib(code, major, minor, args) + files_checked_this_version += stdlib_files_checked + print() + + if major == 2: + return TestResults(code, files_checked_this_version) + + code, third_party_files_checked = test_third_party_stubs(code, major, minor, args) + files_checked_this_version += third_party_files_checked + print() + + if minor >= 9: + # Run mypy against our own test suite + # + # Skip this on earlier Python versions, + # as we're using new syntax and new functions in some test files + code, test_script_files_checked = test_the_test_scripts(code, major, minor, args) + files_checked_this_version += test_script_files_checked + print() + + code, test_case_files_checked = test_the_test_cases(code, major, minor, args) + files_checked_this_version += test_case_files_checked + print() + + return TestResults(code, files_checked_this_version) + + def main() -> None: args = parser.parse_args() @@ -291,63 +415,17 @@ def main() -> None: sys.exit(1) code = 0 - files_checked = 0 + total_files_checked = 0 for major, minor in versions: - print(f"*** Testing Python {major}.{minor}") - - seen = {"__builtin__", "builtins", "typing"} # Always ignore these. - - # Test standard library files. - files: list[str] = [] - if major == 2: - root = os.path.join("stdlib", "@python2") - for name in os.listdir(root): - mod, _ = os.path.splitext(name) - if mod in seen or mod.startswith("."): - continue - add_files(files, seen, root, name, args) - else: - supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS")) - root = "stdlib" - for name in os.listdir(root): - if name == "@python2" or name == "VERSIONS" or name.startswith("."): - continue - mod, _ = os.path.splitext(name) - if supported_versions[mod][0] <= (major, minor) <= supported_versions[mod][1]: - add_files(files, seen, root, name, args) - - if files: - print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/...", custom_typeshed=True))) - print(f"testing stdlib ({len(files)} files)...") - this_code = run_mypy(args, [], major, minor, files, custom_typeshed=True) - code = max(code, this_code) - files_checked += len(files) - - # Test files of all third party distributions. - if major != 2: - print("Running mypy " + " ".join(get_mypy_flags(args, major, minor, "/tmp/..."))) - for distribution in sorted(os.listdir("stubs")): - if distribution == "SQLAlchemy": - continue # Crashes - - distribution_path = Path("stubs", distribution) - - if not is_probably_stubs_folder(distribution, distribution_path): - continue - - this_code, checked = test_third_party_distribution(distribution, major, minor, args) - code = max(code, this_code) - files_checked += checked - - print() - + code, files_checked_this_version = test_typeshed(code, major, minor, args) + total_files_checked += files_checked_this_version if code: - print(f"--- exit status {code}, {files_checked} files checked ---") + print(f"--- exit status {code}, {total_files_checked} files checked ---") sys.exit(code) - if not files_checked: + if not total_files_checked: print("--- nothing to do; exit 1 ---") sys.exit(1) - print(f"--- success, {files_checked} files checked ---") + print(f"--- success, {total_files_checked} files checked ---") if __name__ == "__main__": diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 37eea8e7f..a7bb36466 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -19,8 +19,8 @@ import sys import traceback from collections.abc import Sequence -from pytype import config as pytype_config, load_pytd -from pytype.pytd import typeshed +from pytype import config as pytype_config, load_pytd # type: ignore[import] +from pytype.pytd import typeshed # type: ignore[import] TYPESHED_SUBDIRS = ["stdlib", "stubs"] TYPESHED_HOME = "TYPESHED_HOME"