From 875f0ca7fcc68e7bd6c9b807cdceeff8a6f734c8 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 22 Aug 2022 17:16:03 +0100 Subject: [PATCH] `mypy_test.py`: Move type-checking of our tests and scripts into a different test (#8587) --- .github/workflows/tests.yml | 4 +- .github/workflows/typecheck_typeshed_code.yml | 36 +++++++ tests/README.md | 22 +++- tests/mypy_test.py | 95 +++------------- tests/typecheck_typeshed.py | 102 ++++++++++++++++++ 5 files changed, 171 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/typecheck_typeshed_code.yml create mode 100644 tests/typecheck_typeshed.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index de70935b8..0dd89281f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: - run: ./tests/pytype_test.py --print-stderr mypy: - name: Run mypy against typeshed + name: Test the stubs with mypy runs-on: ubuntu-latest strategy: matrix: @@ -77,7 +77,7 @@ jobs: - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} pyright: - name: Run pyright against typeshed + name: Test the stubs with pyright runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/typecheck_typeshed_code.yml b/.github/workflows/typecheck_typeshed_code.yml new file mode 100644 index 000000000..e28b99221 --- /dev/null +++ b/.github/workflows/typecheck_typeshed_code.yml @@ -0,0 +1,36 @@ +name: Typecheck-typeshed-code + +on: + workflow_dispatch: + push: + branches: + - main + - master + pull_request: + paths: + - 'scripts/**' + - 'tests/**' + - '.github/workflows/typecheck_typeshed_code.yml' + - 'requirements-tests.txt' + +permissions: + contents: read + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +jobs: + mypy: + name: Run mypy against the scripts and tests directories + runs-on: ubuntu-latest + strategy: + matrix: + platform: ["linux", "win32"] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.9 + - run: pip install -r requirements-tests.txt + - run: python ./tests/typecheck_typeshed.py --platform=${{ matrix.platform }} diff --git a/tests/README.md b/tests/README.md index d87a69641..c04cc3795 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,9 @@ This directory contains several tests: - `tests/mypy_test.py` -tests typeshed with [mypy](https://github.com/python/mypy/) -- `tests/pytype_test.py` tests typeshed with +tests the stubs with [mypy](https://github.com/python/mypy/) +- `tests/pytype_test.py` tests the stubs with [pytype](https://github.com/google/pytype/). -- `tests/pyright_test.py` tests typeshed with +- `tests/pyright_test.py` tests the stubs with [pyright](https://github.com/microsoft/pyright). - `tests/check_consistent.py` checks certain files in typeshed remain consistent with each other. @@ -11,6 +11,8 @@ consistent with each other. objects at runtime. - `tests/stubtest_third_party.py` checks third-party stubs against the objects at runtime. +- `tests/typecheck_typeshed.py` runs mypy against typeshed's own code +in the `tests` and `scripts` directories. To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment) in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.9+. @@ -22,10 +24,9 @@ Run using: (.venv3)$ python3 tests/mypy_test.py ``` -The test has four parts. Each part uses mypy with slightly different configuration options: +The test has three parts. Each part uses mypy with slightly different configuration options: - Running mypy on the stdlib stubs - Running mypy on the third-party stubs -- Running mypy `--strict` on the scripts in the `tests` directory - Running mypy `--strict` on the regression tests in the `test_cases` directory. When running mypy on the stubs, this test is shallow — it verifies that all stubs can be @@ -118,3 +119,14 @@ check on the command line: For each distribution, stubtest ignores definitions listed in a `@tests/stubtest_allowlist.txt` file, relative to the distribution. Additional packages that are needed to run stubtest for a distribution can be added to `@tests/requirements-stubtest.txt`. + +## typecheck\_typeshed.py + +Run using +``` +(.venv3)$ python3 tests/typecheck_typeshed.py +``` + +This is a small wrapper script that uses mypy to typecheck typeshed's own code in the +`scripts` and `tests` directories. Run `python tests/typecheck_typeshed.py --help` for +information on the various configuration options. diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 5a4e97ab1..9518d8ad0 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -29,11 +29,14 @@ import tomli from colors import colored, print_error, print_success_msg SUPPORTED_VERSIONS = [(3, 11), (3, 10), (3, 9), (3, 8), (3, 7)] -SUPPORTED_PLATFORMS = frozenset({"linux", "win32", "darwin"}) -TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs", "tests", "test_cases", "scripts"}) +SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") +TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs", "test_cases"}) +ReturnCode: TypeAlias = int MajorVersion: TypeAlias = int MinorVersion: TypeAlias = int +MinVersion: TypeAlias = tuple[MajorVersion, MinorVersion] +MaxVersion: TypeAlias = tuple[MajorVersion, MinorVersion] Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"] Directory: TypeAlias = Annotated[str, "Must be one of the entries in TYPESHED_DIRECTORIES"] @@ -122,8 +125,6 @@ def match(fn: str, args: TestConfig) -> bool: _VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$") -MinVersion: TypeAlias = tuple[MajorVersion, MinorVersion] -MaxVersion: TypeAlias = tuple[MajorVersion, MinorVersion] def parse_versions(fname: StrPath) -> dict[str, tuple[MinVersion, MaxVersion]]: @@ -210,7 +211,7 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) -> configurations.append(MypyDistConf(module_name, values.copy())) -def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[str], *, custom_typeshed: bool = False) -> int: +def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[str]) -> ReturnCode: try: from mypy.api import run as mypy_run except ImportError: @@ -225,7 +226,7 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[s temp.write(f"{k} = {v}\n") temp.flush() - flags = get_mypy_flags(args, temp.name, custom_typeshed=custom_typeshed) + flags = get_mypy_flags(args, temp.name) mypy_args = [*flags, *files] if args.verbose: print("running mypy", " ".join(mypy_args)) @@ -253,9 +254,6 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[s return exit_code -ReturnCode: TypeAlias = int - - def run_mypy_as_subprocess(directory: StrPath, flags: Iterable[str]) -> ReturnCode: result = subprocess.run([sys.executable, "-m", "mypy", directory, *flags], capture_output=True) stdout, stderr = result.stdout, result.stderr @@ -267,14 +265,7 @@ def run_mypy_as_subprocess(directory: StrPath, flags: Iterable[str]) -> ReturnCo def get_mypy_flags( - args: TestConfig, - temp_name: str | None, - *, - custom_typeshed: bool = False, - strict: bool = False, - test_suite_run: bool = False, - enforce_error_codes: bool = True, - ignore_missing_imports: bool = False, + args: TestConfig, temp_name: str | None, *, strict: bool = False, enforce_error_codes: bool = True ) -> list[str]: flags = [ "--python-version", @@ -285,6 +276,9 @@ def get_mypy_flags( "--no-error-summary", "--platform", args.platform, + "--no-site-packages", + "--custom-typeshed-dir", + os.path.dirname(os.path.dirname(__file__)), ] if strict: flags.append("--strict") @@ -292,20 +286,8 @@ def get_mypy_flags( flags.extend(["--no-implicit-optional", "--disallow-untyped-decorators", "--disallow-any-generics", "--strict-equality"]) if temp_name is not None: flags.extend(["--config-file", temp_name]) - if custom_typeshed: - # Setting custom typeshed dir prevents mypy from falling back to its bundled - # typeshed in case of stub deletions - flags.extend(["--custom-typeshed-dir", os.path.dirname(os.path.dirname(__file__))]) - if test_suite_run: - flags.append("--namespace-packages") - if args.platform == "win32": - flags.extend(["--exclude", "tests/pytype_test.py"]) - else: - flags.append("--no-site-packages") if enforce_error_codes: flags.extend(["--enable-error-code", "ignore-without-code"]) - if ignore_missing_imports: - flags.append("--ignore-missing-imports") return flags @@ -389,8 +371,8 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: if files: print(f"Testing stdlib ({len(files)} files)...") - print("Running mypy " + " ".join(get_mypy_flags(args, "/tmp/...", custom_typeshed=True))) - this_code = run_mypy(args, [], files, custom_typeshed=True) + print("Running mypy " + " ".join(get_mypy_flags(args, "/tmp/..."))) + this_code = run_mypy(args, [], files) code = max(code, this_code) return TestResults(code, len(files)) @@ -414,44 +396,10 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults: return TestResults(code, files_checked) -def test_the_test_scripts(code: int, args: TestConfig) -> TestResults: - files_to_test = list(Path("tests").rglob("*.py")) - if args.platform == "win32": - files_to_test.remove(Path("tests/pytype_test.py")) - num_test_files_to_test = len(files_to_test) - flags = get_mypy_flags(args, None, strict=True, test_suite_run=True) - print(f"Testing the test suite ({num_test_files_to_test} files)...") - print("Running mypy " + " ".join(flags)) - if args.dry_run: - this_code = 0 - else: - this_code = run_mypy_as_subprocess("tests", flags) - if not this_code: - print_success_msg() - code = max(code, this_code) - return TestResults(code, num_test_files_to_test) - - -def test_scripts_directory(code: int, args: TestConfig) -> TestResults: - files_to_test = list(Path("scripts").rglob("*.py")) - num_test_files_to_test = len(files_to_test) - flags = get_mypy_flags(args, None, strict=True, ignore_missing_imports=True) - print(f"Testing the scripts directory ({num_test_files_to_test} files)...") - print("Running mypy " + " ".join(flags)) - if args.dry_run: - this_code = 0 - else: - this_code = run_mypy_as_subprocess("scripts", flags) - if not this_code: - print_success_msg() - code = max(code, this_code) - return TestResults(code, num_test_files_to_test) - - def test_the_test_cases(code: int, args: TestConfig) -> TestResults: test_case_files = list(map(str, Path("test_cases").rglob("*.py"))) num_test_case_files = len(test_case_files) - flags = get_mypy_flags(args, None, strict=True, custom_typeshed=True, enforce_error_codes=False) + flags = get_mypy_flags(args, None, strict=True, enforce_error_codes=False) print(f"Running mypy on the test_cases directory ({num_test_case_files} files)...") print("Running mypy " + " ".join(flags)) if args.dry_run: @@ -481,21 +429,6 @@ def test_typeshed(code: int, args: TestConfig) -> TestResults: files_checked_this_version += third_party_files_checked print() - if args.minor >= 9: - # Run mypy against our own test suite and the scripts directory - # - # Skip this on earlier Python versions, - # as we're using new syntax and new functions in some test files - if "tests" in args.directories: - code, test_script_files_checked = test_the_test_scripts(code, args) - files_checked_this_version += test_script_files_checked - print() - - if "scripts" in args.directories: - code, script_files_checked = test_scripts_directory(code, args) - files_checked_this_version += script_files_checked - print() - if "test_cases" in args.directories: code, test_case_files_checked = test_the_test_cases(code, args) files_checked_this_version += test_case_files_checked diff --git a/tests/typecheck_typeshed.py b/tests/typecheck_typeshed.py new file mode 100644 index 000000000..8e3f0c8c8 --- /dev/null +++ b/tests/typecheck_typeshed.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Run mypy on the "tests" and "scripts" directories.""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from itertools import product +from typing_extensions import TypeAlias + +from colors import colored, print_error + +ReturnCode: TypeAlias = int + +SUPPORTED_PLATFORMS = ("linux", "darwin", "win32") +SUPPORTED_VERSIONS = ("3.11", "3.10", "3.9") +DIRECTORIES_TO_TEST = ("scripts", "tests") + +parser = argparse.ArgumentParser(description="Run mypy on typeshed's own code in the `scripts` and `tests` directories.") +parser.add_argument( + "--platform", + choices=SUPPORTED_PLATFORMS, + nargs="*", + action="extend", + help="Run mypy for certain OS platforms (defaults to sys.platform)", +) +parser.add_argument( + "-p", + "--python-version", + choices=SUPPORTED_VERSIONS, + nargs="*", + action="extend", + help="Run mypy for certain Python versions (defaults to sys.version_info[:2])", +) +parser.add_argument( + "-d", + "--dir", + choices=DIRECTORIES_TO_TEST, + nargs="*", + action="extend", + help=f"Test only these top-level typeshed directories (defaults to {DIRECTORIES_TO_TEST!r})", +) + + +def run_mypy_as_subprocess(directory: str, platform: str, version: str) -> ReturnCode: + command = [ + sys.executable, + "-m", + "mypy", + directory, + "--platform", + platform, + "--python-version", + version, + "--strict", + "--show-traceback", + "--show-error-codes", + "--no-error-summary", + "--enable-error-code", + "ignore-without-code", + "--namespace-packages", + ] + if directory == "tests": + if platform == "win32": + command.extend(["--exclude", "tests/pytype_test.py"]) + else: + command.append("--ignore-missing-imports") + result = subprocess.run(command, capture_output=True) + stdout, stderr = result.stdout, result.stderr + if stderr: + print_error(stderr.decode()) + if stdout: + print_error(stdout.decode()) + return result.returncode + + +def main() -> ReturnCode: + args = parser.parse_args() + directories = args.dir or DIRECTORIES_TO_TEST + platforms = args.platform or [sys.platform] + versions = args.python_version or [f"3.{sys.version_info[1]}"] + + code = 0 + + for directory, platform, version in product(directories, platforms, versions): + print(f'Running "mypy --platform {platform} --python-version {version}" on the "{directory}" directory...') + code = max(code, run_mypy_as_subprocess(directory, platform, version)) + + if code: + print_error("Test completed with errors") + else: + print(colored("Test completed successfully!", "green")) + return code + + +if __name__ == "__main__": + try: + code = main() + except KeyboardInterrupt: + print_error("\n\n!!!\nTest aborted due to KeyboardInterrupt\n!!!") + code = 1 + raise SystemExit(code)