From 9d450cb50ca76b92f0c1ab704cdd51bd59f7b9b7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 29 Apr 2022 21:55:12 -0600 Subject: [PATCH] Make test scripts pass `mypy --strict` (#7745) - Add several type hints to untyped functions. - While we're at it, upgrade several annotations to use modern syntax by using `from __future__ import annotations`. This means that the tests can't be run on Python 3.6, but 3.6 is EOL, and it is already the case that some scripts in this directory can only be run on more recent Python versions. E.g. `check_new_syntax.py` uses `ast.unparse`, which is only available in Python 3.9+. - Fix a few pieces of code that didn't type check. --- tests/check_consistent.py | 15 +++++++------ tests/mypy_test.py | 40 ++++++++++++++++++----------------- tests/pytype_test.py | 23 ++++++++++---------- tests/stubtest_third_party.py | 2 +- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 2b6ed496a..fd97a3d22 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -10,6 +10,7 @@ # symlinks but that doesn't always work on Windows, so now you must # manually update both files, and this test verifies that they are # identical. The list below indicates which sets of files must match. +from __future__ import annotations import filecmp import os @@ -22,7 +23,7 @@ metadata_keys = {"version", "requires", "extra_description", "obsolete_since", " allowed_files = {"README.md"} -def assert_stubs_only(directory): +def assert_stubs_only(directory: str) -> None: """Check that given directory contains only valid stub files.""" top = directory.split(os.sep)[-1] assert top.isidentifier(), f"Bad directory name: {top}" @@ -37,7 +38,7 @@ def assert_stubs_only(directory): assert subdir.isidentifier(), f"Directories must be valid packages, got: {subdir}" -def check_stdlib(): +def check_stdlib() -> None: for entry in os.listdir("stdlib"): if os.path.isfile(os.path.join("stdlib", entry)): name, ext = os.path.splitext(entry) @@ -57,7 +58,7 @@ def check_stdlib(): assert_stubs_only(os.path.join("stdlib/@python2", entry)) -def check_stubs(): +def check_stubs() -> None: for distribution in os.listdir("stubs"): assert not os.path.isfile(distribution), f"Only directories allowed in stubs, got {distribution}" for entry in os.listdir(os.path.join("stubs", distribution)): @@ -73,7 +74,7 @@ def check_stubs(): assert_stubs_only(os.path.join("stubs", distribution, entry)) -def check_same_files(): +def check_same_files() -> None: files = [os.path.join(root, file) for root, dir, files in os.walk(".") for file in files] no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link." for file in files: @@ -94,7 +95,7 @@ def check_same_files(): _VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): [23]\.\d{1,2}-(?:[23]\.\d{1,2})?$") -def check_versions(): +def check_versions() -> None: versions = set() with open("stdlib/VERSIONS") as f: data = f.read().splitlines() @@ -131,7 +132,7 @@ def _find_stdlib_modules() -> set[str]: return modules -def _strip_dep_version(dependency): +def _strip_dep_version(dependency: str) -> tuple[str, str, str]: dep_version_pos = len(dependency) for pos, c in enumerate(dependency): if c in "=<>": @@ -151,7 +152,7 @@ def _strip_dep_version(dependency): return stripped, relation, version -def check_metadata(): +def check_metadata() -> None: for distribution in os.listdir("stubs"): with open(os.path.join("stubs", distribution, "METADATA.toml")) as f: data = tomli.loads(f.read()) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 50b39ce13..c72958d90 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -9,6 +9,7 @@ Approach: 2. Compute appropriate arguments for mypy 3. Pass those arguments to mypy.api.run() """ +from __future__ import annotations import argparse import os @@ -16,7 +17,7 @@ import re import sys import tempfile from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, NamedTuple, Set, Tuple +from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: from _typeshed import StrPath @@ -61,10 +62,11 @@ def match(fn: str, args: argparse.Namespace) -> bool: _VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$") -VersionsTuple: TypeAlias = Tuple[int, int] +MinVersion: TypeAlias = tuple[int, int] +MaxVersion: TypeAlias = tuple[int, int] -def parse_versions(fname: "StrPath") -> Dict[str, Tuple[VersionsTuple, VersionsTuple]]: +def parse_versions(fname: StrPath) -> dict[str, tuple[MinVersion, MaxVersion]]: result = {} with open(fname) as f: for line in f: @@ -84,13 +86,13 @@ def parse_versions(fname: "StrPath") -> Dict[str, Tuple[VersionsTuple, VersionsT _VERSION_RE = re.compile(r"^([23])\.(\d+)$") -def parse_version(v_str: str) -> Tuple[int, int]: +def parse_version(v_str: str) -> tuple[int, int]: m = _VERSION_RE.match(v_str) assert m, "invalid version: " + v_str return int(m.group(1)), int(m.group(2)) -def add_files(files: List[str], seen: Set[str], root: str, name: str, args: argparse.Namespace) -> None: +def add_files(files: list[str], seen: set[str], root: str, name: str, args: argparse.Namespace) -> None: """Add all files in package or module represented by 'name' located in 'root'.""" full = os.path.join(root, name) mod, ext = os.path.splitext(name) @@ -113,7 +115,7 @@ def add_files(files: List[str], seen: Set[str], root: str, name: str, args: argp class MypyDistConf(NamedTuple): module_name: str - values: Dict + values: dict # The configuration section in the metadata file looks like the following, with multiple module sections possible @@ -125,7 +127,7 @@ class MypyDistConf(NamedTuple): # disallow_untyped_defs = true -def add_configuration(configurations: List[MypyDistConf], distribution: str) -> None: +def add_configuration(configurations: list[MypyDistConf], distribution: str) -> None: with open(os.path.join("stubs", distribution, "METADATA.toml")) as f: data = dict(tomli.loads(f.read())) @@ -150,10 +152,10 @@ def add_configuration(configurations: List[MypyDistConf], distribution: str) -> def run_mypy( args: argparse.Namespace, - configurations: List[MypyDistConf], + configurations: list[MypyDistConf], major: int, minor: int, - files: List[str], + files: list[str], *, custom_typeshed: bool = False, ) -> int: @@ -186,7 +188,7 @@ def run_mypy( def get_mypy_flags( args: argparse.Namespace, major: int, minor: int, temp_name: str, *, custom_typeshed: bool = False -) -> List[str]: +) -> list[str]: flags = [ "--python-version", "%d.%d" % (major, minor), @@ -213,7 +215,7 @@ def get_mypy_flags( return flags -def read_dependencies(distribution: str) -> List[str]: +def read_dependencies(distribution: str) -> list[str]: with open(os.path.join("stubs", distribution, "METADATA.toml")) as f: data = dict(tomli.loads(f.read())) requires = data.get("requires", []) @@ -229,10 +231,10 @@ def read_dependencies(distribution: str) -> List[str]: def add_third_party_files( distribution: str, major: int, - files: List[str], + files: list[str], args: argparse.Namespace, - configurations: List[MypyDistConf], - seen_dists: Set[str], + configurations: list[MypyDistConf], + seen_dists: set[str], ) -> None: if distribution in seen_dists: return @@ -251,16 +253,16 @@ def add_third_party_files( add_configuration(configurations, distribution) -def test_third_party_distribution(distribution: str, major: int, minor: int, args: argparse.Namespace) -> Tuple[int, int]: +def test_third_party_distribution(distribution: str, major: int, minor: int, args: argparse.Namespace) -> tuple[int, int]: """Test the stubs of a third-party distribution. Return a tuple, where the first element indicates mypy's return code and the second element is the number of checked files. """ - files: List[str] = [] - configurations: List[MypyDistConf] = [] - seen_dists: Set[str] = set() + files: list[str] = [] + configurations: list[MypyDistConf] = [] + seen_dists: set[str] = set() add_third_party_files(distribution, major, files, args, configurations, seen_dists) print(f"testing {distribution} ({len(files)} files)...") @@ -296,7 +298,7 @@ def main() -> None: seen = {"__builtin__", "builtins", "typing"} # Always ignore these. # Test standard library files. - files: List[str] = [] + files: list[str] = [] if major == 2: root = os.path.join("stdlib", "@python2") for name in os.listdir(root): diff --git a/tests/pytype_test.py b/tests/pytype_test.py index d99406bdd..37eea8e7f 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -11,21 +11,19 @@ Option two will load the file and all the builtins, typeshed dependencies. This will also discover incorrect usage of imported modules. """ +from __future__ import annotations + import argparse import os import sys import traceback -from typing import List, Optional, Sequence +from collections.abc import Sequence from pytype import config as pytype_config, load_pytd from pytype.pytd import typeshed TYPESHED_SUBDIRS = ["stdlib", "stubs"] - - TYPESHED_HOME = "TYPESHED_HOME" -UNSET = object() # marker for tracking the TYPESHED_HOME environment variable - _LOADERS = {} @@ -34,13 +32,13 @@ def main() -> None: typeshed_location = args.typeshed_location or os.getcwd() subdir_paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS] check_subdirs_discoverable(subdir_paths) - old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET) + old_typeshed_home = os.environ.get(TYPESHED_HOME) os.environ[TYPESHED_HOME] = typeshed_location files_to_test = determine_files_to_test(typeshed_location=typeshed_location, paths=args.files or subdir_paths) run_all_tests( files_to_test=files_to_test, typeshed_location=typeshed_location, print_stderr=args.print_stderr, dry_run=args.dry_run ) - if old_typeshed_home is UNSET: + if old_typeshed_home is None: del os.environ[TYPESHED_HOME] else: os.environ[TYPESHED_HOME] = old_typeshed_home @@ -61,13 +59,14 @@ def create_parser() -> argparse.ArgumentParser: return parser -def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> Optional[str]: +def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> str | None: """Runs pytype, returning the stderr if any.""" if python_version not in _LOADERS: options = pytype_config.Options.create("", parse_pyi=True, python_version=python_version) loader = load_pytd.create_loader(options) _LOADERS[python_version] = (options, loader) options, loader = _LOADERS[python_version] + stderr: str | None try: with pytype_config.verbosity_from(options): ast = loader.load_file(_get_module_name(filename), filename) @@ -108,13 +107,13 @@ def _is_version(path: str, version: str) -> bool: return any("{}{}{}".format(d, os.path.sep, version) in path for d in TYPESHED_SUBDIRS) -def check_subdirs_discoverable(subdir_paths: List[str]) -> None: +def check_subdirs_discoverable(subdir_paths: list[str]) -> None: for p in subdir_paths: if not os.path.isdir(p): raise SystemExit("Cannot find typeshed subdir at {} (specify parent dir via --typeshed-location)".format(p)) -def determine_files_to_test(*, typeshed_location: str, paths: Sequence[str]) -> List[str]: +def determine_files_to_test(*, typeshed_location: str, paths: Sequence[str]) -> list[str]: """Determine all files to test, checking if it's in the exclude list and which Python versions to use. Returns a list of pairs of the file path and Python version as an int.""" @@ -130,8 +129,8 @@ def determine_files_to_test(*, typeshed_location: str, paths: Sequence[str]) -> return files -def find_stubs_in_paths(paths: Sequence[str]) -> List[str]: - filenames = [] +def find_stubs_in_paths(paths: Sequence[str]) -> list[str]: + filenames: list[str] = [] for path in paths: if os.path.isdir(path): for root, _, fns in os.walk(path): diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 191f95cbf..b02a7cec2 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -16,7 +16,7 @@ import tomli @functools.lru_cache() -def get_mypy_req(): +def get_mypy_req() -> str: with open("requirements-tests.txt") as f: return next(line.strip() for line in f if "mypy" in line)