VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2025-12-16 00:37:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

regr_test.py: Allow non-types dependencies (#9382)

Browse Source
This commit is contained in:
Alex Waygood
2022-12-23 21:55:54 +00:00
committed by GitHub
parent 4379a6a509
commit 8671fc5c0f
5 changed files with 250 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/tests.yml vendored
View File

@@ -100,7 +100,7 @@ jobs:
cache: pip
cache-dependency-path: requirements-tests.txt
- run: pip install -r requirements-tests.txt
- run: python ./tests/regr_test.py --all --quiet
- run: python ./tests/regr_test.py --all --verbosity QUIET
pyright:
name: Test typeshed with pyright

2
tests/mypy_test.py
View File

@@ -274,7 +274,7 @@ def add_third_party_files(
seen_dists.add(distribution)
stubs_dir = Path("stubs")
dependencies = get_recursive_requirements(distribution)
dependencies = get_recursive_requirements(distribution).typeshed_pkgs
for dependency in dependencies:
if dependency in seen_dists:

191
tests/regr_test.py
View File

@@ -10,15 +10,19 @@ import shutil
import subprocess
import sys
import tempfile
from enum import IntEnum
from itertools import product
from pathlib import Path
from typing_extensions import TypeAlias
from utils import (
PackageInfo,
VenvInfo,
colored,
get_all_testcase_directories,
get_mypy_req,
get_recursive_requirements,
make_venv,
print_error,
print_success_msg,
testcase_dir_from_package_name,
@@ -26,6 +30,10 @@ from utils import (
ReturnCode: TypeAlias = int
TEST_CASES = "test_cases"
VENV_DIR = ".venv"
TYPESHED = "typeshed"
SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"]
SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
@@ -34,7 +42,7 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
"""Helper function for argument-parsing"""
if package_name == "stdlib":
return PackageInfo("stdlib", Path("test_cases"))
return PackageInfo("stdlib", Path(TEST_CASES))
test_case_dir = testcase_dir_from_package_name(package_name)
if test_case_dir.is_dir():
if not os.listdir(test_case_dir):
@@ -43,6 +51,12 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!")
class Verbosity(IntEnum):
QUIET = 0
NORMAL = 1
VERBOSE = 2
parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs")
parser.add_argument(
"packages_to_test",
@@ -59,7 +73,12 @@ parser.add_argument(
"Note that this cannot be specified if --platform and/or --python-version are specified."
),
)
parser.add_argument("--quiet", action="store_true", help="Print less output to the terminal")
parser.add_argument(
"--verbosity",
choices=[member.name for member in Verbosity],
default=Verbosity.NORMAL.name,
help="Control how much output to print to the terminal",
)
parser.add_argument(
"--platform",
dest="platforms_to_test",
@@ -85,16 +104,64 @@ parser.add_argument(
)
def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
package_name, test_case_directory = package
is_stdlib = package_name == "stdlib"
def verbose_log(msg: str) -> None:
print(colored("\n" + msg, "blue"))
msg = f"Running mypy --platform {platform} --python-version {version} on the "
msg += "standard library test cases..." if is_stdlib else f"test cases for {package_name!r}..."
if not quiet:
print(msg, end=" ")
# "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path, verbosity: Verbosity) -> None:
if verbosity is verbosity.VERBOSE:
verbose_log(f"Setting up testcase dir in {tempdir}")
# --warn-unused-ignores doesn't work for files inside typeshed.
# SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
# and run the test cases inside of that.
shutil.copytree(package.test_case_directory, new_test_case_dir)
if package.is_stdlib:
return
# HACK: we want to run these test cases in an isolated environment --
# we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
# (and all stub packages required by those stub packages, etc. etc.),
# but none of the other stubs in typeshed.
#
# The best way of doing that without stopping --warn-unused-ignore from working
# seems to be to create a "new typeshed" directory in a tempdir
# that has only the required stubs copied over.
new_typeshed = tempdir / TYPESHED
new_typeshed.mkdir()
shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
requirements = get_recursive_requirements(package.name)
# mypy refuses to consider a directory a "valid typeshed directory"
# unless there's a stubs/mypy-extensions path inside it,
# so add that to the list of stubs to copy over to the new directory
for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
if requirements.external_pkgs:
if verbosity is Verbosity.VERBOSE:
verbose_log(f"Setting up venv in {tempdir / VENV_DIR}")
pip_exe = make_venv(tempdir / VENV_DIR).pip_exe
pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
if verbosity is Verbosity.VERBOSE:
verbose_log(f"{pip_command=}")
try:
subprocess.run(pip_command, check=True, capture_output=True, text=True)
except subprocess.CalledProcessError as e:
print(e.stderr)
raise
def run_testcases(
package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
) -> subprocess.CompletedProcess[str]:
env_vars = dict(os.environ)
new_test_case_dir = tempdir / TEST_CASES
testcasedir_already_setup = new_test_case_dir.exists() and new_test_case_dir.is_dir()
if not testcasedir_already_setup:
setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir, verbosity=verbosity)
# "--enable-error-code ignore-without-code" is purposefully ommited.
# See https://github.com/python/typeshed/pull/8083
flags = [
"--python-version",
version,
@@ -103,67 +170,70 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
"--no-error-summary",
"--platform",
platform,
"--no-site-packages",
"--strict",
"--pretty",
]
# --warn-unused-ignores doesn't work for files inside typeshed.
# SO, to work around this, we copy the test_cases directory into a TemporaryDirectory.
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
new_test_case_dir = td_path / "test_cases"
shutil.copytree(test_case_directory, new_test_case_dir)
env_vars = dict(os.environ)
if is_stdlib:
flags.extend(["--custom-typeshed-dir", str(Path(__file__).parent.parent)])
if package.is_stdlib:
python_exe = sys.executable
custom_typeshed = Path(__file__).parent.parent
flags.append("--no-site-packages")
else:
custom_typeshed = tempdir / TYPESHED
env_vars["MYPYPATH"] = os.pathsep.join(map(str, custom_typeshed.glob("stubs/*")))
has_non_types_dependencies = (tempdir / VENV_DIR).exists()
if has_non_types_dependencies:
python_exe = VenvInfo.of_existing_venv(tempdir / VENV_DIR).python_exe
else:
# HACK: we want to run these test cases in an isolated environment --
# we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
# (and all stub packages required by those stub packages, etc. etc.),
# but none of the other stubs in typeshed.
#
# The best way of doing that without stopping --warn-unused-ignore from working
# seems to be to create a "new typeshed" directory in a tempdir
# that has only the required stubs copied over.
new_typeshed = td_path / "typeshed"
os.mkdir(new_typeshed)
shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
requirements = get_recursive_requirements(package_name)
# mypy refuses to consider a directory a "valid typeshed directory"
# unless there's a stubs/mypy-extensions path inside it,
# so add that to the list of stubs to copy over to the new directory
for requirement in requirements + ["mypy-extensions"]:
shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")])
python_exe = sys.executable
flags.append("--no-site-packages")
# If the test-case filename ends with -py39,
# only run the test if --python-version was set to 3.9 or higher (for example)
for path in new_test_case_dir.rglob("*.py"):
if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
minor_version_required = int(match[1])
assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
if minor_version_required <= int(version.split(".")[1]):
flags.append(str(path))
else:
flags.append(str(path))
flags.extend(["--custom-typeshed-dir", str(custom_typeshed)])
result = subprocess.run([sys.executable, "-m", "mypy", *flags], capture_output=True, env=env_vars)
# If the test-case filename ends with -py39,
# only run the test if --python-version was set to 3.9 or higher (for example)
for path in new_test_case_dir.rglob("*.py"):
if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
minor_version_required = int(match[1])
assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
python_minor_version = int(version.split(".")[1])
if minor_version_required > python_minor_version:
continue
flags.append(str(path))
mypy_command = [python_exe, "-m", "mypy"] + flags
if verbosity is Verbosity.VERBOSE:
verbose_log(f"{mypy_command=}")
if "MYPYPATH" in env_vars:
verbose_log(f"{env_vars['MYPYPATH']=}")
else:
verbose_log("MYPYPATH not set")
return subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
def test_testcase_directory(
package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path
) -> ReturnCode:
msg = f"Running mypy --platform {platform} --python-version {version} on the "
msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
if verbosity > Verbosity.QUIET:
print(msg, end=" ", flush=True)
result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity)
if result.returncode:
if quiet:
if verbosity > Verbosity.QUIET:
# We'll already have printed this if --quiet wasn't passed.
# If--quiet was passed, only print this if there were errors.
# If --quiet was passed, only print this if there were errors.
# If there are errors, the output is inscrutable if this isn't printed.
print(msg, end=" ")
print_error("failure\n")
replacements = (str(new_test_case_dir), str(test_case_directory))
replacements = (str(tempdir / TEST_CASES), str(package.test_case_directory))
if result.stderr:
print_error(result.stderr.decode(), fix_path=replacements)
print_error(result.stderr, fix_path=replacements)
if result.stdout:
print_error(result.stdout.decode(), fix_path=replacements)
elif not quiet:
print_error(result.stdout, fix_path=replacements)
elif verbosity > Verbosity.QUIET:
print_success_msg()
return result.returncode
@@ -172,6 +242,7 @@ def main() -> ReturnCode:
args = parser.parse_args()
testcase_directories = args.packages_to_test or get_all_testcase_directories()
verbosity = Verbosity[args.verbosity]
if args.all:
if args.platforms_to_test:
parser.error("Cannot specify both --platform and --all")
@@ -183,8 +254,12 @@ def main() -> ReturnCode:
versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"]
code = 0
for platform, version, directory in product(platforms_to_test, versions_to_test, testcase_directories):
code = max(code, test_testcase_directory(directory, version, platform, args.quiet))
for testcase_dir in testcase_directories:
with tempfile.TemporaryDirectory() as td:
tempdir = Path(td)
for platform, version in product(platforms_to_test, versions_to_test):
this_code = test_testcase_directory(testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir)
code = max(code, this_code)
if code:
print_error("\nTest completed with errors")
else:

31
tests/stubtest_third_party.py
View File

@@ -4,23 +4,15 @@
from __future__ import annotations
import argparse
import functools
import os
import subprocess
import sys
import tempfile
import venv
from pathlib import Path
from typing import NoReturn
import tomli
from utils import colored, print_error, print_success_msg
@functools.lru_cache()
def get_mypy_req() -> str:
with open("requirements-tests.txt", encoding="UTF-8") as f:
return next(line.strip() for line in f if "mypy" in line)
from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg
def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: bool = False) -> bool:
@@ -44,25 +36,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: boo
with tempfile.TemporaryDirectory() as tmp:
venv_dir = Path(tmp)
try:
venv.create(venv_dir, with_pip=True, clear=True)
except subprocess.CalledProcessError as e:
if "ensurepip" in e.cmd:
print_error("fail")
print_error(
"stubtest requires a Python installation with ensurepip. "
"If on Linux, you may need to install the python3-venv package."
)
pip_exe, python_exe = make_venv(venv_dir)
except Exception:
print_error("fail")
raise
if sys.platform == "win32":
pip = venv_dir / "Scripts" / "pip.exe"
python = venv_dir / "Scripts" / "python.exe"
else:
pip = venv_dir / "bin" / "pip"
python = venv_dir / "bin" / "python"
pip_exe, python_exe = str(pip), str(python)
dist_version = metadata["version"]
extras = stubtest_meta.get("extras", [])
assert isinstance(dist_version, str)

126
tests/utils.py
View File

@@ -4,13 +4,18 @@ from __future__ import annotations
import os
import re
import subprocess
import sys
import venv
from collections.abc import Mapping
from functools import cache
from itertools import filterfalse
from pathlib import Path
from typing import NamedTuple
from typing_extensions import Annotated
import pathspec # type: ignore[import]
import tomli
from packaging.requirements import Requirement
# Used to install system-wide packages for different OS types:
METADATA_MAPPING = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
@@ -45,25 +50,112 @@ def print_success_msg() -> None:
# ====================================================================
class PackageDependencies(NamedTuple):
typeshed_pkgs: tuple[str, ...]
external_pkgs: tuple[str, ...]
@cache
def read_dependencies(distribution: str) -> tuple[str, ...]:
def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
stub_name_map = {}
for typeshed_name in os.listdir("stubs"):
with Path("stubs", typeshed_name, "METADATA.toml").open("rb") as f:
pypi_name = tomli.load(f).get("stub_distribution", f"types-{typeshed_name}")
assert isinstance(pypi_name, str)
stub_name_map[pypi_name] = typeshed_name
return stub_name_map
@cache
def read_dependencies(distribution: str) -> PackageDependencies:
"""Read the dependencies listed in a METADATA.toml file for a stubs package.
Once the dependencies have been read,
determine which dependencies are typeshed-internal dependencies,
and which dependencies are external (non-types) dependencies.
For typeshed dependencies, translate the "dependency name" into the "package name";
for external dependencies, leave them as they are in the METADATA.toml file.
Note that this function may consider things to be typeshed stubs
even if they haven't yet been uploaded to PyPI.
If a typeshed stub is removed, this function will consider it to be an external dependency.
"""
pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
data = tomli.load(f)
requires = data.get("requires", [])
assert isinstance(requires, list)
dependencies = []
for dependency in requires:
dependencies = tomli.load(f).get("requires", [])
assert isinstance(dependencies, list)
typeshed, external = [], []
for dependency in dependencies:
assert isinstance(dependency, str)
assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}"
dependencies.append(dependency[6:].split("<")[0])
return tuple(dependencies)
maybe_typeshed_dependency = Requirement(dependency).name
if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping:
typeshed.append(pypi_name_to_typeshed_name_mapping[maybe_typeshed_dependency])
else:
external.append(dependency)
return PackageDependencies(tuple(typeshed), tuple(external))
def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> list[str]:
seen = seen if seen is not None else {package_name}
for dependency in filterfalse(seen.__contains__, read_dependencies(package_name)):
seen.update(get_recursive_requirements(dependency, seen))
return sorted(seen | {package_name})
@cache
def get_recursive_requirements(package_name: str) -> PackageDependencies:
"""Recursively gather dependencies for a single stubs package.
For example, if the stubs for `caldav`
declare a dependency on typeshed's stubs for `requests`,
and the stubs for requests declare a dependency on typeshed's stubs for `urllib3`,
`get_recursive_requirements("caldav")` will determine that the stubs for `caldav`
have both `requests` and `urllib3` as typeshed-internal dependencies.
"""
typeshed: set[str] = set()
external: set[str] = set()
non_recursive_requirements = read_dependencies(package_name)
typeshed.update(non_recursive_requirements.typeshed_pkgs)
external.update(non_recursive_requirements.external_pkgs)
for pkg in non_recursive_requirements.typeshed_pkgs:
reqs = get_recursive_requirements(pkg)
typeshed.update(reqs.typeshed_pkgs)
external.update(reqs.external_pkgs)
return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external)))
# ====================================================================
# Dynamic venv creation
# ====================================================================
class VenvInfo(NamedTuple):
pip_exe: Annotated[str, "A path to the venv's pip executable"]
python_exe: Annotated[str, "A path to the venv's python executable"]
@staticmethod
def of_existing_venv(venv_dir: Path) -> VenvInfo:
if sys.platform == "win32":
pip = venv_dir / "Scripts" / "pip.exe"
python = venv_dir / "Scripts" / "python.exe"
else:
pip = venv_dir / "bin" / "pip"
python = venv_dir / "bin" / "python"
return VenvInfo(str(pip), str(python))
def make_venv(venv_dir: Path) -> VenvInfo:
try:
venv.create(venv_dir, with_pip=True, clear=True)
except subprocess.CalledProcessError as e:
if "ensurepip" in e.cmd:
print_error(
"stubtest requires a Python installation with ensurepip. "
"If on Linux, you may need to install the python3-venv package."
)
raise
return VenvInfo.of_existing_venv(venv_dir)
@cache
def get_mypy_req() -> str:
with open("requirements-tests.txt", encoding="UTF-8") as f:
return next(line.strip() for line in f if "mypy" in line)
# ====================================================================
@@ -83,6 +175,10 @@ class PackageInfo(NamedTuple):
name: str
test_case_directory: Path
@property
def is_stdlib(self) -> bool:
return self.name == "stdlib"
def testcase_dir_from_package_name(package_name: str) -> Path:
return Path("stubs", package_name, "@tests/test_cases")