Unify test directory handling (#11864)

Previously, handling of test directories (`@tests` and `test_cases`) was
distributed over multiple files and redundant. This unifies the handling
in the `utils` module. This also fixes some instances where "package"
was used instead of "distribution". And in a few instances paths were
joined by using a `/` in a string, which is incompatible with Windows.

Also move `runtests.py` from `scripts` to `tests`. This is required so that
we can import `utils`, but it's also arguably the better fit. The only
mention of the script is actually in the `tests/README.md` file.

Helps with #11762.

tests/README.md

@@ -36,7 +36,7 @@ You can list or install all of a stubs package's external dependencies using the
 
 Run using:
 ```bash
-(.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
+(.venv3)$ python3 tests/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
 
 This script will run all tests below for a specific typeshed directory. If a
@@ -46,7 +46,7 @@ be selected. A summary of the results will be printed to the terminal.
 You must provide a single argument which is a path to the stubs to test, like
 so: `stdlib/os` or `stubs/requests`.
 
-Run `python scripts/runtests.py --help` for information on the various configuration options
+Run `python tests/runtests.py --help` for information on the various configuration options
 for this script. Note that if you use the `--run-stubtest` flag with the stdlib stubs,
 whether or not the test passes will depend on the exact version of Python
 you're using, as well as various other details regarding your local environment.

tests/check_typeshed_structure.py

@@ -15,12 +15,15 @@ from pathlib import Path
 from parse_metadata import read_metadata
 from utils import (
     REQS_FILE,
+    TEST_CASES_DIR,
+    TESTS_DIR,
     VERSIONS_RE,
     get_all_testcase_directories,
     get_gitignore_spec,
     parse_requirements,
     spec_matches_path,
     strip_comments,
+    tests_path,
 )
 
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
@@ -73,13 +76,15 @@ def check_stubs() -> None:
         ), f"Directory name must be a valid distribution name: {dist}"
         assert not dist.name.startswith("types-"), f"Directory name not allowed to start with 'types-': {dist}"
 
-        allowed = {"METADATA.toml", "README", "README.md", "README.rst", "@tests"}
+        allowed = {"METADATA.toml", "README", "README.md", "README.rst", TESTS_DIR}
         assert_consistent_filetypes(dist, kind=".pyi", allowed=allowed)
 
-        tests_dir = dist / "@tests"
+        tests_dir = tests_path(dist.name)
         if tests_dir.exists() and tests_dir.is_dir():
             py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir())
-            error_message = "Test-case files must be in an `@tests/test_cases/` directory, not in the `@tests/` directory"
+            error_message = (
+                f"Test-case files must be in an `{TESTS_DIR}/{TEST_CASES_DIR}` directory, not in the `{TESTS_DIR}` directory"
+            )
             assert not py_files_present, error_message
 
 
@@ -101,7 +106,7 @@ def check_test_cases() -> None:
     """Check that the test_cases directory contains only the correct files."""
     for _, testcase_dir in get_all_testcase_directories():
         assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True)
-        bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"'
+        bad_test_case_filename = f'Files in a `{TEST_CASES_DIR}` directory must have names starting with "check_"; got "{{}}"'
         for file in testcase_dir.rglob("*.py"):
             assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
 

tests/regr_test.py

@@ -24,18 +24,18 @@ from typing_extensions import TypeAlias
 from parse_metadata import get_recursive_requirements, read_metadata
 from utils import (
     PYTHON_VERSION,
-    PackageInfo,
+    TEST_CASES_DIR,
+    DistributionTests,
     colored,
+    distribution_info,
     get_all_testcase_directories,
     get_mypy_req,
     print_error,
-    testcase_dir_from_package_name,
     venv_python,
 )
 
 ReturnCode: TypeAlias = int
 
-TEST_CASES = "test_cases"
 VENV_DIR = ".venv"
 TYPESHED = "typeshed"
 
@@ -43,17 +43,13 @@ SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"]
 SUPPORTED_VERSIONS = ["3.12", "3.11", "3.10", "3.9", "3.8"]
 
 
-def package_with_test_cases(package_name: str) -> PackageInfo:
-    """Helper function for argument-parsing"""
+def distribution_with_test_cases(distribution_name: str) -> DistributionTests:
+    """Helper function for argument-parsing."""
 
-    if package_name == "stdlib":
-        return PackageInfo("stdlib", Path(TEST_CASES))
-    test_case_dir = testcase_dir_from_package_name(package_name)
-    if test_case_dir.is_dir():
-        if not os.listdir(test_case_dir):
-            raise argparse.ArgumentTypeError(f"{package_name!r} has a 'test_cases' directory but it is empty!")
-        return PackageInfo(package_name, test_case_dir)
-    raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!")
+    try:
+        return distribution_info(distribution_name)
+    except RuntimeError as exc:
+        raise argparse.ArgumentTypeError(str(exc)) from exc
 
 
 class Verbosity(IntEnum):
@@ -65,7 +61,7 @@ class Verbosity(IntEnum):
 parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs")
 parser.add_argument(
     "packages_to_test",
-    type=package_with_test_cases,
+    type=distribution_with_test_cases,
     nargs="*",
     action="extend",
     help=(
@@ -118,13 +114,13 @@ def verbose_log(msg: str) -> None:
     _PRINT_QUEUE.put(colored(msg, "blue"))
 
 
-def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity) -> None:
+def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Verbosity) -> None:
     if verbosity is verbosity.VERBOSE:
         verbose_log(f"{package.name}: Setting up testcase dir in {tempdir}")
     # --warn-unused-ignores doesn't work for files inside typeshed.
     # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
     # and run the test cases inside of that.
-    shutil.copytree(package.test_case_directory, tempdir / TEST_CASES)
+    shutil.copytree(package.test_cases_path, tempdir / TEST_CASES_DIR)
     if package.is_stdlib:
         return
 
@@ -163,10 +159,10 @@ def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity
 
 
 def run_testcases(
-    package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
+    package: DistributionTests, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
 ) -> subprocess.CompletedProcess[str]:
     env_vars = dict(os.environ)
-    new_test_case_dir = tempdir / TEST_CASES
+    new_test_case_dir = tempdir / TEST_CASES_DIR
 
     # "--enable-error-code ignore-without-code" is purposefully omitted.
     # See https://github.com/python/typeshed/pull/8083
@@ -239,14 +235,16 @@ class Result:
         if self.code:
             print(f"{self.command_run}:", end=" ")
             print_error("FAILURE\n")
-            replacements = (str(self.tempdir / TEST_CASES), str(self.test_case_dir))
+            replacements = (str(self.tempdir / TEST_CASES_DIR), str(self.test_case_dir))
             if self.stderr:
                 print_error(self.stderr, fix_path=replacements)
             if self.stdout:
                 print_error(self.stdout, fix_path=replacements)
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path) -> Result:
+def test_testcase_directory(
+    package: DistributionTests, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path
+) -> Result:
     msg = f"mypy --platform {platform} --python-version {version} on the "
     msg += "standard library test cases" if package.is_stdlib else f"test cases for {package.name!r}"
     if verbosity > Verbosity.QUIET:
@@ -258,7 +256,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, *
         command_run=msg,
         stderr=proc_info.stderr,
         stdout=proc_info.stdout,
-        test_case_dir=package.test_case_directory,
+        test_case_dir=package.test_cases_path,
         tempdir=tempdir,
     )
 
@@ -278,13 +276,13 @@ def print_queued_messages(ev: threading.Event) -> None:
 
 def concurrently_run_testcases(
     stack: ExitStack,
-    testcase_directories: list[PackageInfo],
+    testcase_directories: list[DistributionTests],
     verbosity: Verbosity,
     platforms_to_test: list[str],
     versions_to_test: list[str],
 ) -> list[Result]:
     packageinfo_to_tempdir = {
-        package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories
+        distribution_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for distribution_info in testcase_directories
     }
     to_do: list[Callable[[], Result]] = []
     for testcase_dir, tempdir in packageinfo_to_tempdir.items():

tests/runtests.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from utils import TEST_CASES_DIR, test_cases_path
+
+try:
+    from termcolor import colored  # pyright: ignore[reportAssignmentType]
+except ImportError:
+
+    def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type: ignore[misc]
+        return text
+
+
+_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
+_TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"
+_NPX_ERROR_PATTERN = r"error (runn|find)ing npx"
+_NPX_ERROR_MESSAGE = colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow")
+_SUCCESS = colored("Success", "green")
+_SKIPPED = colored("Skipped", "yellow")
+_FAILED = colored("Failed", "red")
+# We're using the oldest fully supported version because it's the most likely to produce errors
+# due to unsupported syntax, feature, or bug in a tool.
+_PYTHON_VERSION = "3.8"
+
+
+def _parse_jsonc(json_text: str) -> str:
+    # strip comments from the file
+    lines = [line for line in json_text.split("\n") if not line.strip().startswith("//")]
+    # strip trailing commas from the file
+    valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
+    return valid_json
+
+
+def _get_strict_params(stub_path: str) -> list[str]:
+    with open(_STRICTER_CONFIG_FILE, encoding="UTF-8") as file:
+        data = json.loads(_parse_jsonc(file.read()))
+    lower_stub_path = stub_path.lower()
+    if any(lower_stub_path == stub.lower() for stub in data["exclude"]):
+        return []
+    return ["-p", _STRICTER_CONFIG_FILE]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--run-stubtest",
+        action="store_true",
+        help=(
+            "Run stubtest for the selected package(s). Running stubtest may download and execute arbitrary code from PyPI: "
+            "only use this option if you trust the package you are testing."
+        ),
+    )
+    parser.add_argument(
+        "--python-version",
+        default=_PYTHON_VERSION,
+        choices=("3.8", "3.9", "3.10", "3.11", "3.12"),
+        help="Target Python version for the test (default: %(default)s).",
+    )
+    parser.add_argument("path", help="Path of the stub to test in format <folder>/<stub>, from the root of the project.")
+    args = parser.parse_args()
+    path: str = args.path
+    run_stubtest: bool = args.run_stubtest
+    python_version: str = args.python_version
+
+    path_tokens = Path(path).parts
+    if len(path_tokens) != 2:
+        parser.error("'path' argument should be in format <folder>/<stub>.")
+    folder, stub = path_tokens
+    if folder not in {"stdlib", "stubs"}:
+        parser.error("Only the 'stdlib' and 'stubs' folders are supported.")
+    if not os.path.exists(path):
+        parser.error(rf"'path' {path} does not exist.")
+    stubtest_result: subprocess.CompletedProcess[bytes] | None = None
+    pytype_result: subprocess.CompletedProcess[bytes] | None = None
+
+    print("\nRunning pre-commit...")
+    pre_commit_result = subprocess.run(["pre-commit", "run", "--all-files"])
+
+    print("\nRunning check_typeshed_structure.py...")
+    check_structure_result = subprocess.run([sys.executable, "tests/check_typeshed_structure.py"])
+
+    strict_params = _get_strict_params(path)
+    print(f"\nRunning Pyright ({'stricter' if strict_params else 'base' } configs) for Python {python_version}...")
+    pyright_result = subprocess.run(
+        [sys.executable, "tests/pyright_test.py", path, "--pythonversion", python_version, *strict_params],
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    if re.match(_NPX_ERROR_PATTERN, pyright_result.stderr):
+        print(_NPX_ERROR_MESSAGE)
+        pyright_returncode = 0
+        pyright_skipped = True
+    else:
+        print(pyright_result.stderr)
+        pyright_returncode = pyright_result.returncode
+        pyright_skipped = False
+
+    print(f"\nRunning mypy for Python {python_version}...")
+    mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", python_version])
+    # If mypy failed, stubtest will fail without any helpful error
+    if mypy_result.returncode == 0:
+        if folder == "stdlib":
+            print("\nRunning stubtest...")
+            stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
+        else:
+            if run_stubtest:
+                print("\nRunning stubtest...")
+                stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
+            else:
+                print(
+                    colored(
+                        f"\nSkipping stubtest for {stub!r}..."
+                        + "\nNOTE: Running third-party stubtest involves downloading and executing arbitrary code from PyPI."
+                        + f"\nOnly run stubtest if you trust the {stub!r} package.",
+                        "yellow",
+                    )
+                )
+    else:
+        print(colored("\nSkipping stubtest since mypy failed.", "yellow"))
+
+    if sys.platform == "win32":
+        print(colored("\nSkipping pytype on Windows. You can run the test with WSL.", "yellow"))
+    else:
+        print("\nRunning pytype...")
+        pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
+
+    cases_path = test_cases_path(stub if folder == "stubs" else "stdlib")
+    if not cases_path.exists():
+        # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
+        print(colored(f"\nRegression tests: No {TEST_CASES_DIR} folder for {stub!r}!", "green"))
+        pyright_testcases_returncode = 0
+        pyright_testcases_skipped = False
+        regr_test_returncode = 0
+    else:
+        print(f"\nRunning Pyright regression tests for Python {python_version}...")
+        command = [
+            sys.executable,
+            "tests/pyright_test.py",
+            str(cases_path),
+            "--pythonversion",
+            python_version,
+            "-p",
+            _TESTCASES_CONFIG_FILE,
+        ]
+        pyright_testcases_result = subprocess.run(command, stderr=subprocess.PIPE, text=True)
+        if re.match(_NPX_ERROR_PATTERN, pyright_testcases_result.stderr):
+            print(_NPX_ERROR_MESSAGE)
+            pyright_testcases_returncode = 0
+            pyright_testcases_skipped = True
+        else:
+            print(pyright_result.stderr)
+            pyright_testcases_returncode = pyright_testcases_result.returncode
+            pyright_testcases_skipped = False
+
+        print(f"\nRunning mypy regression tests for Python {python_version}...")
+        regr_test_result = subprocess.run(
+            [sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", python_version],
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
+        if "No test cases found" in regr_test_result.stderr:
+            regr_test_returncode = 0
+            print(colored(f"\nNo test cases found for {stub!r}!", "green"))
+        else:
+            regr_test_returncode = regr_test_result.returncode
+            print(regr_test_result.stderr)
+
+    any_failure = any(
+        [
+            pre_commit_result.returncode,
+            check_structure_result.returncode,
+            pyright_returncode,
+            mypy_result.returncode,
+            getattr(stubtest_result, "returncode", 0),
+            getattr(pytype_result, "returncode", 0),
+            pyright_testcases_returncode,
+            regr_test_returncode,
+        ]
+    )
+
+    if any_failure:
+        print(colored("\n\n--- TEST SUMMARY: One or more tests failed. See above for details. ---\n", "red"))
+    else:
+        print(colored("\n\n--- TEST SUMMARY: All tests passed! ---\n", "green"))
+    if pre_commit_result.returncode == 0:
+        print("pre-commit", _SUCCESS)
+    else:
+        print("pre-commit", _FAILED)
+        print(
+            """\
+  Check the output of pre-commit for more details.
+  This could mean that there's a lint failure on your code,
+  but could also just mean that one of the pre-commit tools
+  applied some autofixes. If the latter, you may want to check
+  that the autofixes did sensible things."""
+        )
+    print("Check structure:", _SUCCESS if check_structure_result.returncode == 0 else _FAILED)
+    if pyright_skipped:
+        print("Pyright:", _SKIPPED)
+    else:
+        print("Pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
+    print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
+    if stubtest_result is None:
+        print("stubtest:", _SKIPPED)
+    else:
+        print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
+    if not pytype_result:
+        print("pytype:", _SKIPPED)
+    else:
+        print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
+    if pyright_testcases_skipped:
+        print("Pyright regression tests:", _SKIPPED)
+    else:
+        print("Pyright regression tests:", _SUCCESS if pyright_testcases_returncode == 0 else _FAILED)
+    print("mypy regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
+
+    sys.exit(int(any_failure))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print(colored("\nTests aborted due to KeyboardInterrupt!\n", "red"))
+        sys.exit(1)

tests/stubtest_third_party.py

@@ -13,7 +13,7 @@ from textwrap import dedent
 from typing import NoReturn
 
 from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata
-from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg
+from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg, tests_path
 
 
 def run_stubtest(
@@ -112,10 +112,10 @@ def run_stubtest(
         # "bisect" to see which variables are actually needed.
         stubtest_env = os.environ | {"MYPYPATH": mypypath, "MYPY_FORCE_COLOR": "1"}
 
-        allowlist_path = dist / "@tests/stubtest_allowlist.txt"
+        allowlist_path = tests_path(dist_name) / "stubtest_allowlist.txt"
         if allowlist_path.exists():
             stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
-        platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt"
+        platform_allowlist = tests_path(dist_name) / f"stubtest_allowlist_{sys.platform}.txt"
         if platform_allowlist.exists():
             stubtest_cmd.extend(["--allowlist", str(platform_allowlist)])
 
@@ -271,7 +271,7 @@ def setup_uwsgi_stubtest_command(dist: Path, venv_dir: Path, stubtest_cmd: list[
     so both scripts will be cleaned up after this function
     has been executed.
     """
-    uwsgi_ini = dist / "@tests/uwsgi.ini"
+    uwsgi_ini = tests_path(dist.name) / "uwsgi.ini"
 
     if sys.platform == "win32":
         print_error("uWSGI is not supported on Windows")

tests/utils.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 import re
 import sys
 from collections.abc import Iterable, Mapping
@@ -23,6 +22,8 @@ except ImportError:
 
 PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+STUBS_PATH = Path("stubs")
+
 
 # A backport of functools.cache for Python <3.9
 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI
@@ -109,30 +110,55 @@ VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d
 
 
 # ====================================================================
-# Getting test-case directories from package names
+# Test Directories
 # ====================================================================
 
 
-class PackageInfo(NamedTuple):
+TESTS_DIR: Final = "@tests"
+TEST_CASES_DIR: Final = "test_cases"
+
+
+class DistributionTests(NamedTuple):
     name: str
-    test_case_directory: Path
+    test_cases_path: Path
 
     @property
     def is_stdlib(self) -> bool:
         return self.name == "stdlib"
 
 
-def testcase_dir_from_package_name(package_name: str) -> Path:
-    return Path("stubs", package_name, "@tests/test_cases")
+def distribution_info(distribution_name: str) -> DistributionTests:
+    if distribution_name == "stdlib":
+        return DistributionTests("stdlib", test_cases_path("stdlib"))
+    test_path = test_cases_path(distribution_name)
+    if test_path.is_dir():
+        if not list(test_path.iterdir()):
+            raise RuntimeError(f"{distribution_name!r} has a '{TEST_CASES_DIR}' directory but it is empty!")
+        return DistributionTests(distribution_name, test_path)
+    raise RuntimeError(f"No test cases found for {distribution_name!r}!")
 
 
-def get_all_testcase_directories() -> list[PackageInfo]:
-    testcase_directories: list[PackageInfo] = []
-    for package_name in os.listdir("stubs"):
-        potential_testcase_dir = testcase_dir_from_package_name(package_name)
-        if potential_testcase_dir.is_dir():
-            testcase_directories.append(PackageInfo(package_name, potential_testcase_dir))
-    return [PackageInfo("stdlib", Path("test_cases")), *sorted(testcase_directories)]
+def tests_path(distribution_name: str) -> Path:
+    assert distribution_name != "stdlib"
+    return STUBS_PATH / distribution_name / TESTS_DIR
+
+
+def test_cases_path(distribution_name: str) -> Path:
+    if distribution_name == "stdlib":
+        return Path(TEST_CASES_DIR)
+    else:
+        return tests_path(distribution_name) / TEST_CASES_DIR
+
+
+def get_all_testcase_directories() -> list[DistributionTests]:
+    testcase_directories: list[DistributionTests] = []
+    for distribution_path in STUBS_PATH.iterdir():
+        try:
+            pkg_info = distribution_info(distribution_path.name)
+        except RuntimeError:
+            continue
+        testcase_directories.append(pkg_info)
+    return [distribution_info("stdlib"), *sorted(testcase_directories)]
 
 
 # ====================================================================