Unify test directory handling (#11864)

Previously, handling of test directories (`@tests` and `test_cases`) was
distributed over multiple files and redundant. This unifies the handling
in the `utils` module. This also fixes some instances where "package"
was used instead of "distribution". And in a few instances paths were
joined by using a `/` in a string, which is incompatible with Windows.

Also move `runtests.py` from `scripts` to `tests`. This is required so that
we can import `utils`, but it's also arguably the better fit. The only
mention of the script is actually in the `tests/README.md` file.

Helps with #11762.

tests/README.md

@@ -36,7 +36,7 @@ You can list or install all of a stubs package's external dependencies using the
 
 Run using:
 ```bash
-(.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
+(.venv3)$ python3 tests/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
 
 This script will run all tests below for a specific typeshed directory. If a
@@ -46,7 +46,7 @@ be selected. A summary of the results will be printed to the terminal.
 You must provide a single argument which is a path to the stubs to test, like
 so: `stdlib/os` or `stubs/requests`.
 
-Run `python scripts/runtests.py --help` for information on the various configuration options
+Run `python tests/runtests.py --help` for information on the various configuration options
 for this script. Note that if you use the `--run-stubtest` flag with the stdlib stubs,
 whether or not the test passes will depend on the exact version of Python
 you're using, as well as various other details regarding your local environment.

tests/check_typeshed_structure.py

@@ -15,12 +15,15 @@ from pathlib import Path
 from parse_metadata import read_metadata
 from utils import (
     REQS_FILE,
+    TEST_CASES_DIR,
+    TESTS_DIR,
     VERSIONS_RE,
     get_all_testcase_directories,
     get_gitignore_spec,
     parse_requirements,
     spec_matches_path,
     strip_comments,
+    tests_path,
 )
 
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
@@ -73,13 +76,15 @@ def check_stubs() -> None:
         ), f"Directory name must be a valid distribution name: {dist}"
         assert not dist.name.startswith("types-"), f"Directory name not allowed to start with 'types-': {dist}"
 
-        allowed = {"METADATA.toml", "README", "README.md", "README.rst", "@tests"}
+        allowed = {"METADATA.toml", "README", "README.md", "README.rst", TESTS_DIR}
         assert_consistent_filetypes(dist, kind=".pyi", allowed=allowed)
 
-        tests_dir = dist / "@tests"
+        tests_dir = tests_path(dist.name)
         if tests_dir.exists() and tests_dir.is_dir():
             py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir())
-            error_message = "Test-case files must be in an `@tests/test_cases/` directory, not in the `@tests/` directory"
+            error_message = (
+                f"Test-case files must be in an `{TESTS_DIR}/{TEST_CASES_DIR}` directory, not in the `{TESTS_DIR}` directory"
+            )
             assert not py_files_present, error_message
 
 
@@ -101,7 +106,7 @@ def check_test_cases() -> None:
     """Check that the test_cases directory contains only the correct files."""
     for _, testcase_dir in get_all_testcase_directories():
         assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True)
-        bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"'
+        bad_test_case_filename = f'Files in a `{TEST_CASES_DIR}` directory must have names starting with "check_"; got "{{}}"'
         for file in testcase_dir.rglob("*.py"):
             assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
 

tests/regr_test.py

@@ -24,18 +24,18 @@ from typing_extensions import TypeAlias
 from parse_metadata import get_recursive_requirements, read_metadata
 from utils import (
     PYTHON_VERSION,
-    PackageInfo,
+    TEST_CASES_DIR,
+    DistributionTests,
     colored,
+    distribution_info,
     get_all_testcase_directories,
     get_mypy_req,
     print_error,
-    testcase_dir_from_package_name,
     venv_python,
 )
 
 ReturnCode: TypeAlias = int
 
-TEST_CASES = "test_cases"
 VENV_DIR = ".venv"
 TYPESHED = "typeshed"
 
@@ -43,17 +43,13 @@ SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"]
 SUPPORTED_VERSIONS = ["3.12", "3.11", "3.10", "3.9", "3.8"]
 
 
-def package_with_test_cases(package_name: str) -> PackageInfo:
-    """Helper function for argument-parsing"""
+def distribution_with_test_cases(distribution_name: str) -> DistributionTests:
+    """Helper function for argument-parsing."""
 
-    if package_name == "stdlib":
-        return PackageInfo("stdlib", Path(TEST_CASES))
-    test_case_dir = testcase_dir_from_package_name(package_name)
-    if test_case_dir.is_dir():
-        if not os.listdir(test_case_dir):
-            raise argparse.ArgumentTypeError(f"{package_name!r} has a 'test_cases' directory but it is empty!")
-        return PackageInfo(package_name, test_case_dir)
-    raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!")
+    try:
+        return distribution_info(distribution_name)
+    except RuntimeError as exc:
+        raise argparse.ArgumentTypeError(str(exc)) from exc
 
 
 class Verbosity(IntEnum):
@@ -65,7 +61,7 @@ class Verbosity(IntEnum):
 parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs")
 parser.add_argument(
     "packages_to_test",
-    type=package_with_test_cases,
+    type=distribution_with_test_cases,
     nargs="*",
     action="extend",
     help=(
@@ -118,13 +114,13 @@ def verbose_log(msg: str) -> None:
     _PRINT_QUEUE.put(colored(msg, "blue"))
 
 
-def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity) -> None:
+def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Verbosity) -> None:
     if verbosity is verbosity.VERBOSE:
         verbose_log(f"{package.name}: Setting up testcase dir in {tempdir}")
     # --warn-unused-ignores doesn't work for files inside typeshed.
     # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
     # and run the test cases inside of that.
-    shutil.copytree(package.test_case_directory, tempdir / TEST_CASES)
+    shutil.copytree(package.test_cases_path, tempdir / TEST_CASES_DIR)
     if package.is_stdlib:
         return
 
@@ -163,10 +159,10 @@ def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity
 
 
 def run_testcases(
-    package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
+    package: DistributionTests, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
 ) -> subprocess.CompletedProcess[str]:
     env_vars = dict(os.environ)
-    new_test_case_dir = tempdir / TEST_CASES
+    new_test_case_dir = tempdir / TEST_CASES_DIR
 
     # "--enable-error-code ignore-without-code" is purposefully omitted.
     # See https://github.com/python/typeshed/pull/8083
@@ -239,14 +235,16 @@ class Result:
         if self.code:
             print(f"{self.command_run}:", end=" ")
             print_error("FAILURE\n")
-            replacements = (str(self.tempdir / TEST_CASES), str(self.test_case_dir))
+            replacements = (str(self.tempdir / TEST_CASES_DIR), str(self.test_case_dir))
             if self.stderr:
                 print_error(self.stderr, fix_path=replacements)
             if self.stdout:
                 print_error(self.stdout, fix_path=replacements)
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path) -> Result:
+def test_testcase_directory(
+    package: DistributionTests, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path
+) -> Result:
     msg = f"mypy --platform {platform} --python-version {version} on the "
     msg += "standard library test cases" if package.is_stdlib else f"test cases for {package.name!r}"
     if verbosity > Verbosity.QUIET:
@@ -258,7 +256,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, *
         command_run=msg,
         stderr=proc_info.stderr,
         stdout=proc_info.stdout,
-        test_case_dir=package.test_case_directory,
+        test_case_dir=package.test_cases_path,
         tempdir=tempdir,
     )
 
@@ -278,13 +276,13 @@ def print_queued_messages(ev: threading.Event) -> None:
 
 def concurrently_run_testcases(
     stack: ExitStack,
-    testcase_directories: list[PackageInfo],
+    testcase_directories: list[DistributionTests],
     verbosity: Verbosity,
     platforms_to_test: list[str],
     versions_to_test: list[str],
 ) -> list[Result]:
     packageinfo_to_tempdir = {
-        package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories
+        distribution_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for distribution_info in testcase_directories
     }
     to_do: list[Callable[[], Result]] = []
     for testcase_dir, tempdir in packageinfo_to_tempdir.items():

tests/runtests.py

@@ -10,6 +10,8 @@ import sys
 from pathlib import Path
 from typing import Any
 
+from utils import TEST_CASES_DIR, test_cases_path
+
 try:
     from termcolor import colored  # pyright: ignore[reportAssignmentType]
 except ImportError:
@@ -20,7 +22,6 @@ except ImportError:
 
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
 _TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"
-_TESTCASES = "test_cases"
 _NPX_ERROR_PATTERN = r"error (runn|find)ing npx"
 _NPX_ERROR_MESSAGE = colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow")
 _SUCCESS = colored("Success", "green")
@@ -132,10 +133,10 @@ def main() -> None:
         print("\nRunning pytype...")
         pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
 
-    test_cases_path = Path(path) / "@tests" / _TESTCASES if folder == "stubs" else Path(_TESTCASES)
-    if not test_cases_path.exists():
+    cases_path = test_cases_path(stub if folder == "stubs" else "stdlib")
+    if not cases_path.exists():
         # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
-        print(colored(f"\nRegression tests: No {_TESTCASES} folder for {stub!r}!", "green"))
+        print(colored(f"\nRegression tests: No {TEST_CASES_DIR} folder for {stub!r}!", "green"))
         pyright_testcases_returncode = 0
         pyright_testcases_skipped = False
         regr_test_returncode = 0
@@ -144,7 +145,7 @@ def main() -> None:
         command = [
             sys.executable,
             "tests/pyright_test.py",
-            str(test_cases_path),
+            str(cases_path),
             "--pythonversion",
             python_version,
             "-p",

tests/stubtest_third_party.py

@@ -13,7 +13,7 @@ from textwrap import dedent
 from typing import NoReturn
 
 from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata
-from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg
+from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg, tests_path
 
 
 def run_stubtest(
@@ -112,10 +112,10 @@ def run_stubtest(
         # "bisect" to see which variables are actually needed.
         stubtest_env = os.environ | {"MYPYPATH": mypypath, "MYPY_FORCE_COLOR": "1"}
 
-        allowlist_path = dist / "@tests/stubtest_allowlist.txt"
+        allowlist_path = tests_path(dist_name) / "stubtest_allowlist.txt"
         if allowlist_path.exists():
             stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
-        platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt"
+        platform_allowlist = tests_path(dist_name) / f"stubtest_allowlist_{sys.platform}.txt"
         if platform_allowlist.exists():
             stubtest_cmd.extend(["--allowlist", str(platform_allowlist)])
 
@@ -271,7 +271,7 @@ def setup_uwsgi_stubtest_command(dist: Path, venv_dir: Path, stubtest_cmd: list[
     so both scripts will be cleaned up after this function
     has been executed.
     """
-    uwsgi_ini = dist / "@tests/uwsgi.ini"
+    uwsgi_ini = tests_path(dist.name) / "uwsgi.ini"
 
     if sys.platform == "win32":
         print_error("uWSGI is not supported on Windows")

tests/utils.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 import re
 import sys
 from collections.abc import Iterable, Mapping
@@ -23,6 +22,8 @@ except ImportError:
 
 PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+STUBS_PATH = Path("stubs")
+
 
 # A backport of functools.cache for Python <3.9
 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI
@@ -109,30 +110,55 @@ VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d
 
 
 # ====================================================================
-# Getting test-case directories from package names
+# Test Directories
 # ====================================================================
 
 
-class PackageInfo(NamedTuple):
+TESTS_DIR: Final = "@tests"
+TEST_CASES_DIR: Final = "test_cases"
+
+
+class DistributionTests(NamedTuple):
     name: str
-    test_case_directory: Path
+    test_cases_path: Path
 
     @property
     def is_stdlib(self) -> bool:
         return self.name == "stdlib"
 
 
-def testcase_dir_from_package_name(package_name: str) -> Path:
-    return Path("stubs", package_name, "@tests/test_cases")
+def distribution_info(distribution_name: str) -> DistributionTests:
+    if distribution_name == "stdlib":
+        return DistributionTests("stdlib", test_cases_path("stdlib"))
+    test_path = test_cases_path(distribution_name)
+    if test_path.is_dir():
+        if not list(test_path.iterdir()):
+            raise RuntimeError(f"{distribution_name!r} has a '{TEST_CASES_DIR}' directory but it is empty!")
+        return DistributionTests(distribution_name, test_path)
+    raise RuntimeError(f"No test cases found for {distribution_name!r}!")
 
 
-def get_all_testcase_directories() -> list[PackageInfo]:
-    testcase_directories: list[PackageInfo] = []
-    for package_name in os.listdir("stubs"):
-        potential_testcase_dir = testcase_dir_from_package_name(package_name)
-        if potential_testcase_dir.is_dir():
-            testcase_directories.append(PackageInfo(package_name, potential_testcase_dir))
-    return [PackageInfo("stdlib", Path("test_cases")), *sorted(testcase_directories)]
+def tests_path(distribution_name: str) -> Path:
+    assert distribution_name != "stdlib"
+    return STUBS_PATH / distribution_name / TESTS_DIR
+
+
+def test_cases_path(distribution_name: str) -> Path:
+    if distribution_name == "stdlib":
+        return Path(TEST_CASES_DIR)
+    else:
+        return tests_path(distribution_name) / TEST_CASES_DIR
+
+
+def get_all_testcase_directories() -> list[DistributionTests]:
+    testcase_directories: list[DistributionTests] = []
+    for distribution_path in STUBS_PATH.iterdir():
+        try:
+            pkg_info = distribution_info(distribution_path.name)
+        except RuntimeError:
+            continue
+        testcase_directories.append(pkg_info)
+    return [distribution_info("stdlib"), *sorted(testcase_directories)]
 
 
 # ====================================================================