Move test_cases to stdlib/@tests/test_cases (#11865)

pyrightconfig.json

@@ -7,7 +7,7 @@
     ],
     "exclude": [
         // test cases use a custom config file
-        "stubs/**/@tests/test_cases"
+        "**/@tests/test_cases",
     ],
     "typeCheckingMode": "strict",
     // Allowed in base settings for incomplete stubs, checked in stricter settings

pyrightconfig.stricter.json

@@ -7,7 +7,7 @@
     ],
     "exclude": [
         // test cases use a custom pyrightconfig file
-        "stubs/**/@tests/test_cases",
+        "**/@tests/test_cases",
         "stdlib/distutils/command",
         "stdlib/distutils/dist.pyi",
         "stdlib/importlib/readers.pyi",

pyrightconfig.testcases.json

@@ -2,8 +2,7 @@
     "$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
     "typeshedPath": ".",
     "include": [
-        "test_cases",
-        "stubs/**/@tests/test_cases"
+        "**/@tests/test_cases",
     ],
     "typeCheckingMode": "strict",
     // Extra strict settings

tests/README.md

@@ -102,7 +102,8 @@ the stubs in typeshed (including the standard library).
 ## regr\_test.py
 
 This test runs mypy against the test cases for typeshed's stdlib and third-party
-stubs. See [the README in the `test_cases` directory](../test_cases/README.md)
+stubs. See [the REGRESSION.md document](./REGRESSION.md)
+in this directory
 for more information about what
 these test cases are for and how they work. Run `python tests/regr_test.py --help`
 for information on the various configuration options.

tests/REGRESSION.md

@@ -1,32 +1,27 @@
 ## Regression tests for typeshed
 
-This directory contains code samples that act as a regression test for
-typeshed's stdlib stubs.
+Regression tests for the standard library stubs can be found in the
+`stdlib/@tests/test_cases` directory. Stubs for third-party libraries that do
+have test cases can be found in `@tests/test_cases` subdirectories for each
+stubs package. For example, the test cases for `requests` can be found in the
+`stubs/requests/@tests/test_cases` directory.
 
-**This directory should *only* contain test cases for functions and classes which
+**Regression test cases should only be written for functions and classes which
 are known to have caused problems in the past, where the stubs are difficult to
 get right.** 100% test coverage for typeshed is neither necessary nor
 desirable, as it would lead to code duplication. Moreover, typeshed has
 multiple other mechanisms for spotting errors in the stubs.
 
-### Where are the third-party test cases?
-
-Not all third-party stubs packages in typeshed have test cases, and not all of
-them need test cases. For those that do have test cases, however, the samples
-can be found in `@tests/test_cases` subdirectories for each stubs package. For
-example, the test cases for `requests` can be found in the
-`stubs/requests/@tests/test_cases` directory.
-
 ### The purpose of these tests
 
 Different test cases in this directory serve different purposes. For some stubs in
 typeshed, the type annotations are complex enough that it's useful to have
 sanity checks that test whether a type checker understands the intent of
 the annotations correctly. Examples of tests like these are
-`stdlib/builtins/check_pow.py` and `stdlib/asyncio/check_gather.py`.
+`builtins/check_pow.py` and `asyncio/check_gather.py`.
 
-Other test cases, such as the samples for `ExitStack` in `stdlib/check_contextlib.py`
-and the samples for `LogRecord` in `stdlib/check_logging.py`, do not relate to
+Other test cases, such as the samples for `ExitStack` in `check_contextlib.py`
+and the samples for `LogRecord` in `check_logging.py`, do not relate to
 stubs where the annotations are particularly complex, but they *do* relate to
 stubs where decisions have been taken that might be slightly unusual. These
 test cases serve a different purpose: to check that type checkers do not emit
@@ -74,10 +69,10 @@ Use the same top-level name for the module / package you would like to test.
 Use the `check_${thing}.py` naming pattern for individual test files.
 
 By default, test cases go into a file with the same name as the stub file, prefixed with `check_`.
-For example: `stdlib/check_contextlib.py`.
+For example: `check_contextlib.py`.
 
 If that file becomes too big, we instead create a directory with files named after individual objects being tested.
-For example: `stdlib/builtins/check_dict.py`.
+For example: `builtins/check_dict.py`.
 
 ### Differences to the rest of typeshed
 
@@ -114,7 +109,7 @@ with a specific Python version passed on the command line to the `tests/regr_tes
 To mark a test-case file as being skippable on lower versions of Python,
 append `-py3*` to the filename.
 For example, if `foo` is a stdlib feature that's new in Python 3.11,
-test cases for `foo` should be put in a file named `test_cases/stdlib/check_foo-py311.py`.
+test cases for `foo` should be put in a file named `check_foo-py311.py`.
 This means that mypy will only run the test case
 if `--python-version 3.11`, `--python-version 3.12`, etc.
 is passed on the command line to `tests/regr_test.py`,
@@ -123,4 +118,4 @@ is passed on the command line.
 
 However, `if sys.version_info >= (3, target):` is still required for `pyright`
 in the test file itself.
-Example: [`check_exception_group-py311.py`](https://github.com/python/typeshed/blob/main/test_cases/stdlib/builtins/check_exception_group-py311.py)
+Example: [`check_exception_group-py311.py`](../stdlib/@tests/test_cases/builtins/check_exception_group-py311.py)

tests/check_typeshed_structure.py

@@ -15,6 +15,7 @@ from pathlib import Path
 from parse_metadata import read_metadata
 from utils import (
     REQS_FILE,
+    STDLIB_PATH,
     TEST_CASES_DIR,
     TESTS_DIR,
     VERSIONS_RE,
@@ -59,7 +60,8 @@ def assert_consistent_filetypes(
 
 def check_stdlib() -> None:
     """Check that the stdlib directory contains only the correct files."""
-    assert_consistent_filetypes(Path("stdlib"), kind=".pyi", allowed={"_typeshed/README.md", "VERSIONS"})
+    assert_consistent_filetypes(STDLIB_PATH, kind=".pyi", allowed={"_typeshed/README.md", "VERSIONS", TESTS_DIR})
+    check_tests_dir(tests_path("stdlib"))
 
 
 def check_stubs() -> None:
@@ -81,11 +83,13 @@ def check_stubs() -> None:
 
         tests_dir = tests_path(dist.name)
         if tests_dir.exists() and tests_dir.is_dir():
-            py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir())
-            error_message = (
-                f"Test-case files must be in an `{TESTS_DIR}/{TEST_CASES_DIR}` directory, not in the `{TESTS_DIR}` directory"
-            )
-            assert not py_files_present, error_message
+            check_tests_dir(tests_dir)
+
+
+def check_tests_dir(tests_dir: Path) -> None:
+    py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir())
+    error_message = f"Test-case files must be in an `{TESTS_DIR}/{TEST_CASES_DIR}` directory, not in the `{TESTS_DIR}` directory"
+    assert not py_files_present, error_message
 
 
 def check_distutils() -> None:
@@ -146,7 +150,7 @@ def check_versions_file() -> None:
 
 def _find_stdlib_modules() -> set[str]:
     modules = set[str]()
-    for path, _, files in os.walk("stdlib"):
+    for path, _, files in os.walk(STDLIB_PATH):
         for filename in files:
             base_module = ".".join(os.path.normpath(path).split(os.sep)[1:])
             if filename == "__init__.pyi":

tests/mypy_test.py

@@ -29,6 +29,7 @@ import tomli
 from parse_metadata import PackageDependencies, get_recursive_requirements, read_metadata
 from utils import (
     PYTHON_VERSION,
+    TESTS_DIR,
     VERSIONS_RE as VERSION_LINE_RE,
     colored,
     get_gitignore_spec,
@@ -366,7 +367,7 @@ def test_stdlib(args: TestConfig) -> TestResult:
     stdlib = Path("stdlib")
     supported_versions = parse_versions(stdlib / "VERSIONS")
     for name in os.listdir(stdlib):
-        if name == "VERSIONS" or name.startswith("."):
+        if name in ("VERSIONS", TESTS_DIR) or name.startswith("."):
             continue
         module = Path(name).stem
         module_min_version, module_max_version = supported_versions[module]

tests/utils.py

@@ -22,6 +22,7 @@ except ImportError:
 
 PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+STDLIB_PATH = Path("stdlib")
 STUBS_PATH = Path("stubs")
 
 
@@ -139,15 +140,14 @@ def distribution_info(distribution_name: str) -> DistributionTests:
 
 
 def tests_path(distribution_name: str) -> Path:
-    assert distribution_name != "stdlib"
-    return STUBS_PATH / distribution_name / TESTS_DIR
+    if distribution_name == "stdlib":
+        return STDLIB_PATH / TESTS_DIR
+    else:
+        return STUBS_PATH / distribution_name / TESTS_DIR
 
 
 def test_cases_path(distribution_name: str) -> Path:
-    if distribution_name == "stdlib":
-        return Path(TEST_CASES_DIR)
-    else:
-        return tests_path(distribution_name) / TEST_CASES_DIR
+    return tests_path(distribution_name) / TEST_CASES_DIR
 
 
 def get_all_testcase_directories() -> list[DistributionTests]: