Add infrastructure allowing for test cases for third-party stubs (#8700)

- Move the logic for running mypy on the test cases from `tests/mypy_test.py` to a separate script, `tests/regr_test.py`.
- Add the necessary logic in order to be able to have test cases for third-party stubs.
- Move logic common to `tests/mypy_test.py` and `tests/regr_test.py` into `tests/colors.py`, and rename `tests/colors.py` to `tests/utils.py`.
- Add a new check to `tests/check_consistent.py`, to enforce the use of `# pyright: reportUnnecessaryTypeIgnoreComment=true` comments in third-party test cases. These are essential if we want to have our tests against false-negatives work with pyright.
- Update the relevant documentation to account for the new test file.
- Add a new job to the `tests.yml` GitHub workflow, to run the new test in CI.
- Add a simple proof-of-concept test case for `requests`, as a regression test for #7998.

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Co-authored-by: Sebastian Rittau <srittau@rittau.biz>

tests/utils.py

@@ -0,0 +1,81 @@
+"""Utilities that are imported by multiple scripts in the tests directory."""
+
+import os
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING, NamedTuple
+
+import tomli
+
+# ====================================================================
+# Some simple hacks so we don't have to install types-termcolor in CI,
+# and so that tests can be run locally without termcolor installed,
+# if desired
+# ====================================================================
+
+if TYPE_CHECKING:
+
+    def colored(__str: str, __style: str) -> str:
+        ...
+
+else:
+    try:
+        from termcolor import colored
+    except ImportError:
+
+        def colored(s: str, _: str) -> str:
+            return s
+
+
+def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None:
+    error_split = error.split("\n")
+    old, new = fix_path
+    for line in error_split[:-1]:
+        print(colored(line.replace(old, new), "red"))
+    print(colored(error_split[-1], "red"), end=end)
+
+
+def print_success_msg() -> None:
+    print(colored("success", "green"))
+
+
+# ====================================================================
+# Reading dependencies from METADATA.toml files
+# ====================================================================
+
+
+@cache
+def read_dependencies(distribution: str) -> tuple[str, ...]:
+    with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
+        data = tomli.load(f)
+    requires = data.get("requires", [])
+    assert isinstance(requires, list)
+    dependencies = []
+    for dependency in requires:
+        assert isinstance(dependency, str)
+        assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}"
+        dependencies.append(dependency[6:].split("<")[0])
+    return tuple(dependencies)
+
+
+# ====================================================================
+# Getting test-case directories from package names
+# ====================================================================
+
+
+class PackageInfo(NamedTuple):
+    name: str
+    test_case_directory: Path
+
+
+def testcase_dir_from_package_name(package_name: str) -> Path:
+    return Path("stubs", package_name, "@tests/test_cases")
+
+
+def get_all_testcase_directories() -> list[PackageInfo]:
+    testcase_directories = [PackageInfo("stdlib", Path("test_cases"))]
+    for package_name in os.listdir("stubs"):
+        potential_testcase_dir = testcase_dir_from_package_name(package_name)
+        if potential_testcase_dir.is_dir():
+            testcase_directories.append(PackageInfo(package_name, potential_testcase_dir))
+    return sorted(testcase_directories)