Add infrastructure allowing for test cases for third-party stubs (#8700)

- Move the logic for running mypy on the test cases from `tests/mypy_test.py` to a separate script, `tests/regr_test.py`.
- Add the necessary logic in order to be able to have test cases for third-party stubs.
- Move logic common to `tests/mypy_test.py` and `tests/regr_test.py` into `tests/colors.py`, and rename `tests/colors.py` to `tests/utils.py`.
- Add a new check to `tests/check_consistent.py`, to enforce the use of `# pyright: reportUnnecessaryTypeIgnoreComment=true` comments in third-party test cases. These are essential if we want to have our tests against false-negatives work with pyright.
- Update the relevant documentation to account for the new test file.
- Add a new job to the `tests.yml` GitHub workflow, to run the new test in CI.
- Add a simple proof-of-concept test case for `requests`, as a regression test for #7998.

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Co-authored-by: Sebastian Rittau <srittau@rittau.biz>

tests/check_consistent.py

@@ -15,6 +15,7 @@ import yaml
 from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from utils import get_all_testcase_directories
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
@@ -58,10 +59,21 @@ def check_stubs() -> None:
 
 
 def check_test_cases() -> None:
-    assert_consistent_filetypes(Path("test_cases"), kind=".py", allowed={"README.md"})
-    bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "check_"; got "{}"'
-    for file in Path("test_cases").rglob("*.py"):
-        assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
+    for package_name, testcase_dir in get_all_testcase_directories():
+        assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"})
+        bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"'
+        for file in testcase_dir.rglob("*.py"):
+            assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
+            if package_name != "stdlib":
+                with open(file) as f:
+                    lines = {line.strip() for line in f}
+                pyright_setting_not_enabled_msg = (
+                    f'Third-party test-case file "{file}" must have '
+                    f'"# pyright: reportUnnecessaryTypeIgnoreComment=true" '
+                    f"at the top of the file"
+                )
+                has_pyright_setting_enabled = "# pyright: reportUnnecessaryTypeIgnoreComment=true" in lines
+                assert has_pyright_setting_enabled, pyright_setting_not_enabled_msg
 
 
 def check_no_symlinks() -> None: