Improve pyright verification of third-party test cases in CI (#9650)

Co-authored-by: Avasam <samuel.06@hotmail.com>

.github/workflows/tests.yml

@@ -140,26 +140,29 @@ jobs:
         with:
           file: "pyproject.toml"
           field: "tool.typeshed.pyright_version"
-      - uses: jakebailey/pyright-action@v1
+      - name: Run pyright with basic settings on all the stubs
+        uses: jakebailey/pyright-action@v1
+        with:
+          version: ${{ steps.pyright_version.outputs.value }}
+          python-platform: ${{ matrix.python-platform }}
+          python-version: ${{ matrix.python-version }}
+          no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
+      - name: Run pyright with stricter settings on some of the stubs
+        uses: jakebailey/pyright-action@v1
         with:
           version: ${{ steps.pyright_version.outputs.value }}
           python-platform: ${{ matrix.python-platform }}
           python-version: ${{ matrix.python-version }}
           no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
           project: ./pyrightconfig.stricter.json
-      - uses: jakebailey/pyright-action@v1
+      - name: Run pyright on the test cases
+        uses: jakebailey/pyright-action@v1
         with:
           version: ${{ steps.pyright_version.outputs.value }}
           python-platform: ${{ matrix.python-platform }}
           python-version: ${{ matrix.python-version }}
           no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
           project: ./pyrightconfig.testcases.json
-      - uses: jakebailey/pyright-action@v1
-        with:
-          version: ${{ steps.pyright_version.outputs.value }}
-          python-platform: ${{ matrix.python-platform }}
-          python-version: ${{ matrix.python-version }}
-          no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
 
   stub-uploader:
     name: Run the stub_uploader tests

pyrightconfig.json

@@ -5,6 +5,10 @@
         "stdlib",
         "stubs",
     ],
+    "exclude": [
+        // test cases use a custom config file
+        "stubs/**/@tests/test_cases"
+    ],
     "typeCheckingMode": "basic",
     "strictListInference": true,
     "strictDictionaryInference": true,

pyrightconfig.stricter.json

@@ -6,6 +6,8 @@
         "stubs",
     ],
     "exclude": [
+        // test cases use a custom pyrightconfig file
+        "stubs/**/@tests/test_cases",
         "stdlib/distutils/command",
         "stdlib/lib2to3/refactor.pyi",
         "stdlib/_tkinter.pyi",

pyrightconfig.testcases.json

@@ -3,6 +3,7 @@
     "typeshedPath": ".",
     "include": [
         "test_cases",
+        "stubs/**/@tests/test_cases"
     ],
     "typeCheckingMode": "strict",
     // Using unspecific "type ignore" comments in test_cases.

stubs/invoke/@tests/test_cases/check_task.py

@@ -1,4 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
 from __future__ import annotations
 
 from invoke import Context, task

stubs/protobuf/@tests/test_cases/check_struct.py

@@ -1,4 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
 from __future__ import annotations
 
 from google.protobuf.struct_pb2 import ListValue, Struct

stubs/regex/@tests/test_cases/check_finditer.py

@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 from __future__ import annotations
 
 from typing import List

stubs/requests/@tests/test_cases/check_post.py

@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 from __future__ import annotations
 
 from collections.abc import Iterable

tests/check_consistent.py

@@ -65,9 +65,15 @@ def check_stubs() -> None:
         allowed = {"METADATA.toml", "README", "README.md", "README.rst", "@tests"}
         assert_consistent_filetypes(dist, kind=".pyi", allowed=allowed)
 
+        tests_dir = dist / "@tests"
+        if tests_dir.exists() and tests_dir.is_dir():
+            py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir())
+            error_message = "Test-case files must be in an `@tests/test_cases/` directory, not in the `@tests/` directory"
+            assert not py_files_present, error_message
+
 
 def check_test_cases() -> None:
-    for package_name, testcase_dir in get_all_testcase_directories():
+    for _, testcase_dir in get_all_testcase_directories():
         assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True)
         bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"'
         for file in testcase_dir.rglob("*.py"):
@@ -75,14 +81,6 @@ def check_test_cases() -> None:
             with open(file, encoding="UTF-8") as f:
                 lines = {line.strip() for line in f}
             assert "from __future__ import annotations" in lines, "Test-case files should use modern typing syntax where possible"
-            if package_name != "stdlib":
-                pyright_setting_not_enabled_msg = (
-                    f'Third-party test-case file "{file}" must have '
-                    f'"# pyright: reportUnnecessaryTypeIgnoreComment=true" '
-                    f"at the top of the file"
-                )
-                has_pyright_setting_enabled = "# pyright: reportUnnecessaryTypeIgnoreComment=true" in lines
-                assert has_pyright_setting_enabled, pyright_setting_not_enabled_msg
 
 
 def check_no_symlinks() -> None: