check_consistent.py: ignore .gitignored files (#8804)

requirements-tests.txt

@@ -7,6 +7,7 @@ flake8-pyi==22.8.2      # must match .pre-commit-config.yaml
 isort==5.10.1           # must match .pre-commit-config.yaml
 mypy==0.981
 packaging==21.3
+pathspec
 pycln==2.1.1            # must match .pre-commit-config.yaml
 pyyaml==6.0
 pytype==2022.08.30; platform_system != "Windows"

tests/check_consistent.py

@@ -15,7 +15,7 @@ import yaml
 from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
-from utils import VERSIONS_RE, get_all_testcase_directories, strip_comments
+from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
@@ -26,8 +26,11 @@ def assert_consistent_filetypes(directory: Path, *, kind: str, allowed: set[str]
     """Check that given directory contains only valid Python files of a certain kind."""
     allowed_paths = {Path(f) for f in allowed}
     contents = list(directory.iterdir())
+    gitignore_spec = get_gitignore_spec()
     while contents:
         entry = contents.pop()
+        if spec_matches_path(gitignore_spec, entry):
+            continue
         if entry.relative_to(directory) in allowed_paths:
             # Note if a subdirectory is allowed, we will not check its contents
             continue
@@ -45,7 +48,10 @@ def check_stdlib() -> None:
 
 
 def check_stubs() -> None:
+    gitignore_spec = get_gitignore_spec()
     for dist in Path("stubs").iterdir():
+        if spec_matches_path(gitignore_spec, dist):
+            continue
         assert dist.is_dir(), f"Only directories allowed in stubs, got {dist}"
 
         valid_dist_name = "^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$"  # courtesy of PEP 426

tests/mypy_test.py

@@ -21,7 +21,16 @@ if TYPE_CHECKING:
 from typing_extensions import Annotated, TypeAlias
 
 import tomli
-from utils import VERSIONS_RE as VERSION_LINE_RE, colored, print_error, print_success_msg, read_dependencies, strip_comments
+from utils import (
+    VERSIONS_RE as VERSION_LINE_RE,
+    colored,
+    get_gitignore_spec,
+    print_error,
+    print_success_msg,
+    read_dependencies,
+    spec_matches_path,
+    strip_comments,
+)
 
 SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
@@ -305,11 +314,6 @@ def test_third_party_distribution(distribution: str, args: TestConfig) -> TestRe
     return TestResults(code, len(files))
 
 
-def is_probably_stubs_folder(distribution: str, distribution_path: Path) -> bool:
-    """Validate that `dist_path` is a folder containing stubs"""
-    return distribution != ".mypy_cache" and distribution_path.is_dir()
-
-
 def test_stdlib(code: int, args: TestConfig) -> TestResults:
     seen = {"builtins", "typing"}  # Always ignore these.
     files: list[Path] = []
@@ -336,11 +340,12 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults:
     print("Testing third-party packages...")
     print("Running mypy " + " ".join(get_mypy_flags(args, "/tmp/...")))
     files_checked = 0
+    gitignore_spec = get_gitignore_spec()
 
     for distribution in sorted(os.listdir("stubs")):
         distribution_path = Path("stubs", distribution)
 
-        if not is_probably_stubs_folder(distribution, distribution_path):
+        if spec_matches_path(gitignore_spec, distribution_path):
             continue
 
         this_code, checked = test_third_party_distribution(distribution, args)

tests/utils.py

@@ -6,6 +6,7 @@ from functools import cache
 from pathlib import Path
 from typing import NamedTuple
 
+import pathspec  # type: ignore[import]
 import tomli
 
 
@@ -81,3 +82,21 @@ def get_all_testcase_directories() -> list[PackageInfo]:
         if potential_testcase_dir.is_dir():
             testcase_directories.append(PackageInfo(package_name, potential_testcase_dir))
     return sorted(testcase_directories)
+
+
+# ====================================================================
+# Parsing .gitignore
+# ====================================================================
+
+
+@cache
+def get_gitignore_spec() -> pathspec.PathSpec:
+    with open(".gitignore") as f:
+        return pathspec.PathSpec.from_lines("gitwildmatch", f.readlines())
+
+
+def spec_matches_path(spec: pathspec.PathSpec, path: Path) -> bool:
+    normalized_path = path.as_posix()
+    if path.is_dir():
+        normalized_path += "/"
+    return spec.match_file(normalized_path)  # type: ignore[no-any-return]