Centralize METADATA.toml parsing in the test suite (#9534)

tests/utils.py

@@ -7,15 +7,13 @@ import re
 import subprocess
 import sys
 import venv
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
 from typing import NamedTuple
 from typing_extensions import Annotated
 
 import pathspec
-import tomli
-from packaging.requirements import Requirement
 
 try:
     from termcolor import colored as colored
@@ -29,9 +27,6 @@ except ImportError:
 # This module is imported by mypy_test.py, which needs to run on 3.7 in CI
 cache = lru_cache(None)
 
-# Used to install system-wide packages for different OS types:
-METADATA_MAPPING = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
-
 
 def strip_comments(text: str) -> str:
     return text.split("#")[0].strip()
@@ -49,81 +44,6 @@ def print_success_msg() -> None:
     print(colored("success", "green"))
 
 
-# ====================================================================
-# Reading dependencies from METADATA.toml files
-# ====================================================================
-
-
-class PackageDependencies(NamedTuple):
-    typeshed_pkgs: tuple[str, ...]
-    external_pkgs: tuple[str, ...]
-
-
-@cache
-def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
-    stub_name_map = {}
-    for typeshed_name in os.listdir("stubs"):
-        with Path("stubs", typeshed_name, "METADATA.toml").open("rb") as f:
-            pypi_name = tomli.load(f).get("stub_distribution", f"types-{typeshed_name}")
-        assert isinstance(pypi_name, str)
-        stub_name_map[pypi_name] = typeshed_name
-    return stub_name_map
-
-
-@cache
-def read_dependencies(distribution: str) -> PackageDependencies:
-    """Read the dependencies listed in a METADATA.toml file for a stubs package.
-
-    Once the dependencies have been read,
-    determine which dependencies are typeshed-internal dependencies,
-    and which dependencies are external (non-types) dependencies.
-    For typeshed dependencies, translate the "dependency name" into the "package name";
-    for external dependencies, leave them as they are in the METADATA.toml file.
-
-    Note that this function may consider things to be typeshed stubs
-    even if they haven't yet been uploaded to PyPI.
-    If a typeshed stub is removed, this function will consider it to be an external dependency.
-    """
-    pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
-    with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
-        dependencies = tomli.load(f).get("requires", [])
-    assert isinstance(dependencies, list)
-    typeshed, external = [], []
-    for dependency in dependencies:
-        assert isinstance(dependency, str)
-        maybe_typeshed_dependency = Requirement(dependency).name
-        if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping:
-            typeshed.append(pypi_name_to_typeshed_name_mapping[maybe_typeshed_dependency])
-        else:
-            # convert to Requirement and then back to str
-            # to make sure that the requirements all have a normalised string representation
-            # (This will also catch any malformed requirements early)
-            external.append(str(Requirement(dependency)))
-    return PackageDependencies(tuple(typeshed), tuple(external))
-
-
-@cache
-def get_recursive_requirements(package_name: str) -> PackageDependencies:
-    """Recursively gather dependencies for a single stubs package.
-
-    For example, if the stubs for `caldav`
-    declare a dependency on typeshed's stubs for `requests`,
-    and the stubs for requests declare a dependency on typeshed's stubs for `urllib3`,
-    `get_recursive_requirements("caldav")` will determine that the stubs for `caldav`
-    have both `requests` and `urllib3` as typeshed-internal dependencies.
-    """
-    typeshed: set[str] = set()
-    external: set[str] = set()
-    non_recursive_requirements = read_dependencies(package_name)
-    typeshed.update(non_recursive_requirements.typeshed_pkgs)
-    external.update(non_recursive_requirements.external_pkgs)
-    for pkg in non_recursive_requirements.typeshed_pkgs:
-        reqs = get_recursive_requirements(pkg)
-        typeshed.update(reqs.typeshed_pkgs)
-        external.update(reqs.external_pkgs)
-    return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external)))
-
-
 # ====================================================================
 # Dynamic venv creation
 # ====================================================================