VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2025-12-07 12:44:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bundle path handling in ts_utils.paths (#12805)

Browse Source
This commit is contained in:
Sebastian Rittau
2024-10-17 08:16:10 +02:00
committed by GitHub
parent 36fb63ebc8
commit 2d0f6d8277
10 changed files with 81 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/ts_utils/metadata.py
View File

@@ -5,7 +5,6 @@
from __future__ import annotations
import os
import re
import urllib.parse
from collections.abc import Mapping
@@ -18,6 +17,7 @@ import tomli
from packaging.requirements import Requirement
from packaging.specifiers import Specifier
from .paths import PYPROJECT_PATH, STUBS_PATH, distribution_path
from .utils import cache
__all__ = [
@@ -43,20 +43,15 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]:
@cache
def _get_oldest_supported_python() -> str:
with open("pyproject.toml", "rb") as config:
with PYPROJECT_PATH.open("rb") as config:
val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"]
assert type(val) is str
return val
def stubs_path(distribution: str) -> Path:
"""Return the path to the directory of a third-party distribution."""
return Path("stubs", distribution)
def metadata_path(distribution: str) -> Path:
"""Return the path to the METADATA.toml file of a third-party distribution."""
return stubs_path(distribution) / "METADATA.toml"
return distribution_path(distribution) / "METADATA.toml"
@final
@@ -317,7 +312,7 @@ class PackageDependencies(NamedTuple):
@cache
def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
return {read_metadata(typeshed_name).stub_distribution: typeshed_name for typeshed_name in os.listdir("stubs")}
return {read_metadata(dir.name).stub_distribution: dir.name for dir in STUBS_PATH.iterdir()}
@cache

39
lib/ts_utils/paths.py Normal file
View File

@@ -0,0 +1,39 @@
from pathlib import Path
from typing import Final
# TODO: Use base path relative to this file. Currently, ts_utils gets
# installed into the user's virtual env, so we can't determine the path
# to typeshed. Installing ts_utils editable would solve that, see
# https://github.com/python/typeshed/pull/12806.
TS_BASE_PATH: Final = Path("")
STDLIB_PATH: Final = TS_BASE_PATH / "stdlib"
STUBS_PATH: Final = TS_BASE_PATH / "stubs"
PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml"
REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt"
TESTS_DIR: Final = "@tests"
TEST_CASES_DIR: Final = "test_cases"
def distribution_path(distribution_name: str) -> Path:
"""Return the path to the directory of a third-party distribution."""
return STUBS_PATH / distribution_name
def tests_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return STDLIB_PATH / TESTS_DIR
else:
return STUBS_PATH / distribution_name / TESTS_DIR
def test_cases_path(distribution_name: str) -> Path:
return tests_path(distribution_name) / TEST_CASES_DIR
def allowlists_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return tests_path("stdlib") / "stubtest_allowlists"
else:
return tests_path(distribution_name)

35
lib/ts_utils/utils.py
View File

@@ -21,10 +21,9 @@ except ImportError:
return text
PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
from .paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path
STDLIB_PATH = Path("stdlib")
STUBS_PATH = Path("stubs")
PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
# A backport of functools.cache for Python <3.9
@@ -90,16 +89,14 @@ def venv_python(venv_dir: Path) -> Path:
# ====================================================================
REQS_FILE: Final = "requirements-tests.txt"
@cache
def parse_requirements() -> Mapping[str, Requirement]:
"""Return a dictionary of requirements from the requirements file."""
with open(REQS_FILE, encoding="UTF-8") as requirements_file:
with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file:
stripped_lines = map(strip_comments, requirements_file)
requirements = map(Requirement, filter(None, stripped_lines))
stripped_more = [li for li in stripped_lines if not li.startswith("-")]
requirements = map(Requirement, filter(None, stripped_more))
return {requirement.name: requirement for requirement in requirements}
@@ -155,10 +152,6 @@ def _parse_version(v_str: str) -> tuple[int, int]:
# ====================================================================
TESTS_DIR: Final = "@tests"
TEST_CASES_DIR: Final = "test_cases"
class DistributionTests(NamedTuple):
name: str
test_cases_path: Path
@@ -179,17 +172,6 @@ def distribution_info(distribution_name: str) -> DistributionTests:
raise RuntimeError(f"No test cases found for {distribution_name!r}!")
def tests_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return STDLIB_PATH / TESTS_DIR
else:
return STUBS_PATH / distribution_name / TESTS_DIR
def test_cases_path(distribution_name: str) -> Path:
return tests_path(distribution_name) / TEST_CASES_DIR
def get_all_testcase_directories() -> list[DistributionTests]:
testcase_directories: list[DistributionTests] = []
for distribution_path in STUBS_PATH.iterdir():
@@ -201,13 +183,6 @@ def get_all_testcase_directories() -> list[DistributionTests]:
return [distribution_info("stdlib"), *sorted(testcase_directories)]
def allowlists_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return tests_path("stdlib") / "stubtest_allowlists"
else:
return tests_path(distribution_name)
def allowlists(distribution_name: str) -> list[str]:
prefix = "" if distribution_name == "stdlib" else "stubtest_allowlist_"
version_id = f"py{sys.version_info.major}{sys.version_info.minor}"

7
scripts/stubsabot.py
View File

@@ -29,7 +29,8 @@ import tomlkit
from packaging.specifiers import Specifier
from termcolor import colored
from ts_utils.metadata import StubMetadata, metadata_path, read_metadata, stubs_path
from ts_utils.metadata import StubMetadata, metadata_path, read_metadata
from ts_utils.paths import STUBS_PATH, distribution_path
TYPESHED_OWNER = "python"
TYPESHED_API_URL = f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed"
@@ -463,7 +464,7 @@ async def analyze_diff(
assert isinstance(json_resp, dict)
# https://docs.github.com/en/rest/commits/commits#compare-two-commits
py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"]
stub_path = stubs_path(distribution)
stub_path = distribution_path(distribution)
files_in_typeshed = set(stub_path.rglob("*.pyi"))
py_files_stubbed_in_typeshed = [file for file in py_files if (stub_path / f"{file['filename']}i") in files_in_typeshed]
return DiffAnalysis(py_files=py_files, py_files_stubbed_in_typeshed=py_files_stubbed_in_typeshed)
@@ -759,7 +760,7 @@ async def main() -> None:
if args.distributions:
dists_to_update = args.distributions
else:
dists_to_update = [path.name for path in Path("stubs").iterdir()]
dists_to_update = [path.name for path in STUBS_PATH.iterdir()]
if args.action_level > ActionLevel.nothing:
subprocess.run(["git", "update-index", "--refresh"], capture_output=True)

18
tests/check_typeshed_structure.py
View File

@@ -13,17 +13,13 @@ import sys
from pathlib import Path
from ts_utils.metadata import read_metadata
from ts_utils.paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path
from ts_utils.utils import (
REQS_FILE,
STDLIB_PATH,
TEST_CASES_DIR,
TESTS_DIR,
get_all_testcase_directories,
get_gitignore_spec,
parse_requirements,
parse_stdlib_versions_file,
spec_matches_path,
tests_path,
)
extension_descriptions = {".pyi": "stub", ".py": ".py"}
@@ -66,7 +62,7 @@ def check_stdlib() -> None:
def check_stubs() -> None:
"""Check that the stubs directory contains only the correct files."""
gitignore_spec = get_gitignore_spec()
for dist in Path("stubs").iterdir():
for dist in STUBS_PATH.iterdir():
if spec_matches_path(gitignore_spec, dist):
continue
assert dist.is_dir(), f"Only directories allowed in stubs, got {dist}"
@@ -97,8 +93,8 @@ def check_distutils() -> None:
def all_relative_paths_in_directory(path: Path) -> set[Path]:
return {pyi.relative_to(path) for pyi in path.rglob("*.pyi")}
all_setuptools_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "setuptools", "_distutils"))
all_distutils_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "distutils"))
all_setuptools_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "setuptools" / "_distutils")
all_distutils_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "distutils")
assert all_setuptools_files and all_distutils_files, "Looks like this test might be out of date!"
extra_files = all_setuptools_files - all_distutils_files
joined = "\n".join(f" * {f}" for f in extra_files)
@@ -164,10 +160,10 @@ def check_requirement_pins() -> None:
"""Check that type checkers and linters are pinned to an exact version."""
requirements = parse_requirements()
for package in linters:
assert package in requirements, f"type checker/linter '{package}' not found in {REQS_FILE}"
assert package in requirements, f"type checker/linter '{package}' not found in {REQUIREMENTS_PATH.name}"
spec = requirements[package].specifier
assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQS_FILE}"
msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQS_FILE}"
assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQUIREMENTS_PATH.name}"
msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQUIREMENTS_PATH.name}"
assert str(spec).startswith("=="), msg

25
tests/mypy_test.py
View File

@@ -22,11 +22,10 @@ from typing_extensions import Annotated, TypeAlias
import tomli
from packaging.requirements import Requirement
from ts_utils.metadata import PackageDependencies, get_recursive_requirements, read_metadata
from ts_utils.metadata import PackageDependencies, get_recursive_requirements, metadata_path, read_metadata
from ts_utils.paths import STDLIB_PATH, STUBS_PATH, TESTS_DIR, TS_BASE_PATH, distribution_path
from ts_utils.utils import (
PYTHON_VERSION,
STDLIB_PATH,
TESTS_DIR,
colored,
get_gitignore_spec,
get_mypy_req,
@@ -47,7 +46,7 @@ except ImportError:
SUPPORTED_VERSIONS = ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
DIRECTORIES_TO_TEST = [STDLIB_PATH, STUBS_PATH]
VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
@@ -170,7 +169,7 @@ class MypyDistConf(NamedTuple):
def add_configuration(configurations: list[MypyDistConf], distribution: str) -> None:
with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
with metadata_path(distribution).open("rb") as f:
data = tomli.load(f)
# TODO: This could be added to ts_utils.metadata, but is currently unused
@@ -229,7 +228,7 @@ def run_mypy(
"--platform",
args.platform,
"--custom-typeshed-dir",
str(Path(__file__).parent.parent),
str(TS_BASE_PATH),
"--strict",
# Stub completion is checked by pyright (--allow-*-defs)
"--allow-untyped-defs",
@@ -283,7 +282,7 @@ def add_third_party_files(
return
seen_dists.add(distribution)
seen_dists.update(r.name for r in typeshed_reqs)
root = Path("stubs", distribution)
root = distribution_path(distribution)
for name in os.listdir(root):
if name.startswith("."):
continue
@@ -319,7 +318,7 @@ def test_third_party_distribution(
print_error("no files found")
sys.exit(1)
mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists)
mypypath = os.pathsep.join(str(distribution_path(dist)) for dist in seen_dists)
if args.verbose:
print(colored(f"\nMYPYPATH={mypypath}", "blue"))
result = run_mypy(
@@ -516,16 +515,12 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary:
distributions_to_check: dict[str, PackageDependencies] = {}
for distribution in sorted(os.listdir("stubs")):
distribution_path = Path("stubs", distribution)
dist_path = distribution_path(distribution)
if spec_matches_path(gitignore_spec, distribution_path):
if spec_matches_path(gitignore_spec, dist_path):
continue
if (
distribution_path in args.filter
or Path("stubs") in args.filter
or any(distribution_path in path.parents for path in args.filter)
):
if dist_path in args.filter or STUBS_PATH in args.filter or any(dist_path in path.parents for path in args.filter):
metadata = read_metadata(distribution)
if not metadata.requires_python.contains(PYTHON_VERSION):
msg = (

8
tests/regr_test.py
View File

@@ -22,9 +22,9 @@ from pathlib import Path
from typing_extensions import TypeAlias
from ts_utils.metadata import get_recursive_requirements, read_metadata
from ts_utils.paths import STDLIB_PATH, TEST_CASES_DIR, TS_BASE_PATH, distribution_path
from ts_utils.utils import (
PYTHON_VERSION,
TEST_CASES_DIR,
DistributionTests,
colored,
distribution_info,
@@ -134,14 +134,14 @@ def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Ver
# that has only the required stubs copied over.
new_typeshed = tempdir / TYPESHED
new_typeshed.mkdir()
shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
shutil.copytree(STDLIB_PATH, new_typeshed / "stdlib")
requirements = get_recursive_requirements(package.name)
# mypy refuses to consider a directory a "valid typeshed directory"
# unless there's a stubs/mypy-extensions path inside it,
# so add that to the list of stubs to copy over to the new directory
typeshed_requirements = [r.name for r in requirements.typeshed_pkgs]
for requirement in {package.name, *typeshed_requirements, "mypy-extensions"}:
shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
shutil.copytree(distribution_path(requirement), new_typeshed / "stubs" / requirement)
if requirements.external_pkgs:
venv_location = str(tempdir / VENV_DIR)
@@ -190,7 +190,7 @@ def run_testcases(
if package.is_stdlib:
python_exe = sys.executable
custom_typeshed = Path(__file__).parent.parent
custom_typeshed = TS_BASE_PATH
flags.append("--no-site-packages")
else:
custom_typeshed = tempdir / TYPESHED

3
tests/runtests.py
View File

@@ -10,7 +10,8 @@ import sys
from importlib.util import find_spec
from pathlib import Path
from ts_utils.utils import TEST_CASES_DIR, colored, test_cases_path
from ts_utils.paths import TEST_CASES_DIR, test_cases_path
from ts_utils.utils import colored
_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
_TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"

5
tests/stubtest_stdlib.py
View File

@@ -13,7 +13,8 @@ import subprocess
import sys
from pathlib import Path
from ts_utils.utils import allowlist_stubtest_arguments, allowlists_path
from ts_utils.paths import TS_BASE_PATH, allowlists_path
from ts_utils.utils import allowlist_stubtest_arguments
def run_stubtest(typeshed_dir: Path) -> int:
@@ -57,4 +58,4 @@ def run_stubtest(typeshed_dir: Path) -> int:
if __name__ == "__main__":
sys.exit(run_stubtest(typeshed_dir=Path(".")))
sys.exit(run_stubtest(typeshed_dir=TS_BASE_PATH))

8
tests/stubtest_third_party.py
View File

@@ -14,17 +14,16 @@ from textwrap import dedent
from typing import NoReturn
from ts_utils.metadata import NoSuchStubError, get_recursive_requirements, read_metadata
from ts_utils.paths import STUBS_PATH, allowlists_path, tests_path
from ts_utils.utils import (
PYTHON_VERSION,
allowlist_stubtest_arguments,
allowlists_path,
colored,
get_mypy_req,
print_divider,
print_error,
print_info,
print_success_msg,
tests_path,
)
@@ -386,11 +385,10 @@ def main() -> NoReturn:
parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE)
args = parser.parse_args()
typeshed_dir = Path(".").resolve()
if len(args.dists) == 0:
dists = sorted((typeshed_dir / "stubs").iterdir())
dists = sorted(STUBS_PATH.iterdir())
else:
dists = [typeshed_dir / "stubs" / d for d in args.dists]
dists = [STUBS_PATH / d for d in args.dists]
result = 0
for i, dist in enumerate(dists):