From a2d6d5315b93e7d01bc8f298dd9b1f2002064584 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Mon, 14 Oct 2024 15:41:28 +0200 Subject: [PATCH] [stubsabot] Use common functions from ts_utils (#12772) --- lib/ts_utils/metadata.py | 20 ++++++++++-- scripts/stubsabot.py | 69 ++++++++++++++-------------------------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/lib/ts_utils/metadata.py b/lib/ts_utils/metadata.py index 00efeef5f..7fdfc92b5 100644 --- a/lib/ts_utils/metadata.py +++ b/lib/ts_utils/metadata.py @@ -50,6 +50,16 @@ def _get_oldest_supported_python() -> str: return val +def stubs_path(distribution: str) -> Path: + """Return the path to the directory of a third-party distribution.""" + return Path("stubs", distribution) + + +def metadata_path(distribution: str) -> Path: + """Return the path to the METADATA.toml file of a third-party distribution.""" + return stubs_path(distribution) / "METADATA.toml" + + @final @dataclass(frozen=True) class StubtestSettings: @@ -77,7 +87,7 @@ class StubtestSettings: @cache def read_stubtest_settings(distribution: str) -> StubtestSettings: """Return an object describing the stubtest settings for a single stubs distribution.""" - with Path("stubs", distribution, "METADATA.toml").open("rb") as f: + with metadata_path(distribution).open("rb") as f: data: dict[str, object] = tomli.load(f).get("tool", {}).get("stubtest", {}) skip: object = data.get("skip", False) @@ -130,6 +140,7 @@ class StubMetadata: Don't construct instances directly; use the `read_metadata` function. """ + distribution: Annotated[str, "The name of the distribution on PyPI"] version: str requires: Annotated[list[Requirement], "The parsed requirements as listed in METADATA.toml"] extra_description: str | None @@ -142,6 +153,10 @@ class StubMetadata: stubtest_settings: StubtestSettings requires_python: Annotated[Specifier, "Versions of Python supported by the stub package"] + @property + def is_obsolete(self) -> bool: + return self.obsolete_since is not None + _KNOWN_METADATA_FIELDS: Final = frozenset( { @@ -187,7 +202,7 @@ def read_metadata(distribution: str) -> StubMetadata: given in the `requires` field, for example. """ try: - with Path("stubs", distribution, "METADATA.toml").open("rb") as f: + with metadata_path(distribution).open("rb") as f: data: dict[str, object] = tomli.load(f) except FileNotFoundError: raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None @@ -273,6 +288,7 @@ def read_metadata(distribution: str) -> StubMetadata: assert key in tk, f"Unrecognised {tool} key {key!r} for {distribution!r}" return StubMetadata( + distribution=distribution, version=version, requires=requires, extra_description=extra_description, diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 2a87f916d..9cf2c1e58 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -26,10 +26,11 @@ from typing_extensions import Self, TypeAlias import aiohttp import packaging.specifiers import packaging.version -import tomli import tomlkit from termcolor import colored +from ts_utils.metadata import StubMetadata, metadata_path, read_metadata, stubs_path + TYPESHED_OWNER = "python" TYPESHED_API_URL = f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed" @@ -56,27 +57,6 @@ class ActionLevel(enum.IntEnum): everything = 3, "do everything, e.g. open PRs" -@dataclass -class StubInfo: - distribution: str - version_spec: str - upstream_repository: str | None - obsolete: bool - no_longer_updated: bool - - -def read_typeshed_stub_metadata(stub_path: Path) -> StubInfo: - with (stub_path / "METADATA.toml").open("rb") as f: - meta = tomli.load(f) - return StubInfo( - distribution=stub_path.name, - version_spec=meta["version"], - upstream_repository=meta.get("upstream_repository"), - obsolete="obsolete_since" in meta, - no_longer_updated=meta.get("no_longer_updated", False), - ) - - @dataclass class PypiReleaseDownload: distribution: str @@ -141,7 +121,6 @@ async def fetch_pypi_info(distribution: str, session: aiohttp.ClientSession) -> @dataclass class Update: distribution: str - stub_path: Path old_version_spec: str new_version_spec: str links: dict[str, str] @@ -154,7 +133,6 @@ class Update: @dataclass class Obsolete: distribution: str - stub_path: Path obsolete_since_version: str obsolete_since_date: datetime.datetime links: dict[str, str] @@ -304,7 +282,7 @@ class GitHubInfo: tags: list[dict[str, Any]] = field(repr=False) -async def get_github_repo_info(session: aiohttp.ClientSession, stub_info: StubInfo) -> GitHubInfo | None: +async def get_github_repo_info(session: aiohttp.ClientSession, stub_info: StubMetadata) -> GitHubInfo | None: """ If the project represented by `stub_info` is hosted on GitHub, return information regarding the project as it exists on GitHub. @@ -335,7 +313,7 @@ class GitHubDiffInfo(NamedTuple): async def get_diff_info( - session: aiohttp.ClientSession, stub_info: StubInfo, pypi_version: packaging.version.Version + session: aiohttp.ClientSession, stub_info: StubMetadata, pypi_version: packaging.version.Version ) -> GitHubDiffInfo | None: """Return a tuple giving info about the diff between two releases, if possible. @@ -355,7 +333,7 @@ async def get_diff_info( with contextlib.suppress(packaging.version.InvalidVersion): versions_to_tags[packaging.version.Version(tag_name)] = tag_name - curr_specifier = packaging.specifiers.SpecifierSet(f"=={stub_info.version_spec}") + curr_specifier = packaging.specifiers.SpecifierSet(f"=={stub_info.version}") try: new_tag = versions_to_tags[pypi_version] @@ -469,7 +447,7 @@ class DiffAnalysis: async def analyze_diff( - github_repo_path: str, stub_path: Path, old_tag: str, new_tag: str, *, session: aiohttp.ClientSession + github_repo_path: str, distribution: str, old_tag: str, new_tag: str, *, session: aiohttp.ClientSession ) -> DiffAnalysis | None: url = f"https://api.github.com/repos/{github_repo_path}/compare/{old_tag}...{new_tag}" async with session.get(url, headers=get_github_api_headers()) as response: @@ -478,14 +456,15 @@ async def analyze_diff( assert isinstance(json_resp, dict) # https://docs.github.com/en/rest/commits/commits#compare-two-commits py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"] + stub_path = stubs_path(distribution) files_in_typeshed = set(stub_path.rglob("*.pyi")) py_files_stubbed_in_typeshed = [file for file in py_files if (stub_path / f"{file['filename']}i") in files_in_typeshed] return DiffAnalysis(py_files=py_files, py_files_stubbed_in_typeshed=py_files_stubbed_in_typeshed) -async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> Update | NoUpdate | Obsolete: - stub_info = read_typeshed_stub_metadata(stub_path) - if stub_info.obsolete: +async def determine_action(distribution: str, session: aiohttp.ClientSession) -> Update | NoUpdate | Obsolete: + stub_info = read_metadata(distribution) + if stub_info.is_obsolete: return NoUpdate(stub_info.distribution, "obsolete") if stub_info.no_longer_updated: return NoUpdate(stub_info.distribution, "no longer updated") @@ -493,7 +472,7 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U pypi_info = await fetch_pypi_info(stub_info.distribution, session) latest_release = pypi_info.get_latest_release() latest_version = latest_release.version - spec = packaging.specifiers.SpecifierSet(f"=={stub_info.version_spec}") + spec = packaging.specifiers.SpecifierSet(f"=={stub_info.version}") obsolete_since = await find_first_release_with_py_typed(pypi_info, session=session) if obsolete_since is None and latest_version in spec: return NoUpdate(stub_info.distribution, "up to date") @@ -517,7 +496,6 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U if obsolete_since: return Obsolete( stub_info.distribution, - stub_path, obsolete_since_version=str(obsolete_since.version), obsolete_since_date=obsolete_since.upload_date, links=links, @@ -528,7 +506,7 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U else: diff_analysis = await analyze_diff( github_repo_path=diff_info.repo_path, - stub_path=stub_path, + distribution=distribution, old_tag=diff_info.old_tag, new_tag=diff_info.new_tag, session=session, @@ -536,9 +514,8 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U return Update( distribution=stub_info.distribution, - stub_path=stub_path, - old_version_spec=stub_info.version_spec, - new_version_spec=get_updated_version_spec(stub_info.version_spec, latest_version), + old_version_spec=stub_info.version, + new_version_spec=get_updated_version_spec(stub_info.version, latest_version), links=links, diff_analysis=diff_analysis, ) @@ -705,10 +682,10 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession async with _repo_lock: branch_name = f"{BRANCH_PREFIX}/{normalize(update.distribution)}" subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"]) - with open(update.stub_path / "METADATA.toml", "rb") as f: + with metadata_path(update.distribution).open("rb") as f: meta = tomlkit.load(f) meta["version"] = update.new_version_spec - with open(update.stub_path / "METADATA.toml", "w", encoding="UTF-8") as f: + with metadata_path(update.distribution).open("w", encoding="UTF-8") as f: # tomlkit.dump has partially unknown IO type tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType] body = get_update_pr_body(update, meta) @@ -732,12 +709,12 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS async with _repo_lock: branch_name = f"{BRANCH_PREFIX}/{normalize(obsolete.distribution)}" subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"]) - with open(obsolete.stub_path / "METADATA.toml", "rb") as f: + with metadata_path(obsolete.distribution).open("rb") as f: meta = tomlkit.load(f) obs_string = tomlkit.string(obsolete.obsolete_since_version) obs_string.comment(f"Released on {obsolete.obsolete_since_date.date().isoformat()}") meta["obsolete_since"] = obs_string - with open(obsolete.stub_path / "METADATA.toml", "w", encoding="UTF-8") as f: + with metadata_path(obsolete.distribution).open("w", encoding="UTF-8") as f: # tomlkit.dump has partially unknown Mapping type tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType] body = "\n".join(f"{k}: {v}" for k, v in obsolete.links.items()) @@ -774,9 +751,9 @@ async def main() -> None: args = parser.parse_args() if args.distributions: - paths_to_update = [Path("stubs") / distribution for distribution in args.distributions] + dists_to_update = args.distributions else: - paths_to_update = list(Path("stubs").iterdir()) + dists_to_update = [path.name for path in Path("stubs").iterdir()] if args.action_level > ActionLevel.nothing: subprocess.run(["git", "update-index", "--refresh"], capture_output=True) @@ -808,9 +785,9 @@ async def main() -> None: conn = aiohttp.TCPConnector(limit_per_host=10) async with aiohttp.ClientSession(connector=conn) as session: tasks = [ - asyncio.create_task(determine_action(stubs_path, session)) - for stubs_path in paths_to_update - if stubs_path.name not in denylist + asyncio.create_task(determine_action(distribution, session)) + for distribution in dists_to_update + if distribution not in denylist ] action_count = 0