From 4aa4cb9ae2d925e0fc547bc59cb10459637c733e Mon Sep 17 00:00:00 2001 From: Akuli Date: Tue, 28 Dec 2021 12:33:49 +0200 Subject: [PATCH] stubtest_third_party: do not install apt packages in the same script (#6714) --- .github/workflows/stubtest.yml | 4 +++- .github/workflows/tests.yml | 7 ++++++- tests/get_apt_packages.py | 14 ++++++++++++++ tests/stubtest_third_party.py | 30 +++--------------------------- 4 files changed, 26 insertions(+), 29 deletions(-) create mode 100755 tests/get_apt_packages.py diff --git a/.github/workflows/stubtest.yml b/.github/workflows/stubtest.yml index 655b3131b..abb6b2117 100644 --- a/.github/workflows/stubtest.yml +++ b/.github/workflows/stubtest.yml @@ -52,5 +52,7 @@ jobs: python-version: 3.9 - name: Install dependencies run: pip install $(grep tomli== requirements-tests.txt) + - name: Install apt packages + run: sudo apt install -y $(python tests/get_apt_packages.py) - name: Run stubtest - run: python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} --sudo-install-apt + run: python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 459573bf2..be83c23c3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -131,7 +131,12 @@ jobs: STUBS=$(git diff --name-only origin/${{ github.base_ref }} HEAD | egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)) if test -n "$STUBS"; then echo "Testing $STUBS..." - python tests/stubtest_third_party.py --sudo-install-apt $STUBS + APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS) + if test -n "$APT_PACKAGES"; then + echo "Installing apt packages: $APT_PACKAGES" + sudo apt install -y $APT_PACKAGES + fi + python tests/stubtest_third_party.py $STUBS else echo "Nothing to test" fi diff --git a/tests/get_apt_packages.py b/tests/get_apt_packages.py new file mode 100755 index 000000000..20cbed8a8 --- /dev/null +++ b/tests/get_apt_packages.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import os +import sys + +import tomli + +distributions = sys.argv[1:] +if not distributions: + distributions = os.listdir("stubs") + +for distribution in distributions: + with open(f"stubs/{distribution}/METADATA.toml", "rb") as file: + for apt_package in tomli.load(file).get("stubtest_apt_dependencies", []): + print(apt_package) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 3a6565837..0f5a4888b 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -11,7 +11,7 @@ import tempfile import venv from glob import glob from pathlib import Path -from typing import Any, NoReturn +from typing import NoReturn import tomli @@ -22,7 +22,7 @@ def get_mypy_req(): return next(line.strip() for line in f if "mypy" in line) -def run_stubtest(dist: Path, *, install_apt: bool = False) -> bool: +def run_stubtest(dist: Path) -> bool: with open(dist / "METADATA.toml") as f: metadata = dict(tomli.loads(f.read())) @@ -30,9 +30,6 @@ def run_stubtest(dist: Path, *, install_apt: bool = False) -> bool: print(f"Skipping stubtest for {dist.name}\n\n") return True - if not install_apt_packages(dist, metadata, install_apt): - return False - with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) venv.create(venv_dir, with_pip=True, clear=True) @@ -117,31 +114,10 @@ def has_py3_stubs(dist: Path) -> bool: return len(glob(f"{dist}/*.pyi")) > 0 or len(glob(f"{dist}/[!@]*/__init__.pyi")) > 0 -def install_apt_packages(dist: Path, metadata: dict[str, Any], install: bool) -> bool: - apt_packages = metadata.get("stubtest_apt_dependencies", []) - if not apt_packages: - return True - if not install: - print(f"Ensure the following apt packages are installed for {dist.name}: {', '.join(apt_packages)}", file=sys.stderr) - return True - try: - apt_cmd = ["sudo", "apt", "install", "-y", *apt_packages] - print(" ".join(apt_cmd), file=sys.stderr) - subprocess.run(apt_cmd, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print(f"Failed to install APT packages for {dist.name}: {', '.join(apt_packages)}", file=sys.stderr) - print(e.stdout.decode(), file=sys.stderr) - print(e.stderr.decode(), file=sys.stderr) - return False - else: - return True - - def main() -> NoReturn: parser = argparse.ArgumentParser() parser.add_argument("--num-shards", type=int, default=1) parser.add_argument("--shard-index", type=int, default=0) - parser.add_argument("--sudo-install-apt", action="store_true") parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE) args = parser.parse_args() @@ -155,7 +131,7 @@ def main() -> NoReturn: for i, dist in enumerate(dists): if i % args.num_shards != args.shard_index: continue - if not run_stubtest(dist, install_apt=args.sudo_install_apt): + if not run_stubtest(dist): result = 1 sys.exit(result)