From c99e54da3e2236f7589542f667a264f3aa142570 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 9 Feb 2025 16:21:22 -0500 Subject: [PATCH] Cross-platform third-party stubs requirements install script (#13482) --- .github/workflows/tests.yml | 14 ++++------ lib/ts_utils/requirements.py | 28 +++++++++++++++++++ .../install_all_third_party_dependencies.py | 6 ++++ tests/README.md | 4 +-- tests/get_external_stub_requirements.py | 22 ++++----------- tests/get_stubtest_system_requirements.py | 17 ++++------- 6 files changed, 51 insertions(+), 40 deletions(-) create mode 100644 lib/ts_utils/requirements.py create mode 100644 scripts/install_all_third_party_dependencies.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 32cd27c49..1626ebcb5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,11 +47,10 @@ jobs: - run: uv pip install -r requirements-tests.txt --system - name: Install external dependencies for 3rd-party stubs run: | - mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py ) + DEPENDENCIES=$( python tests/get_external_stub_requirements.py ) if [ -n "$DEPENDENCIES" ]; then - echo "Installing packages:" - for DEP in "${DEPENDENCIES[@]}"; do echo " ${DEP}"; done - uv pip install "${DEPENDENCIES[@]}" --system + printf "Installing packages:\n $(echo $DEPENDENCIES | sed 's/ /\n /g')\n" + uv pip install --system $DEPENDENCIES fi - run: uv pip freeze - run: ./tests/pytype_test.py --print-stderr @@ -108,14 +107,13 @@ jobs: run: uv venv .venv - name: Install 3rd-party stub dependencies run: | - mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py ) + DEPENDENCIES=$( python tests/get_external_stub_requirements.py ) if [ -n "$DEPENDENCIES" ]; then - echo "Installing packages:" - for DEP in "${DEPENDENCIES[@]}"; do echo " ${DEP}"; done + printf "Installing packages:\n $(echo $DEPENDENCIES | sed 's/ /\n /g')\n" # TODO: We need to specify the platform here, but the platforms # strings supported by uv are different from the ones supported by # pyright. - uv pip install --python-version ${{ matrix.python-version }} "${DEPENDENCIES[@]}" + uv pip install --python-version ${{ matrix.python-version }} $DEPENDENCIES fi - name: Activate the isolated venv for the rest of the job run: echo "$PWD/.venv/bin" >> $GITHUB_PATH diff --git a/lib/ts_utils/requirements.py b/lib/ts_utils/requirements.py new file mode 100644 index 000000000..e1af0f1ea --- /dev/null +++ b/lib/ts_utils/requirements.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import itertools +import os +import sys +from collections.abc import Iterable + +from packaging.requirements import Requirement + +from ts_utils.metadata import read_dependencies, read_stubtest_settings +from ts_utils.paths import STUBS_PATH + + +def get_external_stub_requirements(distributions: Iterable[str] = ()) -> set[Requirement]: + if not distributions: + distributions = os.listdir(STUBS_PATH) + + return set(itertools.chain.from_iterable([read_dependencies(distribution).external_pkgs for distribution in distributions])) + + +def get_stubtest_system_requirements(distributions: Iterable[str] = (), platform: str = sys.platform) -> list[str]: + if not distributions: + distributions = os.listdir(STUBS_PATH) + + requirements: list[str] = [] + for distribution in distributions: + requirements.extend(read_stubtest_settings(distribution).system_requirements_for_platform(platform)) + return requirements diff --git a/scripts/install_all_third_party_dependencies.py b/scripts/install_all_third_party_dependencies.py new file mode 100644 index 000000000..25e934eb2 --- /dev/null +++ b/scripts/install_all_third_party_dependencies.py @@ -0,0 +1,6 @@ +import subprocess + +from ts_utils.requirements import get_external_stub_requirements + +requirements = get_external_stub_requirements() +subprocess.check_call(("pip", "install", *[str(requirement) for requirement in requirements])) diff --git a/tests/README.md b/tests/README.md index 4f4a8f2ac..a00b17331 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,9 +27,7 @@ You can list or install all of a stubs package's external dependencies using the (.venv3)$ python tests/get_external_stub_requirements.py # List external dependencies for (.venv3)$ python tests/get_external_stub_requirements.py # List external dependencies for and (.venv3)$ python tests/get_external_stub_requirements.py # List external dependencies for all third-party stubs in typeshed -# Install external dependencies for all third-party stubs in typeshed -(.venv3)$ mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py ) -(.venv3)$ if [ -n "$DEPENDENCIES" ]; then pip install "${DEPENDENCIES[@]}"; fi +(.venv3)$ python scripts/install_all_third_party_dependencies.py # Install external dependencies for all third-party stubs in typeshed ``` ## Run all tests for a specific stub diff --git a/tests/get_external_stub_requirements.py b/tests/get_external_stub_requirements.py index 9d015e4e8..177da48ab 100755 --- a/tests/get_external_stub_requirements.py +++ b/tests/get_external_stub_requirements.py @@ -3,23 +3,11 @@ # TODO: It should be possible to specify the Python version and platform # and limit the output to the packages that are compatible with that version # and platform. - -from __future__ import annotations - -import os import sys -from packaging.requirements import Requirement +from ts_utils.requirements import get_external_stub_requirements -from ts_utils.metadata import read_dependencies - -distributions = sys.argv[1:] -if not distributions: - distributions = os.listdir("stubs") - -requirements = set[Requirement]() -for distribution in distributions: - requirements.update(read_dependencies(distribution).external_pkgs) - -for requirement in sorted(requirements, key=str): - print(requirement) +if __name__ == "__main__": + distributions = sys.argv[1:] + for requirement in sorted(get_external_stub_requirements(distributions), key=str): + print(requirement) diff --git a/tests/get_stubtest_system_requirements.py b/tests/get_stubtest_system_requirements.py index 8d81dc6f4..1e01ceb4f 100755 --- a/tests/get_stubtest_system_requirements.py +++ b/tests/get_stubtest_system_requirements.py @@ -1,16 +1,9 @@ #!/usr/bin/env python3 - -import os import sys -from ts_utils.metadata import read_stubtest_settings +from ts_utils.requirements import get_stubtest_system_requirements -platform = sys.platform -distributions = sys.argv[1:] -if not distributions: - distributions = os.listdir("stubs") - -for distribution in distributions: - stubtest_settings = read_stubtest_settings(distribution) - for package in stubtest_settings.system_requirements_for_platform(platform): - print(package) +if __name__ == "__main__": + distributions = sys.argv[1:] + for requirement in get_stubtest_system_requirements(distributions): + print(requirement)