Cross-platform third-party stubs requirements install script (#13482)

.github/workflows/tests.yml

@@ -47,11 +47,10 @@ jobs:
       - run: uv pip install -r requirements-tests.txt --system
       - name: Install external dependencies for 3rd-party stubs
         run: |
-          mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py )
+          DEPENDENCIES=$( python tests/get_external_stub_requirements.py )
           if [ -n "$DEPENDENCIES" ]; then
-              echo "Installing packages:"
-              for DEP in "${DEPENDENCIES[@]}"; do echo "  ${DEP}"; done
-              uv pip install "${DEPENDENCIES[@]}" --system
+              printf "Installing packages:\n  $(echo $DEPENDENCIES | sed 's/ /\n  /g')\n"
+              uv pip install --system $DEPENDENCIES
           fi
       - run: uv pip freeze
       - run: ./tests/pytype_test.py --print-stderr
@@ -108,14 +107,13 @@ jobs:
         run: uv venv .venv
       - name: Install 3rd-party stub dependencies
         run: |
-          mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py )
+          DEPENDENCIES=$( python tests/get_external_stub_requirements.py )
           if [ -n "$DEPENDENCIES" ]; then
-              echo "Installing packages:"
-              for DEP in "${DEPENDENCIES[@]}"; do echo "  ${DEP}"; done
+              printf "Installing packages:\n  $(echo $DEPENDENCIES | sed 's/ /\n  /g')\n"
               # TODO: We need to specify the platform here, but the platforms
               # strings supported by uv are different from the ones supported by
               # pyright.
-              uv pip install --python-version ${{ matrix.python-version }} "${DEPENDENCIES[@]}"
+              uv pip install --python-version ${{ matrix.python-version }} $DEPENDENCIES
           fi
       - name: Activate the isolated venv for the rest of the job
         run: echo "$PWD/.venv/bin" >> $GITHUB_PATH

lib/ts_utils/requirements.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import itertools
+import os
+import sys
+from collections.abc import Iterable
+
+from packaging.requirements import Requirement
+
+from ts_utils.metadata import read_dependencies, read_stubtest_settings
+from ts_utils.paths import STUBS_PATH
+
+
+def get_external_stub_requirements(distributions: Iterable[str] = ()) -> set[Requirement]:
+    if not distributions:
+        distributions = os.listdir(STUBS_PATH)
+
+    return set(itertools.chain.from_iterable([read_dependencies(distribution).external_pkgs for distribution in distributions]))
+
+
+def get_stubtest_system_requirements(distributions: Iterable[str] = (), platform: str = sys.platform) -> list[str]:
+    if not distributions:
+        distributions = os.listdir(STUBS_PATH)
+
+    requirements: list[str] = []
+    for distribution in distributions:
+        requirements.extend(read_stubtest_settings(distribution).system_requirements_for_platform(platform))
+    return requirements

scripts/install_all_third_party_dependencies.py

@@ -0,0 +1,6 @@
+import subprocess
+
+from ts_utils.requirements import get_external_stub_requirements
+
+requirements = get_external_stub_requirements()
+subprocess.check_call(("pip", "install", *[str(requirement) for requirement in requirements]))

tests/README.md

@@ -27,9 +27,7 @@ You can list or install all of a stubs package's external dependencies using the
 (.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub>  # List external dependencies for <third_party_stub>
 (.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub1> <third_party_stub2>  # List external dependencies for <third_party_stub1> and <third_party_stub2>
 (.venv3)$ python tests/get_external_stub_requirements.py  # List external dependencies for all third-party stubs in typeshed
-# Install external dependencies for all third-party stubs in typeshed
-(.venv3)$ mapfile -t DEPENDENCIES < <( python tests/get_external_stub_requirements.py )
-(.venv3)$ if [ -n "$DEPENDENCIES" ]; then pip install "${DEPENDENCIES[@]}"; fi
+(.venv3)$ python scripts/install_all_third_party_dependencies.py  # Install external dependencies for all third-party stubs in typeshed
 ```
 
 ## Run all tests for a specific stub

tests/get_external_stub_requirements.py

@@ -3,23 +3,11 @@
 # TODO: It should be possible to specify the Python version and platform
 # and limit the output to the packages that are compatible with that version
 # and platform.
-
-from __future__ import annotations
-
-import os
 import sys
 
-from packaging.requirements import Requirement
+from ts_utils.requirements import get_external_stub_requirements
 
-from ts_utils.metadata import read_dependencies
-
-distributions = sys.argv[1:]
-if not distributions:
-    distributions = os.listdir("stubs")
-
-requirements = set[Requirement]()
-for distribution in distributions:
-    requirements.update(read_dependencies(distribution).external_pkgs)
-
-for requirement in sorted(requirements, key=str):
-    print(requirement)
+if __name__ == "__main__":
+    distributions = sys.argv[1:]
+    for requirement in sorted(get_external_stub_requirements(distributions), key=str):
+        print(requirement)

tests/get_stubtest_system_requirements.py

@@ -1,16 +1,9 @@
 #!/usr/bin/env python3
-
-import os
 import sys
 
-from ts_utils.metadata import read_stubtest_settings
+from ts_utils.requirements import get_stubtest_system_requirements
 
-platform = sys.platform
-distributions = sys.argv[1:]
-if not distributions:
-    distributions = os.listdir("stubs")
-
-for distribution in distributions:
-    stubtest_settings = read_stubtest_settings(distribution)
-    for package in stubtest_settings.system_requirements_for_platform(platform):
-        print(package)
+if __name__ == "__main__":
+    distributions = sys.argv[1:]
+    for requirement in get_stubtest_system_requirements(distributions):
+        print(requirement)