Rework how pyright is run in CI to allow for non-types dependencies(#9434)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

.github/workflows/daily.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Run stubtest
         shell: bash
         run: |
-          PACKAGES=$(python tests/get_packages.py)
+          PACKAGES=$(python tests/get_stubtest_system_requirements.py)
 
           if [ "${{ runner.os }}" = "Linux" ]; then
             if [ -n "$PACKAGES" ]; then

.github/workflows/stubtest_third_party.yml

@@ -58,7 +58,7 @@ jobs:
 
           if [ -n "$STUBS" ]; then
             echo "Testing $STUBS..."
-            PACKAGES=$(python tests/get_packages.py $STUBS)
+            PACKAGES=$(python tests/get_stubtest_system_requirements.py $STUBS)
 
             if [ "${{ runner.os }}" = "Linux" ]; then
               if [ -n "$PACKAGES" ]; then

.github/workflows/tests.yml

@@ -112,6 +112,20 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
+      - name: Install 3rd-party dependencies for stubs packages
+        run: |
+          pip install -r requirements-tests.txt
+          DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
+          if [ -n "$DEPENDENCIES" ]; then
+            echo "Installing packages: $DEPENDENCIES"
+            pip install $DEPENDENCIES
+          fi
+      - run: pip freeze --all
       - name: Get pyright version
         uses: SebRollen/toml-action@v1.0.2
         id: pyright_version

tests/README.md

@@ -78,6 +78,19 @@ checks that would typically fail on incomplete stubs (such as `Unknown` checks).
 In typeshed's CI, pyright is run with these configuration settings on a subset of
 the stubs in typeshed (including the standard library).
 
+In order for `pyright_test` to work correctly, some third-party stubs may require
+dependencies external to typeshed to be installed in your virtual environment
+prior to running the test.
+You can list or install all of a stubs package's external dependencies using the following script:
+```bash
+(.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub>  # List external dependencies for <third_party_stub>
+(.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub1> <third_party_stub2>  # List external dependencies for <third_party_stub1> and <third_party_stub2>
+(.venv3)$ python tests/get_external_stub_requirements.py  # List external dependencies for all third-party stubs in typeshed
+# Install external dependencies for all third-party stubs in typeshed
+(.venv3)$ DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
+(.venv3)$ if [ -n "$DEPENDENCIES" ]; then pip install $DEPENDENCIES; fi
+```
+
 ## regr\_test.py
 
 This test runs mypy against the test cases for typeshed's stdlib and third-party

tests/get_external_stub_requirements.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+from utils import read_dependencies
+
+distributions = sys.argv[1:]
+if not distributions:
+    distributions = os.listdir("stubs")
+
+requirements = set[str]()
+for distribution in distributions:
+    requirements.update(read_dependencies(distribution).external_pkgs)
+
+for requirement in sorted(requirements):
+    print(requirement)

tests/utils.py

@@ -7,10 +7,10 @@ import re
 import subprocess
 import sys
 import venv
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from functools import cache
 from pathlib import Path
-from typing import Iterable, NamedTuple
+from typing import NamedTuple
 from typing_extensions import Annotated
 
 import pathspec  # type: ignore[import]