Add the ability to run third-party stubtest on Windows or MacOS when needed (#8923)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

.github/workflows/daily.yml

@@ -44,9 +44,10 @@ jobs:
   stubtest-third-party:
     name: Check third party stubs with stubtest
     if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         shard-index: [0, 1, 2, 3]
       fail-fast: false
     steps:
@@ -56,12 +57,17 @@ jobs:
           python-version: "3.9"
       - name: Install dependencies
         run: pip install -r requirements-tests.txt
-      - name: Install apt packages
-        run: |
-          sudo apt update
-          sudo apt install -y $(python tests/get_apt_packages.py)
       - name: Run stubtest
-        run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+        shell: bash
+        run: |
+          if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+            sudo apt update
+            sudo apt install -y $(python tests/get_packages.py)
+
+            xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          else
+            python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          fi
 
   stub-uploader:
     name: Run the stub_uploader tests

.github/workflows/stubtest_stdlib.yml

@@ -1,4 +1,4 @@
-name: Run stubtest
+name: Stdlib stubtest
 
 on:
   workflow_dispatch:

.github/workflows/stubtest_third_party.yml

@@ -0,0 +1,70 @@
+name: Third-party stubtest
+
+on:
+  pull_request:
+    paths-ignore:
+      - '**/*.md'
+      - 'scripts/**'
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  stubtest-third-party:
+    name: Check third party stubs with stubtest
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install dependencies
+        run: pip install -r requirements-tests.txt
+      - name: Run stubtest
+        shell: bash
+        run: |
+          STUBS=$(
+            git diff --name-only origin/${{ github.base_ref }} HEAD |
+            # Use the daily.yml workflow to run stubtest on all third party stubs
+            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
+          )
+
+          if [ -n "$STUBS" ]; then
+            echo "Testing $STUBS..."
+            PACKAGES=$(python tests/get_packages.py $STUBS)
+
+            if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+              if [ -n "$PACKAGES" ]; then
+                echo "Installing apt packages: $PACKAGES"
+                sudo apt update && sudo apt install -y $PACKAGES
+              fi
+              xvfb-run python tests/stubtest_third_party.py $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "macos-latest" ]; then
+              # Could install brew packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "windows-latest" ]; then
+              # Could install choco packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py $STUBS
+            fi
+          else
+            echo "Nothing to test"
+          fi

.github/workflows/tests.yml

@@ -146,36 +146,3 @@ jobs:
           cd stub_uploader
           pip install -r requirements.txt
           python -m pytest tests
-
-  stubtest-third-party:
-    name: Check third party stubs with stubtest
-    runs-on: ubuntu-20.04
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      - name: Install dependencies
-        run: pip install -r requirements-tests.txt
-      - name: Run stubtest
-        run: |
-          STUBS=$(
-            git diff --name-only origin/${{ github.base_ref }} HEAD |
-            # Uncomment the following to (very slowly) run on all third party stubs:
-            # git ls-files |
-            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
-          )
-          if test -n "$STUBS"; then
-            echo "Testing $STUBS..."
-            APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS)
-            if test -n "$APT_PACKAGES"; then
-              echo "Installing apt packages: $APT_PACKAGES"
-              sudo apt update && sudo apt install -y $APT_PACKAGES
-            fi
-            xvfb-run python tests/stubtest_third_party.py $STUBS
-          else
-            echo "Nothing to test"
-          fi

CONTRIBUTING.md

@@ -192,6 +192,11 @@ This has the following keys:
 * `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages
   that need to be installed for stubtest to run successfully. These are
   usually packages needed to pip install the implementation distribution.
+* `platforms` (default: `["linux"]`): A list of OSes on which to run stubtest.
+  Can contain `win32`, `linux`, and `darwin` values.
+  If not specified, stubtest is run only on `linux`.
+  Only add extra OSes to the test
+  if there are platform-specific branches in a stubs package.
 
 The format of all `METADATA.toml` files can be checked by running
 `python3 ./tests/check_consistent.py`.

stubs/D3DShot/METADATA.toml

@@ -2,6 +2,7 @@ version = "0.1.*"
 requires = ["types-Pillow"]
 
 [tool.stubtest]
-# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI.
-# See #8660
+# TODO: figure out how to run stubtest for this package
+# (the package pins Pillow in a problematic way)
 skip = true
+platforms = ["win32"]

stubs/JACK-Client/METADATA.toml

@@ -1,4 +1,5 @@
 version = "0.5.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["libjack-dev"]

stubs/pyaudio/METADATA.toml

@@ -1,4 +1,5 @@
 version = "0.2.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["portaudio19-dev"]

stubs/pycurl/METADATA.toml

@@ -1,4 +1,5 @@
 version = "7.45.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["libcurl4-openssl-dev"]

stubs/pywin32/METADATA.toml

@@ -1,6 +1,5 @@
 version = "305.*"
+
 [tool.stubtest]
+platforms = ["win32"]
 ignore_missing_stub = false
-# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI.
-# See #8660
-skip = true

tests/check_consistent.py

@@ -18,7 +18,7 @@ from packaging.version import Version
 from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
-tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
+tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub", "platforms"}}
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
 
 

tests/get_apt_packages.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-
-import tomli
-
-distributions = sys.argv[1:]
-if not distributions:
-    distributions = os.listdir("stubs")
-
-for distribution in distributions:
-    with open(f"stubs/{distribution}/METADATA.toml", "rb") as file:
-        for apt_package in tomli.load(file).get("tool", {}).get("stubtest", {}).get("apt_dependencies", []):
-            print(apt_package)

tests/get_packages.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+import tomli
+
+platform = sys.platform
+distributions = sys.argv[1:]
+if not distributions:
+    distributions = os.listdir("stubs")
+
+metadata_mapping = {
+    "linux": "apt_dependencies",
+    # We could add others here if we run into stubs that need it:
+    # "darwin": "brew_dependencies",
+    # "win32": "choco_dependencies",
+}
+
+if platform in metadata_mapping:
+    for distribution in distributions:
+        with open(f"stubs/{distribution}/METADATA.toml", "rb") as file:
+            for package in tomli.load(file).get("tool", {}).get("stubtest", {}).get(metadata_mapping[platform], []):
+                print(package)

tests/stubtest_third_party.py

@@ -34,6 +34,11 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
         print(colored("skipping", "yellow"))
         return True
 
+    platforms_to_test = stubtest_meta.get("platforms", ["linux"])
+    if sys.platform not in platforms_to_test:
+        print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow"))
+        return True
+
     with tempfile.TemporaryDirectory() as tmp:
         venv_dir = Path(tmp)
         venv.create(venv_dir, with_pip=True, clear=True)
@@ -57,8 +62,8 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
         # If @tests/requirements-stubtest.txt exists, run "pip install" on it.
         req_path = dist / "@tests" / "requirements-stubtest.txt"
         if req_path.exists():
+            pip_cmd = [pip_exe, "install", "-r", str(req_path)]
             try:
-                pip_cmd = [pip_exe, "install", "-r", str(req_path)]
                 subprocess.run(pip_cmd, check=True, capture_output=True)
             except subprocess.CalledProcessError as e:
                 print_command_failure("Failed to install requirements", e)
@@ -102,6 +107,9 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
         allowlist_path = dist / "@tests/stubtest_allowlist.txt"
         if allowlist_path.exists():
             stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
+        platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt"
+        if platform_allowlist.exists():
+            stubtest_cmd.extend(["--allowlist", str(platform_allowlist)])
 
         try:
             subprocess.run(stubtest_cmd, env=stubtest_env, check=True, capture_output=True)