Add the ability to run third-party stubtest on Windows or MacOS when needed (#8923)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

.github/workflows/daily.yml

@@ -44,9 +44,10 @@ jobs:
   stubtest-third-party:
     name: Check third party stubs with stubtest
     if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         shard-index: [0, 1, 2, 3]
       fail-fast: false
     steps:
@@ -56,12 +57,17 @@ jobs:
           python-version: "3.9"
       - name: Install dependencies
         run: pip install -r requirements-tests.txt
-      - name: Install apt packages
-        run: |
-          sudo apt update
-          sudo apt install -y $(python tests/get_apt_packages.py)
       - name: Run stubtest
-        run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+        shell: bash
+        run: |
+          if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+            sudo apt update
+            sudo apt install -y $(python tests/get_packages.py)
+
+            xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          else
+            python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          fi
 
   stub-uploader:
     name: Run the stub_uploader tests

.github/workflows/stubtest_stdlib.yml

@@ -1,4 +1,4 @@
-name: Run stubtest
+name: Stdlib stubtest
 
 on:
   workflow_dispatch:

.github/workflows/stubtest_third_party.yml

@@ -0,0 +1,70 @@
+name: Third-party stubtest
+
+on:
+  pull_request:
+    paths-ignore:
+      - '**/*.md'
+      - 'scripts/**'
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  stubtest-third-party:
+    name: Check third party stubs with stubtest
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install dependencies
+        run: pip install -r requirements-tests.txt
+      - name: Run stubtest
+        shell: bash
+        run: |
+          STUBS=$(
+            git diff --name-only origin/${{ github.base_ref }} HEAD |
+            # Use the daily.yml workflow to run stubtest on all third party stubs
+            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
+          )
+
+          if [ -n "$STUBS" ]; then
+            echo "Testing $STUBS..."
+            PACKAGES=$(python tests/get_packages.py $STUBS)
+
+            if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+              if [ -n "$PACKAGES" ]; then
+                echo "Installing apt packages: $PACKAGES"
+                sudo apt update && sudo apt install -y $PACKAGES
+              fi
+              xvfb-run python tests/stubtest_third_party.py $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "macos-latest" ]; then
+              # Could install brew packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "windows-latest" ]; then
+              # Could install choco packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py $STUBS
+            fi
+          else
+            echo "Nothing to test"
+          fi

.github/workflows/tests.yml

@@ -146,36 +146,3 @@ jobs:
           cd stub_uploader
           pip install -r requirements.txt
           python -m pytest tests
-
-  stubtest-third-party:
-    name: Check third party stubs with stubtest
-    runs-on: ubuntu-20.04
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      - name: Install dependencies
-        run: pip install -r requirements-tests.txt
-      - name: Run stubtest
-        run: |
-          STUBS=$(
-            git diff --name-only origin/${{ github.base_ref }} HEAD |
-            # Uncomment the following to (very slowly) run on all third party stubs:
-            # git ls-files |
-            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
-          )
-          if test -n "$STUBS"; then
-            echo "Testing $STUBS..."
-            APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS)
-            if test -n "$APT_PACKAGES"; then
-              echo "Installing apt packages: $APT_PACKAGES"
-              sudo apt update && sudo apt install -y $APT_PACKAGES
-            fi
-            xvfb-run python tests/stubtest_third_party.py $STUBS
-          else
-            echo "Nothing to test"
-          fi