Rework stubtest tests (#6318)

.github/workflows/stubtest.yml

@@ -0,0 +1,52 @@
+name: Run stubtest on all stubs
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+jobs:
+  stubtest-stdlib:
+    name: Check stdlib with stubtest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        # Python 3.9.7 is required due to changes to ForwardRef.
+        python-version: ["3.6", "3.7", "3.8", "3.9.7", "3.10"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Update pip
+        run: python -m pip install -U pip
+      - name: Install dependencies
+        run: pip install $(grep tomli== requirements-tests-py3.txt) $(grep mypy== requirements-tests-py3.txt)
+      - name: Run stubtest
+        run: python tests/stubtest_stdlib.py --ignore-unused-allowlist
+
+  stubtest-third-party:
+    name: Check third party stubs with stubtest
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shard-index: [0, 1, 2, 3]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        run: pip install $(grep tomli== requirements-tests-py3.txt)
+      - name: Run stubtest
+        run: python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}

.github/workflows/tests.yml

@@ -132,16 +132,22 @@ jobs:
   stubtest-third-party:
     name: Check third party stubs with stubtest
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        shard-index: [0, 1, 2, 3]
-      fail-fast: false
+    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
           python-version: 3.9
       - name: Install dependencies
         run: pip install $(grep tomli== requirements-tests-py3.txt)
       - name: Run stubtest
-        run: python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+        run: |
+          STUBS=$(git diff --name-only origin/${{ github.base_ref }} HEAD | egrep ^stubs/ | cut -d "/" -f 2)
+          if test -n "$STUBS"; then
+            echo "Testing $STUBS..."
+            python tests/stubtest_third_party.py $STUBS
+          else
+            echo "Nothing to test"
+          fi