stubtest_stdlib: fail if there are unused allowlist entries (#6424)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/stubtest-unused-allowlist.yml

@@ -1,61 +0,0 @@
-name: Remove unused stubtest allowlist entries
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 4 * * *'
-
-permissions:
-  contents: write
-  pull-requests: write
-
-jobs:
-  stubtest:
-    if: github.repository == 'python/typeshed'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install $(grep mypy== requirements-tests-py3.txt)
-      - name: Run stubtest
-        shell: bash
-        run: ./tests/stubtest_unused.py | tee stubtest-output-${{ matrix.os }}-${{ matrix.python-version }}
-      - name: Store output
-        uses: actions/upload-artifact@v2
-        with:
-          name: stubtest-output
-          path: stubtest-output-${{ matrix.os }}-${{ matrix.python-version }}
-
-  collate:
-    if: github.repository == 'python/typeshed'
-    runs-on: ubuntu-latest
-    needs: stubtest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - name: Get stubtest outputs
-        uses: actions/download-artifact@v2
-        with:
-          name: stubtest-output
-      - name: Collate duplicates
-        run: cat stubtest-output-* | sort -u | tee stubtest-output
-      - name: Remove entries from allowlists
-        run: python scripts/update-stubtest-allowlist.py stubtest-output
-      - name: Create pull request
-        # v3.8.2
-        uses: peter-evans/create-pull-request@052fc72b4198ba9fbc81b818c6e1859f747d49a8
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Remove unused stubtest allowlist entries
-          title: "[gh-action] Remove unused stubtest allowlist entries"
-          base: master

.github/workflows/stubtest.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Install dependencies
         run: pip install $(grep tomli== requirements-tests-py3.txt) $(grep mypy== requirements-tests-py3.txt)
       - name: Run stubtest
-        run: python tests/stubtest_stdlib.py --ignore-unused-allowlist
+        run: python tests/stubtest_stdlib.py
 
   stubtest-third-party:
     name: Check third party stubs with stubtest

.github/workflows/tests.yml

@@ -109,7 +109,7 @@ jobs:
       - name: Install dependencies
         run: pip install $(grep tomli== requirements-tests-py3.txt) $(grep mypy== requirements-tests-py3.txt)
       - name: Run stubtest
-        run: python tests/stubtest_stdlib.py --ignore-unused-allowlist
+        run: python tests/stubtest_stdlib.py
 
   stubtest-third-party:
     name: Check third party stubs with stubtest

tests/stubtest_allowlists/py310.txt

@@ -124,9 +124,6 @@ tkinter.EventType.__new__
 _markupbase.ParserBase.error
 asyncio.proactor_events._ProactorReadPipeTransport.__init__
 distutils.command.bdist_wininst
-logging.Formatter.__init__
-logging.LoggerAdapter.__init__
-logging.PercentStyle.__init__
 py_compile.main
 pyclbr.Class.__init__
 pyclbr.Function.__init__

tests/stubtest_stdlib.py

@@ -57,7 +57,8 @@ def run_stubtest(typeshed_dir: Path) -> int:
             "\nCommand run was: {}\n".format(" ".join(cmd)),
             file=sys.stderr,
         )
-        print("stubtest failed", file=sys.stderr)
+        print("\n\n", file=sys.stderr)
+        print(f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_dir}', file=sys.stderr)
         return e.returncode
     else:
         print("stubtest succeeded", file=sys.stderr)

tests/stubtest_unused.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-# Runs stubtest and prints each unused allowlist entry with filename.
-import subprocess
-import sys
-from pathlib import Path
-from typing import List, Tuple
-
-_UNUSED_NOTE = "note: unused allowlist entry "
-_ALLOWLIST_PATH = Path("tests") / "stubtest_allowlists"
-
-
-def main() -> None:
-    unused = run_stubtest()
-    with_filenames = []
-    for uu in unused:
-        with_filenames.extend(unused_files(uu))
-    for file, uu in with_filenames:
-        print(file + ":" + uu)
-
-
-def run_stubtest() -> List[str]:
-    proc = subprocess.run([sys.executable, "tests/stubtest_stdlib.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    output = proc.stdout.decode("utf-8").splitlines()
-    return [line[len(_UNUSED_NOTE) :].strip() for line in output if line.startswith(_UNUSED_NOTE)]
-
-
-def unused_files(unused: str) -> List[Tuple[str, str]]:
-    version = "py{}{}".format(sys.version_info[0], sys.version_info[1])
-    files = ["py3_common.txt", version + ".txt", sys.platform + ".txt", sys.platform + "-" + version + ".txt"]
-    found = []
-    for file in files:
-        path = _ALLOWLIST_PATH / file
-        if find_unused_in_file(unused, path):
-            found.append((path.as_posix(), unused))
-    if not found:
-        raise ValueError("unused item {} not found in any allowlist file".format(unused))
-    return found
-
-
-def find_unused_in_file(unused: str, path: Path) -> bool:
-    try:
-        with open(path) as f:
-            return any(line.strip().split(" ")[0] == unused for line in f)
-    except FileNotFoundError:
-        return False
-
-
-if __name__ == "__main__":
-    main()