Find unused stubtest whitelist entries (#4157)

* Find unused stubtest whitelist entries

This is currently a GitHub workflow that runs daily and lists all unused
whitelist entries found by running stubtest against current Python
versions on Linux and Windows. The workflow run will succeed if there
are no such entries, and fail otherwise.

In a second step, this should collate the output of the various runs and
create a PR to remove the entries. In that case, the workflow should
probably only run weekly or even monthly to keep the noise down.

Cf. #3728

.github/workflows/stubtest-unused-whitelist.yml

@@ -0,0 +1,26 @@
+name: Find unused stubtest whitelist entries
+
+on:
+  schedule:
+    - cron: '0 4 * * *'
+
+jobs:
+  find:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest"]
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          pip install -U git+git://github.com/python/mypy@b3d43984
+      - name: Run stubtest
+        run: ./tests/stubtest_unused.py

.travis.yml

@@ -42,44 +42,44 @@ jobs:
     - name: "stubtest py38"
       python: 3.8
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py37"
       python: 3.7
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py36"
       python: 3.6
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py35"
       python: 3.5
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: ./tests/stubtest_test.py
+      script: ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py38 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.8.3
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py37 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.7.7
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py36 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.6.8
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist
     - name: "stubtest py35 (Windows)"
       <<: *test_windows
       env:
         <<: *env_windows
         PYTHON_VERSION: 3.5.4
       install: pip install -U git+git://github.com/python/mypy@b3d43984
-      script: python ./tests/stubtest_test.py
+      script: python ./tests/stubtest_test.py --ignore-unused-whitelist

tests/stubtest_test.py

@@ -21,6 +21,8 @@ def run_stubtest(typeshed_dir: Path) -> int:
     platform_whitelist = "{}.txt".format(sys.platform)
     combined_whitelist = "{}-py{}{}.txt".format(sys.platform, sys.version_info.major, sys.version_info.minor)
 
+    ignore_unused_whitelist = "--ignore-unused-whitelist" in sys.argv[1:]
+
     cmd = [
         sys.executable,
         "-m",
@@ -37,6 +39,8 @@ def run_stubtest(typeshed_dir: Path) -> int:
         "--whitelist",
         str(whitelist_dir / version_whitelist),
     ]
+    if ignore_unused_whitelist:
+        cmd += ["--ignore-unused-whitelist"]
     if (whitelist_dir / platform_whitelist).exists():
         cmd += [
             "--whitelist",
@@ -59,9 +63,6 @@ def run_stubtest(typeshed_dir: Path) -> int:
             "\nNB: stubtest output depends on the Python version (and system) it is run with. "
             "See README.md for more details.\n"
             "NB: We only check positional-only arg accuracy for Python 3.9.\n"
-            "If stubtest is complaining about 'unused whitelist entry' after your fix, please "
-            "remove the entry from the whitelist file. Note you may have to do this for other "
-            "version-specific whitelists as well. Thanks for helping burn the backlog of errors!\n"
             "\nCommand run was: {}\n".format(" ".join(cmd)),
             file=sys.stderr,
         )

tests/stubtest_unused.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Runs stubtest and prints each unused whitelist entry with filename.
+
+from typing import List, Tuple
+import os.path
+import subprocess
+import sys
+
+_UNUSED_NOTE = "note: unused whitelist entry "
+_WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists")
+
+
+def main() -> int:
+    unused = run_stubtest()
+    with_filenames = []
+    for uu in unused:
+        with_filenames.extend(unused_files(uu))
+    for file, uu in with_filenames:
+        print(file + ":" + uu)
+    return 1 if with_filenames else 0
+
+
+def run_stubtest() -> List[str]:
+    popen = subprocess.Popen(
+        ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+    )
+    assert popen.stdout is not None
+    unused = []
+    for line in popen.stdout:
+        if line.startswith(_UNUSED_NOTE):
+            unused.append(line[len(_UNUSED_NOTE):].strip())
+    popen.wait()
+    return unused
+
+
+def unused_files(unused: str) -> List[Tuple[str, str]]:
+    version = "py{}{}".format(sys.version_info[0], sys.version_info[1])
+    files = ["py3_common.txt", version + ".txt", sys.platform + ".txt", sys.platform + "-" + version + ".txt"]
+    found = []
+    for file in files:
+        path = os.path.join(_WHITELIST_PATH, file)
+        if find_unused_in_file(unused, path):
+            found.append((path, unused))
+    if not found:
+        raise ValueError("unused item {} not found in any whitelist file".format(unused))
+    return found
+
+
+def find_unused_in_file(unused: str, path: str) -> bool:
+    try:
+        with open(path) as f:
+            return any(line.strip().split(" ")[0] == unused for line in f)
+    except FileNotFoundError:
+        return False
+
+
+if __name__ == "__main__":
+    sys.exit(main())