From d76ad4710e0dd4017f6cfdbca01d02173e5b23b3 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 3 Jun 2020 14:37:38 +0200 Subject: [PATCH] Find unused stubtest whitelist entries (#4157) * Find unused stubtest whitelist entries This is currently a GitHub workflow that runs daily and lists all unused whitelist entries found by running stubtest against current Python versions on Linux and Windows. The workflow run will succeed if there are no such entries, and fail otherwise. In a second step, this should collate the output of the various runs and create a PR to remove the entries. In that case, the workflow should probably only run weekly or even monthly to keep the noise down. Cf. #3728 --- .../workflows/stubtest-unused-whitelist.yml | 26 ++++++++ .travis.yml | 16 ++--- tests/stubtest_test.py | 7 ++- tests/stubtest_unused.py | 59 +++++++++++++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/stubtest-unused-whitelist.yml create mode 100755 tests/stubtest_unused.py diff --git a/.github/workflows/stubtest-unused-whitelist.yml b/.github/workflows/stubtest-unused-whitelist.yml new file mode 100644 index 000000000..1f8e423fd --- /dev/null +++ b/.github/workflows/stubtest-unused-whitelist.yml @@ -0,0 +1,26 @@ +name: Find unused stubtest whitelist entries + +on: + schedule: + - cron: '0 4 * * *' + +jobs: + find: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest"] + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install -U pip + pip install -U git+git://github.com/python/mypy@b3d43984 + - name: Run stubtest + run: ./tests/stubtest_unused.py diff --git a/.travis.yml b/.travis.yml index bb70c9fdb..a0833912e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,44 +42,44 @@ jobs: - name: "stubtest py38" python: 3.8 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py37" python: 3.7 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py36" python: 3.6 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py35" python: 3.5 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py38 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.8.3 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py37 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.7.7 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py36 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.6.8 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py35 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.5.4 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist diff --git a/tests/stubtest_test.py b/tests/stubtest_test.py index fba0634fe..31605e1df 100755 --- a/tests/stubtest_test.py +++ b/tests/stubtest_test.py @@ -21,6 +21,8 @@ def run_stubtest(typeshed_dir: Path) -> int: platform_whitelist = "{}.txt".format(sys.platform) combined_whitelist = "{}-py{}{}.txt".format(sys.platform, sys.version_info.major, sys.version_info.minor) + ignore_unused_whitelist = "--ignore-unused-whitelist" in sys.argv[1:] + cmd = [ sys.executable, "-m", @@ -37,6 +39,8 @@ def run_stubtest(typeshed_dir: Path) -> int: "--whitelist", str(whitelist_dir / version_whitelist), ] + if ignore_unused_whitelist: + cmd += ["--ignore-unused-whitelist"] if (whitelist_dir / platform_whitelist).exists(): cmd += [ "--whitelist", @@ -59,9 +63,6 @@ def run_stubtest(typeshed_dir: Path) -> int: "\nNB: stubtest output depends on the Python version (and system) it is run with. " "See README.md for more details.\n" "NB: We only check positional-only arg accuracy for Python 3.9.\n" - "If stubtest is complaining about 'unused whitelist entry' after your fix, please " - "remove the entry from the whitelist file. Note you may have to do this for other " - "version-specific whitelists as well. Thanks for helping burn the backlog of errors!\n" "\nCommand run was: {}\n".format(" ".join(cmd)), file=sys.stderr, ) diff --git a/tests/stubtest_unused.py b/tests/stubtest_unused.py new file mode 100755 index 000000000..df26e7266 --- /dev/null +++ b/tests/stubtest_unused.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Runs stubtest and prints each unused whitelist entry with filename. + +from typing import List, Tuple +import os.path +import subprocess +import sys + +_UNUSED_NOTE = "note: unused whitelist entry " +_WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists") + + +def main() -> int: + unused = run_stubtest() + with_filenames = [] + for uu in unused: + with_filenames.extend(unused_files(uu)) + for file, uu in with_filenames: + print(file + ":" + uu) + return 1 if with_filenames else 0 + + +def run_stubtest() -> List[str]: + popen = subprocess.Popen( + ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + assert popen.stdout is not None + unused = [] + for line in popen.stdout: + if line.startswith(_UNUSED_NOTE): + unused.append(line[len(_UNUSED_NOTE):].strip()) + popen.wait() + return unused + + +def unused_files(unused: str) -> List[Tuple[str, str]]: + version = "py{}{}".format(sys.version_info[0], sys.version_info[1]) + files = ["py3_common.txt", version + ".txt", sys.platform + ".txt", sys.platform + "-" + version + ".txt"] + found = [] + for file in files: + path = os.path.join(_WHITELIST_PATH, file) + if find_unused_in_file(unused, path): + found.append((path, unused)) + if not found: + raise ValueError("unused item {} not found in any whitelist file".format(unused)) + return found + + +def find_unused_in_file(unused: str, path: str) -> bool: + try: + with open(path) as f: + return any(line.strip().split(" ")[0] == unused for line in f) + except FileNotFoundError: + return False + + +if __name__ == "__main__": + sys.exit(main())