From aa6e3efcca827dc7d89ce4c156f4bd43356764a1 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Sat, 29 Feb 2020 04:58:47 -0800 Subject: [PATCH] README.md: better document test_stubtest pitfall (#3793) Document that the test will likely fail locally, to help cases like #3782 --- README.md | 13 +++++++++---- tests/stubtest_test.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 180fecc97..750333679 100644 --- a/README.md +++ b/README.md @@ -162,10 +162,13 @@ This test requires Python 3.5 or higher. Run using `(.venv3)$ python3 tests/stubtest_test.py` This test compares the stdlib stubs against the objects at runtime. Because of -this, the output depends on which version of Python it is run with. +this, the output depends on which version of Python and on what kind of system +it is run. +Thus the easiest way to run this test is by enabling Travis CI on your fork; +if you run it locally, it'll likely complain about system-specific +differences (in e.g, `socket`) that the type system cannot capture. If you need a specific version of Python to repro a CI failure, -[pyenv](https://github.com/pyenv/pyenv) can help (as can enabling Travis CI on -your fork). +[pyenv](https://github.com/pyenv/pyenv) can help. Due to its dynamic nature, you may run into false positives. In this case, you can add to the whitelists for each affected Python version in @@ -173,7 +176,9 @@ can add to the whitelists for each affected Python version in at [mypy](https://github.com/python/mypy/issues). To run stubtest against third party stubs, it's easiest to use stubtest -directly. stubtest can also help you find things missing from the stubs. +directly, with `(.venv3)$ python3 -m mypy.stubtest --custom-typeshed-dir + `. +stubtest can also help you find things missing from the stubs. ### flake8 diff --git a/tests/stubtest_test.py b/tests/stubtest_test.py index 4937321b0..99b087719 100755 --- a/tests/stubtest_test.py +++ b/tests/stubtest_test.py @@ -44,8 +44,8 @@ def run_stubtest(typeshed_dir: Path) -> int: subprocess.run(cmd, check=True) except subprocess.CalledProcessError as e: print( - "\nNB: stubtest output depends on the Python version it is run with. See README.md " - "for more details.\n" + "\nNB: stubtest output depends on the Python version (and system) it is run with. " + "See README.md for more details.\n" "NB: We only check positional-only arg accuracy for Python 3.8.\n" "If stubtest is complaining about 'unused whitelist entry' after your fix, please " "remove the entry from the whitelist file. Note you may have to do this for other "