From 79d8757f1ffb915350c02267e049e9a4a9311f9c Mon Sep 17 00:00:00 2001 From: Lukasz Langa Date: Tue, 20 Dec 2016 03:19:53 -0800 Subject: [PATCH] Enable flake8 tests Ran both by Travis and locally. There's some setup required, README updated. A few important Flake8 checks are still disabled, we're going to enable them as soon as the stubs are fixed and we can reliably run Flake8 locally with Python 3.6. --- .travis.yml | 5 +-- README.md | 62 +++++++++++++++++++++++++++++--------- requirements-tests-py2.txt | 1 + requirements-tests-py3.txt | 4 +++ runtests.sh | 4 +-- tests/flake8_test.py | 47 +++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+), 20 deletions(-) create mode 100644 requirements-tests-py2.txt create mode 100644 requirements-tests-py3.txt create mode 100755 tests/flake8_test.py diff --git a/.travis.yml b/.travis.yml index 292bf2626..1db0ec621 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: python matrix: include: + - python: "3.5" + env: TEST_CMD="./tests/flake8_test.py" - python: "3.5" env: TEST_CMD="./tests/mypy_test.py" - python: "2.7" @@ -10,9 +12,8 @@ matrix: install: # pytype needs py-2.7, mypy needs py-3.2+. Additional logic in runtests.py - - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy && pip install -U git+git://github.com/dropbox/typed_ast; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy git+git://github.com/dropbox/typed_ast flake8-bugbear; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; fi script: - $TEST_CMD - diff --git a/README.md b/README.md index 49f865f19..adc239748 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ the repo. There are two separate sets of tests: `tests/mypy_test.py` runs tests against [mypy](https://github.com/python/mypy/), while `tests/pytype_tests.py` runs tests against [pytype](https://github.com/google/pytype/). The script runtests.sh -just runs both sets of tests. +runs both sets of tests and flake8 over all .pyi files. Both sets of tests are shallow -- they verify that all stubs can be imported but they don't check whether stubs match their implementation @@ -104,11 +104,52 @@ imported but they don't check whether stubs match their implementation that each set of tests has a blacklist of modules that are not tested at all. The blacklists also live in the tests directory. -To manually run the mypy tests, you need to have Python 3.3 or higher -and `pip3 install mypy-lang`. To run 3.6 tests, you'll need to also -`pip3 install typed-ast` needed for `--fast-parser` support. +To manually run the mypy tests, you need to have Python 3.5 or higher. +Run: +``` +$ python3.5 -m venv .venv3 +$ source .venv3/bin/activate +(.venv3)$ pip install -r requirements-tests-py3.txt +``` +This will install mypy-lang, typed-ast, and flake8. You can then run +mypy tests and flake8 tests by invoking: +``` +(.venv3)$ python tests/mypy_test.py +... +(.venv3)$ python tests/flake8_test.py +... +``` +To run the pytype tests, you need a separate virtual environment with +Python 2.7. Run: +``` +$ virtualenv --python=python2.7 .venv2 +$ source .venv2/bin/activate +(.venv2)$ pip install -r requirements-tests-py2.txt +``` +This will install pytype from its GitHub repo. You can then run pytype +tests by running: +``` +(.venv2)$ python tests/pytype_test.py +``` -To run the pytype tests, you need Python 2.7 and install pytype from its repo. +To be able to everything with ``runtests.sh``, copy the ``pytype`` script +from the Python 2 virtualenv to the Python 3 one: +``` +$ cp .venv2/bin/pytype .venv3/bin/pytype +$ source .venv3/bin/activate +(.venv3)$ ./runtests.sh +running mypy --python-version 3.6 --strict-optional --fast-parser # with 479 files +running mypy --python-version 3.5 --strict-optional # with 469 files +running mypy --python-version 3.4 --strict-optional # with 469 files +running mypy --python-version 3.3 --strict-optional # with 454 files +running mypy --python-version 3.2 --strict-optional # with 453 files +running mypy --python-version 2.7 --strict-optional # with 502 files +Running pytype tests... +Ran pytype with 244 pyis, got 0 errors. +Running flake8 on 886 .pyi files... +flake8 run clean. +(.venv3)$ +``` For mypy, if you are in the typeshed repo that is submodule of the mypy repo (so `..` refers to the mypy repo), there's a shortcut to run @@ -116,16 +157,7 @@ the mypy tests that avoids installing mypy: ```bash $ PYTHONPATH=.. python3 tests/mypy_test.py ``` -This runs six sets of tests: -```bash -running mypy --python-version 3.6 --strict-optional --fast-parser # with 353 files -running mypy --python-version 3.5 --strict-optional # with 343 files -running mypy --python-version 3.4 --strict-optional # with 343 files -running mypy --python-version 3.3 --strict-optional # with 328 files -running mypy --python-version 3.2 --strict-optional # with 327 files -running mypy --python-version 2.7 --strict-optional # with 382 files -``` -You can limit it to a single test by passing `-p2` or `-p3.5` e.g. +You can mypy tests to a single version by passing `-p2` or `-p3.5` e.g. ```bash $ PYTHONPATH=.. python3 tests/mypy_test.py -p3.5 running mypy --python-version 3.5 --strict-optional # with 342 files diff --git a/requirements-tests-py2.txt b/requirements-tests-py2.txt new file mode 100644 index 000000000..cb25c87a5 --- /dev/null +++ b/requirements-tests-py2.txt @@ -0,0 +1 @@ +-e git+https://github.com/google/pytype.git@master#egg=pytype diff --git a/requirements-tests-py3.txt b/requirements-tests-py3.txt new file mode 100644 index 000000000..28aa0670c --- /dev/null +++ b/requirements-tests-py3.txt @@ -0,0 +1,4 @@ +mypy-lang>=0.4.6 +typed-ast>=0.6.1 +flake8>=3.2.1 +flake8-bugbear>=16.12.2 diff --git a/runtests.sh b/runtests.sh index a7e775783..4646841d2 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,6 +2,4 @@ ./tests/mypy_test.py ./tests/pytype_test.py - -# FIXME: Enable when errors listed in `.flake8` are fixed in all stub files. -# find . -name "*.pyi" | xargs -s 1024 flake8 --builtins=StandardError,apply,basestring,buffer,cmp,coerce,execfile,file,intern,long,raw_input,reduce,reload,unichr,unicode,xrange +./tests/flake8_test.py diff --git a/tests/flake8_test.py b/tests/flake8_test.py new file mode 100755 index 000000000..2f00328fc --- /dev/null +++ b/tests/flake8_test.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import pathlib +import subprocess +import sys + +PY2_ONLY_KEYWORDS = [ + 'StandardError', + 'apply', + 'basestring', + 'buffer', + 'cmp', + 'coerce', + 'execfile', + 'file', + 'intern', + 'long', + 'raw_input', + 'reduce', + 'reload', + 'unichr', + 'unicode', + 'xrange', +] + +root = pathlib.Path(__file__).parent.parent +paths = list(sorted(str(p) for p in root.glob('**/*.pyi'))) +window = 0 +size = 100 +returncode = 0 + +print('Running flake8 on {} .pyi files...'.format(len(paths))) + +while True: + chunk = paths[window:window + size] + if not chunk: + break + + proc = subprocess.run( + ['flake8', '--builtins=' + ','.join(PY2_ONLY_KEYWORDS)] + chunk, + ) + if proc.returncode: + print('flake8 run failed!') + sys.exit(1) + + window += size + +print('flake8 run clean.')