Refactor how test are run, add tests to exercise pytype (if installed) (#353)

.travis.yml

@@ -1,15 +1,18 @@
 sudo: false
 language: python
-python:
-  - "3.5"
-  # More versions later.
+
+matrix:
+  include:
+    - python: "3.5"
+      env: TEST_CMD="./tests/mypy_test.py"
+    - python: "2.7"
+      env: TEST_CMD="./tests/pytype_test.py --num-parallel=4"
 
 install:
-  - pip install -U git+git://github.com/python/mypy
+  # pytype needs py-2.7, mypy needs py-3.2+. Additional logic in runtests.py
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; fi
 
 script:
-  - ./runtests.py -p3.5
-  - ./runtests.py -p3.4
-  - ./runtests.py -p3.3
-  - ./runtests.py -p3.2
-  - ./runtests.py -p2.7
+  - $TEST_CMD
+

runtests.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+./tests/mypy_test.py
+./tests/pytype_test.py

tests/mypy_test.py

@@ -22,7 +22,7 @@ parser = argparse.ArgumentParser(description="Test runner for typeshed. "
 parser.add_argument('-v', '--verbose', action='count', default=0, help="More output")
 parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run mypy")
 parser.add_argument('-x', '--exclude', type=str, nargs='*', help="Exclude pattern")
-parser.add_argument('-p', '--python-version', type=str, nargs='*',
+parser.add_argument('-p', '--python-version', type=str, action='append',
                     help="These versions only (major[.minor])")
 parser.add_argument('filter', type=str, nargs='*', help="Include pattern (default all)")
 

tests/pytype_blacklist.txt

@@ -0,0 +1,59 @@
+# Pytype blacklist. Files will not be tested with pytype.
+
+2.7/Cookie.pyi
+2.7/SocketServer.pyi
+2.7/StringIO.pyi
+2.7/__builtin__.pyi
+2.7/__future__.pyi
+2.7/argparse.pyi
+2.7/builtins.pyi
+2.7/calendar.pyi
+2.7/codecs.pyi
+2.7/collections.pyi
+2.7/csv.pyi
+2.7/email/utils.pyi
+2.7/functools.pyi
+2.7/inspect.pyi
+2.7/logging/__init__.pyi
+2.7/os/__init__.pyi
+2.7/platform.pyi
+2.7/resource.pyi
+2.7/rfc822.pyi
+2.7/simplejson/__init__.pyi
+2.7/socket.pyi
+2.7/sqlite3/dbapi2.pyi
+2.7/ssl.pyi
+2.7/subprocess.pyi
+2.7/threading.pyi
+2.7/time.pyi
+2.7/token.pyi
+2.7/types.pyi
+2.7/typing.pyi
+2.7/unittest.pyi
+2.7/urllib2.pyi
+2.7/urlparse.pyi
+2.7/uuid.pyi
+2.7/xml/etree/ElementInclude.pyi
+2.7/xml/etree/ElementPath.pyi
+2.7/xml/etree/ElementTree.pyi
+2and3/cmath.pyi
+2and3/logging/__init__.pyi
+2and3/logging/config.pyi
+2and3/logging/handlers.pyi
+2and3/math.pyi
+2and3/warnings.pyi
+2and3/webbrowser.pyi
+
+# These cause problems for different reasons from above
+2.7/_functools.pyi
+2.7/array.pyi
+2.7/ast.pyi
+2.7/encodings/utf_8.pyi
+2.7/fcntl.pyi
+2.7/itertools.pyi
+2.7/json.pyi
+2.7/multiprocessing/process.pyi
+2.7/pickle.pyi
+2.7/tempfile.pyi
+2and3/argparse.pyi
+2and3/bz2.pyi

tests/pytype_test.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+"""Test runner for typeshed.
+
+Depends on mypy and pytype being installed.
+
+If pytype is installed:
+    1. For every pyi, run "pytd <foo.pyi>" in a separate process
+"""
+
+import os
+import re
+import sys
+import argparse
+import subprocess
+import collections
+
+parser = argparse.ArgumentParser(description="Pytype tests.")
+parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run tests")
+parser.add_argument('--num-parallel', type=int, default=1,
+                    help="Number of test processes to spawn")
+
+
+def main():
+    args = parser.parse_args()
+    code, runs = pytype_test(args)
+
+    if code:
+        print("--- exit status %d ---" % code)
+        sys.exit(code)
+    if not runs:
+        print("--- nothing to do; exit 1 ---")
+        sys.exit(1)
+
+
+def load_blacklist():
+    filename = os.path.join(os.path.dirname(__file__), "pytype_blacklist.txt")
+    regex = r"^\s*([^\s#]+)\s*(?:#.*)?$"
+
+    with open(filename) as f:
+        return re.findall(regex, f.read(), flags=re.M)
+
+
+class PytdRun(object):
+    def __init__(self, args, dry_run=False):
+        self.args = args
+        self.dry_run = dry_run
+        self.results = None
+
+        if dry_run:
+            self.results = (0, "", "")
+        else:
+            self.proc = subprocess.Popen(
+                ["pytd"] + args,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+
+    def communicate(self):
+        if self.results:
+            return self.results
+
+        stdout, stderr = self.proc.communicate()
+        self.results = self.proc.returncode, stdout, stderr
+        return self.results
+
+
+def pytype_test(args):
+    try:
+        PytdRun(["-h"]).communicate()
+    except OSError:
+        print("Cannot run pytd. Did you install pytype?")
+        return 0, 0
+
+    wanted = re.compile(r"stdlib/(2\.7|2and3)/.*\.pyi$")
+    skipped = re.compile("(%s)$" % "|".join(load_blacklist()))
+    files = []
+
+    for root, _, filenames in os.walk("stdlib"):
+        for f in sorted(filenames):
+            f = os.path.join(root, f)
+            if wanted.search(f) and not skipped.search(f):
+                files.append(f)
+
+    running_tests = collections.deque()
+    max_code, runs, errors = 0, 0, 0
+    print("Running pytype tests...")
+    while 1:
+        while files and len(running_tests) < args.num_parallel:
+            test_run = PytdRun([files.pop()], dry_run=args.dry_run)
+            running_tests.append(test_run)
+
+        if not running_tests:
+            break
+
+        test_run = running_tests.popleft()
+        code, stdout, stderr = test_run.communicate()
+        max_code = max(max_code, code)
+        runs += 1
+
+        if code:
+            print("pytd error processing \"%s\":" % test_run.args[0])
+            print(stderr)
+            errors += 1
+
+    print("Ran pytype with %d pyis, got %d errors." % (runs, errors))
+    return max_code, runs
+
+
+if __name__ == '__main__':
+    main()