Make tests version-aware. Now only tornado and sqlalchemy stubs fail.

runtests.py

@@ -17,10 +17,13 @@ import re
 import sys
 import argparse
 
-parser = argparse.ArgumentParser(description="Test runner for typeshed. Patterns are unanchored regexps on the full path.")
+parser = argparse.ArgumentParser(description="Test runner for typeshed. "
+                                             "Patterns are unanchored regexps on the full path.")
 parser.add_argument('-v', '--verbose', action='count', default=0, help="More output")
 parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run mypy")
 parser.add_argument('-x', '--exclude', type=str, nargs='*', help="Exclude pattern")
+parser.add_argument('-p', '--python-version', type=str, nargs='*',
+                    help="These versions only (major[.minor])")
 parser.add_argument('filter', type=str, nargs='*', help="Include pattern (default all)")
 
 
@@ -28,6 +31,7 @@ def log(args, *varargs):
     if args.verbose >= 2:
         print(*varargs)
 
+
 def match(args, fn):
     if not args.filter and not args.exclude:
         log(args, fn, 'accept by default')
@@ -49,6 +53,20 @@ def match(args, fn):
     return True
 
 
+def libpath(major, minor):
+    versions = ['%d.%d' % (major, minor)
+                for minor in reversed(range(minor + 1))]
+    versions.append(str(major))
+    versions.append('2and3')
+    paths = []
+    for v in versions:
+        for top in ['stdlib', 'third_party']:
+            p = os.path.join(top, v)
+            if os.path.isdir(p):
+                paths.append(p)
+    return paths
+
+
 def main():
     args = parser.parse_args()
 
@@ -58,39 +76,62 @@ def main():
         print("Cannot import mypy. Did you install it?")
         sys.exit(1)
 
-    files2 = []
-    files3 = []
-    for dir, subdirs, files in os.walk('.'):
-        for file in files:
-            if file == '__builtin__.pyi':
-                continue  # Special case (alias for builtins.py).
-            if file == 'typing.pyi':
-                continue  # Hack for https://github.com/python/mypy/issues/1254
-            if file.endswith('.pyi') or file.endswith('.py'):
-                full = os.path.join(dir, file)
-                if match(args, full):
-                    if '/2' in dir:
-                        files2.append(full)
-                    if '/3' in dir or '/2and3' in dir:
-                        files3.append(full)
-    if not (files2 or files3):
-        print('--- nothing to do ---')
+    versions = [(3, 5), (3, 4), (3, 3), (3, 2), (2, 7)]
+    if args.python_version:
+        versions = [v for v in versions
+                    if any(('%d.%d' % v).startswith(av) for av in args.python_version)]
+        if not versions:
+            print("--- no versions selected ---")
+            sys.exit(1)
+
     code = 0
-    for flags, files in [([], files3), (['--py2'], files2)]:
+    runs = 0
+    for major, minor in versions:
+        roots = libpath(major, minor)
+        files = []
+        seen = {'__builtin__', 'builtins', 'typing'}  # Always ignore these.
+        for root in roots:
+            names = os.listdir(root)
+            for name in names:
+                full = os.path.join(root, name)
+                mod, ext = os.path.splitext(name)
+                if mod in seen:
+                    continue
+                if ext in ['.pyi', '.py']:
+                    if match(args, full):
+                        seen.add(mod)
+                        files.append(full)
+                elif (os.path.isfile(os.path.join(full, '__init__.pyi')) or
+                      os.path.isfile(os.path.join(full, '__init__.py'))):
+                    for r, ds, fs in os.walk(full):
+                        ds.sort()
+                        fs.sort()
+                        for f in fs:
+                            m, x = os.path.splitext(f)
+                            if x in ['.pyi', '.py']:
+                                fn = os.path.join(r, f)
+                                if match(args, fn):
+                                    seen.add(mod)
+                                    files.append(fn)
         if files:
+            runs += 1
+            flags = ['--python-version', '%d.%d' % (major, minor)]
             sys.argv = ['mypy'] + flags + files
             if args.verbose:
-                print('running', ' '.join(sys.argv))
+                print("running", ' '.join(sys.argv))
             else:
-                print('running mypy', ' '.join(flags), '# with', len(files), 'files')
+                print("running mypy", ' '.join(flags), "# with", len(files), "files")
             try:
                 if not args.dry_run:
                     mypy_main('')
             except SystemExit as err:
                 code = max(code, err.code)
     if code:
-        print('--- exit status', code, '---')
+        print("--- exit status", code, "---")
         sys.exit(code)
+    if not runs:
+        print("--- nothing to do; exit 1 ---")
+        sys.exit(1)
 
 
 if __name__ == '__main__':