diff --git a/.travis.yml b/.travis.yml
index a6e6aaf32..c2a26cac3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,4 @@ install:
   - pip install git+git://github.com/JukkaL/mypy
 
 script:
-  - mypy `find stdlib/{2and3,3*} -name \*.pyi`
-  # - mypy `find third_party/{2and3,3*} -name \*.pyi`
-  - mypy --py2 `find stdlib/2* -name \*.pyi | grep -v __builtin__.pyi`  # builtins and __builtin__ are aliases
-  # - mypy --py2  `find third_party/2* -name \*.pyi | grep -v sqlalchemy`  # sqlalchemy is currently busted
+  - ./runtests.py -x tornado sqlalchemy 3/enum requests
diff --git a/runtests.py b/runtests.py
new file mode 100755
index 000000000..603f421a5
--- /dev/null
+++ b/runtests.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Test runner for typeshed.
+
+Depends on mypy being installed.
+
+Approach:
+
+1. Parse sys.argv
+2. Compute appropriate arguments for mypy
+3. Stuff those arguments into sys.argv
+4. Run mypy.main('')
+5. Repeat steps 2-4 for other mypy runs (e.g. --py2)
+"""
+
+import os
+import re
+import sys
+import argparse
+
+parser = argparse.ArgumentParser(description="Test runner for typeshed. Patterns are unanchored regexps on the full path.")
+parser.add_argument('-v', '--verbose', action='count', default=0, help="More output")
+parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run mypy")
+parser.add_argument('-x', '--exclude', type=str, nargs='*', help="Exclude pattern")
+parser.add_argument('filter', type=str, nargs='*', help="Include pattern (default all)")
+
+
+def log(args, *varargs):
+    if args.verbose >= 2:
+        print(*varargs)
+
+def match(args, fn):
+    if not args.filter and not args.exclude:
+        log(args, fn, 'accept by default')
+        return True
+    if args.exclude:
+        for f in args.exclude:
+            if re.search(f, fn):
+                log(args, fn, 'excluded by pattern', f)
+                return False
+    if args.filter:
+        for f in args.filter:
+            if re.search(f, fn):
+                log(args, fn, 'accepted by pattern', f)
+                return True
+    if args.filter:
+        log(args, fn, 'rejected (no pattern matches)')
+        return False
+    log(args, fn, 'accepted (no exclude pattern matches)')
+    return True
+
+
+def main():
+    args = parser.parse_args()
+
+    try:
+        from mypy.main import main as mypy_main
+    except ImportError:
+        print("Cannot import mypy. Did you install it?")
+        sys.exit(1)
+
+    files2 = []
+    files3 = []
+    for dir, subdirs, files in os.walk('.'):
+        for file in files:
+            if file == '__builtin__.pyi':
+                continue  # Special case (alias for builtins.py).
+            if file.endswith('.pyi') or file.endswith('.py'):
+                full = os.path.join(dir, file)
+                if match(args, full):
+                    if '/2' in dir:
+                        files2.append(full)
+                    if '/3' in dir or '/2and3' in dir:
+                        files3.append(full)
+    if not (files2 or files3):
+        print('--- nothing to do ---')
+    code = 0
+    for flags, files in [([], files3), (['--py2'], files2)]:
+        if files:
+            sys.argv = ['mypy'] + flags + files
+            if args.verbose:
+                print('running', ' '.join(sys.argv))
+            else:
+                print('running mypy', ' '.join(flags), '# with', len(files), 'files')
+            try:
+                if not args.dry_run:
+                    mypy_main('')
+            except SystemExit as err:
+                code = max(code, err.code)
+    if code:
+        print('--- exit status', code, '---')
+        sys.exit(code)
+
+
+if __name__ == '__main__':
+    main()