Add a mypy self test (#4337)

Co-authored-by: Shantanu <hauntsaninja@users.noreply.github.com>
Co-authored-by: hauntsaninja <>

.github/workflows/tests.yml

@@ -75,6 +75,14 @@ jobs:
       - uses: actions/setup-python@v2
       - run: ./tests/mypy_selftest.py
 
+  mypy-test-suite:
+    name: Run the mypy test suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: ./tests/mypy_test_suite.py
+
   stubtest:
     name: Check stubs with stubtest
     runs-on: ${{ matrix.os }}

README.md

@@ -101,7 +101,9 @@ There are several tests:
 runs tests against [mypy](https://github.com/python/mypy/)
 - `tests/pytype_test.py` runs tests against
 [pytype](https://github.com/google/pytype/).
-- `tests/mypy_selftest.py` runs mypy's test suite using this version of
+- `tests/mypy_selftest.py` checks mypy's code base using this version of
+typeshed.
+- `tests/mypy_test_suite.py` runs mypy's test suite using this version of
 typeshed.
 - `tests/check_consistent.py` checks certain files in typeshed remain
 consistent with each other.
@@ -156,6 +158,13 @@ This test works similarly to `mypy_test.py`, except it uses `pytype`.
 This test requires Python 3.5 or higher; Python 3.6.1 or higher is recommended.
 Run using: `(.venv3)$ python3 tests/mypy_selftest.py`
 
+This test checks mypy's code base using mypy and typeshed code in this repo.
+
+### mypy_test_suite.py
+
+This test requires Python 3.5 or higher; Python 3.6.1 or higher is recommended.
+Run using: `(.venv3)$ python3 tests/mypy_test_suite.py`
+
 This test runs mypy's own test suite using the typeshed code in your repo. This
 will sometimes catch issues with incorrectly typed stubs, but is much slower
 than the other tests.

tests/mypy_selftest.py

@@ -1,28 +1,39 @@
 #!/usr/bin/env python3
-"""Script to run mypy's test suite against this version of typeshed."""
+"""Script to run mypy against its own code base."""
 
 from pathlib import Path
-import shutil
 import subprocess
 import sys
 import tempfile
 
+MYPY_VERSION = "0.790"
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     with tempfile.TemporaryDirectory() as tempdir:
         dirpath = Path(tempdir)
-        subprocess.run(['python2.7', '-m', 'pip', 'install', '--user', 'typing'], check=True)
-        subprocess.run(['git', 'clone', '--depth', '1', 'git://github.com/python/mypy',
-                        str(dirpath / 'mypy')], check=True)
-        subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', '-r',
-                        str(dirpath / 'mypy/test-requirements.txt')], check=True)
-        shutil.copytree('stdlib', str(dirpath / 'mypy/mypy/typeshed/stdlib'))
-        shutil.copytree('third_party', str(dirpath / 'mypy/mypy/typeshed/third_party'))
+        subprocess.run(
+            ["git", "clone", "--depth", "1", "git://github.com/python/mypy", str(dirpath)],
+            check=True,
+        )
         try:
-            subprocess.run(['pytest', '-n12'], cwd=str(dirpath / 'mypy'), check=True)
+            subprocess.run([sys.executable, "-m", "pip", "install", f"mypy=={MYPY_VERSION}"], check=True)
+            subprocess.run([sys.executable, "-m", "pip", "install", "-r", dirpath / "test-requirements.txt"], check=True)
+            subprocess.run(
+                [
+                    "mypy",
+                    "--config-file",
+                    dirpath / "mypy_self_check.ini",
+                    "--custom-typeshed-dir",
+                    ".",
+                    dirpath / "mypy",
+                    dirpath / "mypyc",
+                ],
+                check=True,
+            )
         except subprocess.CalledProcessError as e:
-            print('mypy tests failed', file=sys.stderr)
+            print("mypy self test failed", file=sys.stderr)
             sys.exit(e.returncode)
         else:
-            print('mypy tests succeeded', file=sys.stderr)
+            print("mypy self test succeeded", file=sys.stderr)
             sys.exit(0)

tests/mypy_test_suite.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""Script to run mypy's test suite against this version of typeshed."""
+
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+if __name__ == '__main__':
+    with tempfile.TemporaryDirectory() as tempdir:
+        dirpath = Path(tempdir)
+        subprocess.run(['python2.7', '-m', 'pip', 'install', '--user', 'typing'], check=True)
+        subprocess.run(['git', 'clone', '--depth', '1', 'git://github.com/python/mypy',
+                        str(dirpath / 'mypy')], check=True)
+        subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', '-r',
+                        str(dirpath / 'mypy/test-requirements.txt')], check=True)
+        shutil.copytree('stdlib', str(dirpath / 'mypy/mypy/typeshed/stdlib'))
+        shutil.copytree('third_party', str(dirpath / 'mypy/mypy/typeshed/third_party'))
+        try:
+            subprocess.run(['pytest', '-n12'], cwd=str(dirpath / 'mypy'), check=True)
+        except subprocess.CalledProcessError as e:
+            print('mypy tests failed', file=sys.stderr)
+            sys.exit(e.returncode)
+        else:
+            print('mypy tests succeeded', file=sys.stderr)
+            sys.exit(0)