Test third party stubs with stubtest (#5615)

tests/README.md

@@ -7,7 +7,7 @@ tests typeshed with [mypy](https://github.com/python/mypy/)
 [pyright](https://github.com/microsoft/pyright).
 - `tests/check_consistent.py` checks certain files in typeshed remain
 consistent with each other.
-- `tests/stubtest_test.py` checks stubs against the objects at runtime.
+- `tests/stubtest_stdlib.py` checks stubs against the objects at runtime.
 
 To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
 in the `CONTRIBUTING.md` document.
@@ -63,12 +63,12 @@ Run using:
 python3 tests/check_consistent.py
 ```
 
-## stubtest\_test.py
+## stubtest\_stdlib.py
 
 This test requires Python 3.6 or higher.
 Run using
 ```
-(.venv3)$ python3 tests/stubtest_test.py
+(.venv3)$ python3 tests/stubtest_stdlib.py
 ```
 
 This test compares the stdlib stubs against the objects at runtime. Because of
@@ -93,3 +93,14 @@ directly, with
   <third-party-module>
 ```
 stubtest can also help you find things missing from the stubs.
+
+
+## stubtest\_third\_party.py
+
+This test requires Python 3.6 or higher.
+Run using
+```
+(.venv3)$ python3 tests/stubtest_third_party.py
+```
+
+Similar to `stubtest_stdlib.py`, but tests the third party stubs.

tests/check_consistent.py

@@ -71,7 +71,7 @@ def check_stubs():
                 else:
                     assert name.isidentifier(), f"Bad file name '{entry}' in stubs"
             else:
-                if entry == "@python2":
+                if entry in ("@python2", "@tests"):
                     continue
                 assert_stubs_only(os.path.join("stubs", distribution, entry))
         if os.path.isdir(os.path.join("stubs", distribution, "@python2")):

tests/stubtest_stdlib.py

@@ -1,13 +1,10 @@
 #!/usr/bin/env python3
-"""Test typeshed using stubtest
+"""Test typeshed's stdlib using stubtest
 
 stubtest is a script in the mypy project that compares stubs to the actual objects at runtime.
 Note that therefore the output of stubtest depends on which Python version it is run with.
 In typeshed CI, we run stubtest with each currently supported Python minor version, except 2.7.
 
-We pin the version of mypy / stubtest we use in .travis.yml so changes to those don't break
-typeshed CI.
-
 """
 
 import subprocess

tests/stubtest_third_party.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Test typeshed's third party stubs using stubtest"""
+
+import argparse
+import functools
+import subprocess
+import sys
+import tempfile
+import toml
+import venv
+from pathlib import Path
+
+
+EXCLUDE_LIST = [
+    "Flask",  # fails when stubtest tries to stringify some object
+    "pyaudio",  # install failure locally
+    "backports",  # errors on python version
+    "pkg_resources",  # ???
+    "six",  # ???
+    "aiofiles",  # easily fixable, some platform specific difference between local and ci
+    "pycurl"  # install failure, missing libcurl
+]
+
+
+class StubtestFailed(Exception):
+    pass
+
+
+@functools.lru_cache()
+def get_mypy_req():
+    with open("requirements-tests-py3.txt") as f:
+        return next(line.strip() for line in f if "mypy" in line)
+
+
+def run_stubtest(dist: Path) -> None:
+    with open(dist / "METADATA.toml") as f:
+        metadata = dict(toml.loads(f.read()))
+
+    # Ignore stubs that don't support Python 2
+    if not bool(metadata.get("python3", True)):
+        return
+
+    with tempfile.TemporaryDirectory() as tmp:
+        venv_dir = Path(tmp)
+        venv.create(venv_dir, with_pip=True, clear=True)
+
+        pip_exe = str(venv_dir / "bin" / "pip")
+        python_exe = str(venv_dir / "bin" / "python")
+
+        dist_version = metadata.get("version")
+        if dist_version is None or dist_version == "0.1":
+            dist_req = dist.name
+        else:
+            dist_req = f"{dist.name}=={dist_version}.*"
+
+        # We need stubtest to be able to import the package, so install mypy into the venv
+        # Hopefully mypy continues to not need too many dependencies
+        # TODO: Maybe find a way to cache these in CI
+        dists_to_install = [dist_req, get_mypy_req()]
+        dists_to_install.extend(metadata.get("requires", []))
+        pip_cmd = [pip_exe, "install"] + dists_to_install
+        print(" ".join(pip_cmd), file=sys.stderr)
+        try:
+            subprocess.run(pip_cmd, check=True, capture_output=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Failed to install {dist.name}", file=sys.stderr)
+            print(e.stdout.decode(), file=sys.stderr)
+            print(e.stderr.decode(), file=sys.stderr)
+            raise
+
+        packages_to_check = [d.name for d in dist.iterdir() if d.is_dir() and d.name.isidentifier()]
+        modules_to_check = [d.stem for d in dist.iterdir() if d.is_file() and d.suffix == ".pyi"]
+        cmd = [
+            python_exe,
+            "-m",
+            "mypy.stubtest",
+            # Use --ignore-missing-stub, because if someone makes a correct addition, they'll need to
+            # also make a allowlist change and if someone makes an incorrect addition, they'll run into
+            # false negatives.
+            "--ignore-missing-stub",
+            # Use --custom-typeshed-dir in case we make linked changes to stdlib or _typeshed
+            "--custom-typeshed-dir",
+            str(dist.parent.parent),
+            *packages_to_check,
+            *modules_to_check,
+        ]
+        allowlist_path = dist / "@tests/stubtest_allowlist.txt"
+        if allowlist_path.exists():
+            cmd.extend(["--allowlist", str(allowlist_path)])
+
+        try:
+            print(f"MYPYPATH={dist}", " ".join(cmd), file=sys.stderr)
+            subprocess.run(cmd, env={"MYPYPATH": str(dist), "MYPY_FORCE_COLOR": "1"}, check=True)
+        except subprocess.CalledProcessError:
+            print(f"stubtest failed for {dist.name}", file=sys.stderr)
+            if not allowlist_path.exists():
+                print(
+                    "\n\nRe-running stubtest with --generate-allowlist. "
+                    f"Add the following to {allowlist_path}:\n",
+                    file=sys.stderr,
+                )
+                subprocess.run(cmd + ["--generate-allowlist"], env={"MYPYPATH": str(dist)})
+            raise StubtestFailed from None
+        else:
+            print(f"stubtest succeeded for {dist.name}", file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--num-shards", type=int, default=1)
+    parser.add_argument("--shard-index", type=int, default=0)
+    args = parser.parse_args()
+
+    typeshed_dir = Path(".").resolve()
+    dists = sorted((typeshed_dir / "stubs").iterdir())
+    for i, dist in enumerate(dists):
+        if i % args.num_shards != args.shard_index:
+            continue
+        if dist.name in EXCLUDE_LIST:
+            continue
+        run_stubtest(dist)
+
+
+if __name__ == "__main__":
+    main()

tests/stubtest_unused.py

@@ -20,7 +20,7 @@ def main() -> None:
 
 
 def run_stubtest() -> List[str]:
-    proc = subprocess.run([sys.executable, "tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    proc = subprocess.run([sys.executable, "tests/stubtest_stdlib.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     output = proc.stdout.decode("utf-8").splitlines()
     return [line[len(_UNUSED_NOTE) :].strip() for line in output if line.startswith(_UNUSED_NOTE)]