Test third party stubs with stubtest (#5615)

tests/stubtest_stdlib.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Test typeshed's stdlib using stubtest
+
+stubtest is a script in the mypy project that compares stubs to the actual objects at runtime.
+Note that therefore the output of stubtest depends on which Python version it is run with.
+In typeshed CI, we run stubtest with each currently supported Python minor version, except 2.7.
+
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def run_stubtest(typeshed_dir: Path) -> int:
+    allowlist_dir = typeshed_dir / "tests" / "stubtest_allowlists"
+    version_allowlist = "py{}{}.txt".format(sys.version_info.major, sys.version_info.minor)
+    platform_allowlist = "{}.txt".format(sys.platform)
+    combined_allowlist = "{}-py{}{}.txt".format(sys.platform, sys.version_info.major, sys.version_info.minor)
+
+    ignore_unused_allowlist = "--ignore-unused-allowlist" in sys.argv[1:]
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "mypy.stubtest",
+        # Use --ignore-missing-stub, because if someone makes a correct addition, they'll need to
+        # also make a allowlist change and if someone makes an incorrect addition, they'll run into
+        # false negatives.
+        "--ignore-missing-stub",
+        "--check-typeshed",
+        "--custom-typeshed-dir",
+        str(typeshed_dir),
+        "--allowlist",
+        str(allowlist_dir / "py3_common.txt"),
+        "--allowlist",
+        str(allowlist_dir / version_allowlist),
+    ]
+    if ignore_unused_allowlist:
+        cmd += ["--ignore-unused-allowlist"]
+    if (allowlist_dir / platform_allowlist).exists():
+        cmd += [
+            "--allowlist",
+            str(allowlist_dir / platform_allowlist),
+        ]
+    if (allowlist_dir / combined_allowlist).exists():
+        cmd += [
+            "--allowlist",
+            str(allowlist_dir / combined_allowlist),
+        ]
+    if sys.version_info < (3, 10):
+        # As discussed in https://github.com/python/typeshed/issues/3693, we only aim for
+        # positional-only arg accuracy for the latest Python version.
+        cmd += ["--ignore-positional-only"]
+    try:
+        print(" ".join(cmd), file=sys.stderr)
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        print(
+            "\nNB: stubtest output depends on the Python version (and system) it is run with. "
+            "See README.md for more details.\n"
+            "NB: We only check positional-only arg accuracy for Python 3.10.\n"
+            "\nCommand run was: {}\n".format(" ".join(cmd)),
+            file=sys.stderr,
+        )
+        print("stubtest failed", file=sys.stderr)
+        return e.returncode
+    else:
+        print("stubtest succeeded", file=sys.stderr)
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run_stubtest(typeshed_dir=Path(".")))