Add a 'stubtest' flag to METADATA.toml (#6687)

Co-authored-by: Akuli <akuviljanen17@gmail.com>

CONTRIBUTING.md

@@ -177,6 +177,8 @@ supported:
   [removing obsolete third-party libraries](#third-party-library-removal-policy).
   It contains the first version of the corresponding library that ships
   its own `py.typed` file.
+* `stubtest` (default: `true`): Whether stubtest should be run against this
+  package. Please avoid setting this to `false`, and add a comment if you have to.
 
 The format of all `METADATA.toml` files can be checked by running
 `python3 ./tests/check_consistent.py`.

stubs/aiofiles/METADATA.toml

@@ -1,2 +1,3 @@
 version = "0.7.*"
 requires = []
+stubtest = false  # easily fixable, some platform differences between local and CI

stubs/pyaudio/METADATA.toml

@@ -1,2 +1,3 @@
 version = "0.2.*"
 python2 = true
+stubtest = false  # install fails locally

stubs/pycurl/METADATA.toml

@@ -1 +1,2 @@
 version = "7.44.*"
+stubtest = false  # install failure, missing libcurl

stubs/six/METADATA.toml

@@ -1,2 +1,3 @@
 version = "1.16.*"
 python2 = true
+stubtest = false  # TODO: figure out why this is needed

tests/check_consistent.py

@@ -18,7 +18,7 @@ import re
 import tomli
 
 consistent_files = [{"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"}]
-metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since"}
+metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since", "stubtest"}
 allowed_files = {"README.md"}
 
 

tests/stubtest_third_party.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 """Test typeshed's third party stubs using stubtest"""
 
+from __future__ import annotations
+
 import argparse
 import functools
 import subprocess
@@ -9,22 +11,10 @@ import tempfile
 import venv
 from glob import glob
 from pathlib import Path
-from typing import NoReturn
+from typing import Any, NoReturn
 
 import tomli
 
-EXCLUDE_LIST = [
-    "pyaudio",  # install failure locally
-    "backports",  # errors on python version
-    "six",  # ???
-    "aiofiles",  # easily fixable, some platform specific difference between local and ci
-    "pycurl",  # install failure, missing libcurl
-]
-
-
-class StubtestFailed(Exception):
-    pass
-
 
 @functools.lru_cache()
 def get_mypy_req():
@@ -32,13 +22,13 @@ def get_mypy_req():
         return next(line.strip() for line in f if "mypy" in line)
 
 
-def run_stubtest(dist: Path) -> None:
+def run_stubtest(dist: Path) -> bool:
     with open(dist / "METADATA.toml") as f:
         metadata = dict(tomli.loads(f.read()))
 
-    # Ignore stubs that don't support Python 3
-    if not has_py3_stubs(dist):
-        return
+    if not run_stubtest_for(metadata, dist):
+        print(f"Skipping stubtest for {dist.name}\n\n")
+        return True
 
     with tempfile.TemporaryDirectory() as tmp:
         venv_dir = Path(tmp)
@@ -61,7 +51,7 @@ def run_stubtest(dist: Path) -> None:
                 print(f"Failed to install requirements for {dist.name}", file=sys.stderr)
                 print(e.stdout.decode(), file=sys.stderr)
                 print(e.stderr.decode(), file=sys.stderr)
-                raise
+                return False
 
         # We need stubtest to be able to import the package, so install mypy into the venv
         # Hopefully mypy continues to not need too many dependencies
@@ -76,7 +66,7 @@ def run_stubtest(dist: Path) -> None:
             print(f"Failed to install {dist.name}", file=sys.stderr)
             print(e.stdout.decode(), file=sys.stderr)
             print(e.stderr.decode(), file=sys.stderr)
-            raise
+            return False
 
         packages_to_check = [d.name for d in dist.iterdir() if d.is_dir() and d.name.isidentifier()]
         modules_to_check = [d.stem for d in dist.iterdir() if d.is_file() and d.suffix == ".pyi"]
@@ -112,10 +102,15 @@ def run_stubtest(dist: Path) -> None:
                 print(f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path}:", file=sys.stderr)
                 subprocess.run(cmd + ["--generate-allowlist"], env={"MYPYPATH": str(dist)})
                 print("\n\n", file=sys.stderr)
-            raise StubtestFailed from None
+            return False
         else:
             print(f"stubtest succeeded for {dist.name}", file=sys.stderr)
         print("\n\n", file=sys.stderr)
+    return True
+
+
+def run_stubtest_for(metadata: dict[str, Any], dist: Path) -> bool:
+    return has_py3_stubs(dist) and metadata.get("stubtest", True)
 
 
 # Keep this in sync with mypy_test.py
@@ -140,11 +135,7 @@ def main() -> NoReturn:
     for i, dist in enumerate(dists):
         if i % args.num_shards != args.shard_index:
             continue
-        if dist.name in EXCLUDE_LIST:
-            continue
-        try:
-            run_stubtest(dist)
-        except StubtestFailed:
+        if not run_stubtest(dist):
             result = 1
     sys.exit(result)