Split tool.stubtest.platforms metadata key (#13746)

Co-authored-by: Avasam <samuel.06@hotmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/daily.yml

@@ -96,7 +96,7 @@ jobs:
             PYTHON_EXECUTABLE="python"
           fi
 
-          $PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          $PYTHON_EXECUTABLE tests/stubtest_third_party.py --ci-platforms-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
 
   stub-uploader:
     name: stub_uploader tests

.github/workflows/stubtest_third_party.yml

@@ -85,7 +85,7 @@ jobs:
               PYTHON_EXECUTABLE="python"
             fi
 
-            $PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only $STUBS
+            $PYTHON_EXECUTABLE tests/stubtest_third_party.py --ci-platforms-only $STUBS
           else
             echo "Nothing to test"
           fi

CONTRIBUTING.md

@@ -213,18 +213,22 @@ This has the following keys:
   that need to be installed for stubtest to run successfully
 * `choco_dependencies` (default: `[]`): A list of Windows Chocolatey packages
   that need to be installed for stubtest to run successfully
-* `platforms` (default: `["linux"]`): A list of OSes on which to run stubtest.
-  Can contain `win32`, `linux`, and `darwin` values.
-  If not specified, stubtest is run only on `linux`.
-  Only add extra OSes to the test
-  if there are platform-specific branches in a stubs package.
+* `supported_platforms` (default: all platforms): A list of OSes on which
+  stubtest can be run. When a package is not platform-specific, this should
+  not be set. If the package is platform-specific, this should usually be set
+  to the supported platforms, unless stubtest is known to fail on a
+  specific platform.
+* `ci_platforms` (default: `["linux"]`): A list of OSes on which to run
+  stubtest as part of our continuous integration (CI) tests. Can contain
+  `win32`, `linux`, and `darwin` values. If not specified, stubtest is run
+  only on `linux`. Only add extra OSes to the test if there are
+  platform-specific branches in a stubs package.
 * `mypy_plugins` (default: `[]`): A list of Python modules to use as mypy plugins
 when running stubtest. For example: `mypy_plugins = ["mypy_django_plugin.main"]`
 * `mypy_plugins_config` (default: `{}`): A dictionary mapping plugin names to their
 configuration dictionaries for use by mypy plugins. For example:
 `mypy_plugins_config = {"django-stubs" = {"django_settings_module" = "@tests.django_settings"}}`
 
-
 `*_dependencies` are usually packages needed to `pip install` the implementation
 distribution.
 

lib/ts_utils/metadata.py

@@ -40,6 +40,7 @@ __all__ = [
     "read_stubtest_settings",
 ]
 
+DEFAULT_STUBTEST_PLATFORMS = ["linux"]
 
 _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
 # Some older websites have a bad pattern of using query params for navigation.
@@ -81,7 +82,8 @@ class StubtestSettings:
     choco_dependencies: list[str]
     extras: list[str]
     ignore_missing_stub: bool
-    platforms: list[str]
+    supported_platforms: list[str] | None  # None means all platforms
+    ci_platforms: list[str]
     stubtest_requirements: list[str]
     mypy_plugins: list[str]
     mypy_plugins_config: dict[str, dict[str, Any]]
@@ -105,7 +107,8 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     choco_dependencies: object = data.get("choco_dependencies", [])
     extras: object = data.get("extras", [])
     ignore_missing_stub: object = data.get("ignore_missing_stub", False)
-    specified_platforms: object = data.get("platforms", ["linux"])
+    supported_platforms: object = data.get("supported_platforms")
+    ci_platforms: object = data.get("ci_platforms", DEFAULT_STUBTEST_PLATFORMS)
     stubtest_requirements: object = data.get("stubtest_requirements", [])
     mypy_plugins: object = data.get("mypy_plugins", [])
     mypy_plugins_config: object = data.get("mypy_plugins_config", {})
@@ -114,7 +117,8 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     assert type(ignore_missing_stub) is bool
 
     # It doesn't work for type-narrowing if we use a for loop here...
-    assert _is_list_of_strings(specified_platforms)
+    assert supported_platforms is None or _is_list_of_strings(supported_platforms)
+    assert _is_list_of_strings(ci_platforms)
     assert _is_list_of_strings(apt_dependencies)
     assert _is_list_of_strings(brew_dependencies)
     assert _is_list_of_strings(choco_dependencies)
@@ -123,11 +127,16 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     assert _is_list_of_strings(mypy_plugins)
     assert _is_nested_dict(mypy_plugins_config)
 
-    unrecognised_platforms = set(specified_platforms) - _STUBTEST_PLATFORM_MAPPING.keys()
-    assert not unrecognised_platforms, f"Unrecognised platforms specified for {distribution!r}: {unrecognised_platforms}"
+    unrecognised_platforms = set(ci_platforms) - _STUBTEST_PLATFORM_MAPPING.keys()
+    assert not unrecognised_platforms, f"Unrecognised ci_platforms specified for {distribution!r}: {unrecognised_platforms}"
+
+    if supported_platforms is not None:
+        assert set(ci_platforms).issubset(
+            supported_platforms
+        ), f"ci_platforms must be a subset of supported_platforms for {distribution!r}"
 
     for platform, dep_key in _STUBTEST_PLATFORM_MAPPING.items():
-        if platform not in specified_platforms:
+        if platform not in ci_platforms:
             assert dep_key not in data, (
                 f"Stubtest is not run on {platform} in CI for {distribution!r}, "
                 f"but {dep_key!r} are specified in METADATA.toml"
@@ -140,7 +149,8 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
         choco_dependencies=choco_dependencies,
         extras=extras,
         ignore_missing_stub=ignore_missing_stub,
-        platforms=specified_platforms,
+        supported_platforms=supported_platforms,
+        ci_platforms=ci_platforms,
         stubtest_requirements=stubtest_requirements,
         mypy_plugins=mypy_plugins,
         mypy_plugins_config=mypy_plugins_config,
@@ -204,7 +214,8 @@ _KNOWN_METADATA_TOOL_FIELDS: Final = {
         "choco_dependencies",
         "extras",
         "ignore_missing_stub",
-        "platforms",
+        "supported_platforms",
+        "ci_platforms",
         "stubtest_requirements",
         "mypy_plugins",
         "mypy_plugins_config",

lib/ts_utils/utils.py

@@ -50,6 +50,10 @@ def print_info(message: str) -> None:
     print(colored(message, "blue"))
 
 
+def print_warning(message: str) -> None:
+    print(colored(message, "yellow"))
+
+
 def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None:
     error_split = error.split("\n")
     old, new = fix_path

stubs/JACK-Client/METADATA.toml

@@ -5,7 +5,7 @@ requires = ["numpy>=1.20", "types-cffi"]
 
 [tool.stubtest]
 # darwin and win32 are equivalent
-platforms = ["darwin", "linux"]
+ci_platforms = ["darwin", "linux"]
 apt_dependencies = ["libjack-dev"]
 brew_dependencies = ["jack"]
 # No need to install on the CI. Leaving here as information for Windows contributors.

stubs/PyScreeze/METADATA.toml

@@ -4,7 +4,7 @@ requires = ["Pillow>=10.3.0"]
 
 [tool.stubtest]
 # Linux has extra constants, win32 has different definitions
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]
 # PyScreeze has an odd setup.py file
 # that doesn't list Pillow as a dependency for py312+ yet:
 # https://github.com/asweigart/pyscreeze/blob/eeca245a135cf171c163b3691300138518efa64e/setup.py#L38-L46

stubs/RPi.GPIO/METADATA.toml

@@ -2,7 +2,9 @@ version = "0.7.*"
 upstream_repository = "https://sourceforge.net/p/raspberry-gpio-python/code/"
 
 [tool.stubtest]
-# When stubtest tries to import this module:
-# error: RPi.GPIO failed to import. RuntimeError: This module can only be run on a Raspberry Pi!
+# This package is only supported on Raspberry Pi hardware, which identifies
+# itself as 'linux'. When run on other hardware, it raises a RuntimeError:
+# RPi.GPIO failed to import. RuntimeError: This module can only be run on a Raspberry Pi!
 # https://sourceforge.net/p/raspberry-gpio-python/code/ci/08048dd1894a6b09a104557b6eaa6bb68b6baac5/tree/source/py_gpio.c#l1008
-skip = true
+supported_platforms = []
+ci_platforms = []

stubs/aiofiles/METADATA.toml

@@ -3,4 +3,4 @@ upstream_repository = "https://github.com/Tinche/aiofiles"
 
 [tool.stubtest]
 # linux and darwin are equivalent
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

stubs/antlr4-python3-runtime/METADATA.toml

@@ -2,4 +2,4 @@ version = "4.13.*"
 upstream_repository = "https://github.com/antlr/antlr4"
 
 [tool.stubtest]
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

stubs/cffi/METADATA.toml

@@ -4,4 +4,4 @@ requires = ["types-setuptools"]
 
 [tool.stubtest]
 # linux and darwin are mostly equivalent, except for a single `RTLD_DEEPBIND` variable
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

stubs/colorama/METADATA.toml

@@ -2,4 +2,4 @@ version = "0.4.*"
 upstream_repository = "https://github.com/tartley/colorama"
 
 [tool.stubtest]
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

stubs/gdb/METADATA.toml

@@ -11,5 +11,5 @@ extra_description = """\
 """
 
 [tool.stubtest]
-platforms = ["linux"]
+ci_platforms = ["linux"]
 apt_dependencies = ["gdb"]

stubs/gevent/METADATA.toml

@@ -5,7 +5,7 @@ requires = ["types-greenlet", "types-psutil"]
 [tool.stubtest]
 # Run stubtest on all platforms, since there is some platform specific stuff
 # especially in the stdlib module replacement
-platforms = ["linux", "darwin", "win32"]
+ci_platforms = ["linux", "darwin", "win32"]
 # for testing the ffi loop implementations on all platforms
 stubtest_requirements = ["cffi", "dnspython"]
 apt_dependencies = ["libev4", "libev-dev", "libuv1", "libuv1-dev"]

stubs/keyboard/METADATA.toml

@@ -6,4 +6,4 @@ upstream_repository = "https://github.com/boppreh/keyboard"
 # It's only by possible mouse buttons and event literal types.
 # As well as returning a tuple of int/long from keyboard.mouse.get_position
 # The "mouse" module is obsoleted by the "mouse" package.
-# platforms =
+# ci_platforms = ["linux"]

stubs/paramiko/METADATA.toml

@@ -5,4 +5,4 @@ requires = ["cryptography>=37.0.0"]
 
 [tool.stubtest]
 # linux and darwin are equivalent
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

stubs/psutil/METADATA.toml

@@ -2,4 +2,4 @@ version = "7.0.*"
 upstream_repository = "https://github.com/giampaolo/psutil"
 
 [tool.stubtest]
-platforms = ["darwin", "linux", "win32"]
+ci_platforms = ["darwin", "linux", "win32"]

stubs/pyaudio/METADATA.toml

@@ -4,6 +4,6 @@ version = "0.2.*"
 
 [tool.stubtest]
 # linux and win32 are equivalent
-platforms = ["darwin", "linux"]
+ci_platforms = ["darwin", "linux"]
 apt_dependencies = ["portaudio19-dev"]
 brew_dependencies = ["portaudio"]

stubs/pycurl/METADATA.toml

@@ -2,4 +2,4 @@ version = "7.45.6"
 upstream_repository = "https://github.com/pycurl/pycurl"
 
 [tool.stubtest]
-platforms = ["darwin", "linux", "win32"]
+ci_platforms = ["darwin", "linux", "win32"]

stubs/pynput/METADATA.toml

@@ -2,4 +2,4 @@ version = "~=1.8.1"
 upstream_repository = "https://github.com/moses-palmer/pynput"
 
 [tool.stubtest]
-platforms = ["darwin", "linux", "win32"]
+ci_platforms = ["darwin", "linux", "win32"]

stubs/pyperclip/METADATA.toml

@@ -2,5 +2,5 @@ version = "1.9.*"
 upstream_repository = "https://github.com/asweigart/pyperclip"
 
 [tool.stubtest]
-platforms = ["win32", "linux", "darwin"]
+ci_platforms = ["win32", "linux", "darwin"]
 apt_dependencies = ["xclip"]

stubs/pyserial/METADATA.toml

@@ -2,5 +2,5 @@ version = "3.5.*"
 upstream_repository = "https://github.com/pyserial/pyserial"
 
 [tool.stubtest]
-platforms = ["darwin", "linux", "win32"]
+ci_platforms = ["darwin", "linux", "win32"]
 extras = ["cp2110"]

stubs/pywin32/METADATA.toml

@@ -2,4 +2,5 @@ version = "311.*"
 upstream_repository = "https://github.com/mhammond/pywin32"
 
 [tool.stubtest]
-platforms = ["win32"]
+supported_platforms = ["win32"]
+ci_platforms = ["win32"]

stubs/setuptools/METADATA.toml

@@ -7,5 +7,5 @@ it is no longer included with `types-setuptools`.
 
 [tool.stubtest]
 # darwin is equivalent to linux for OS-specific methods
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]
 stubtest_requirements = ["tomli"]

stubs/uWSGI/METADATA.toml

@@ -12,4 +12,4 @@ extra_description = """\
 # Run stubtest on MacOS as well, to check that the
 # uWSGI-specific parts of stubtest_third_party.py
 # also work there
-platforms = ["linux", "darwin"]
+ci_platforms = ["linux", "darwin"]

stubs/waitress/METADATA.toml

@@ -3,4 +3,4 @@ upstream_repository = "https://github.com/Pylons/waitress"
 
 [tool.stubtest]
 # linux and darwin are equivalent
-platforms = ["linux", "win32"]
+ci_platforms = ["linux", "win32"]

tests/stubtest_third_party.py

@@ -28,12 +28,11 @@ from ts_utils.utils import (
     print_info,
     print_success_msg,
     print_time,
+    print_warning,
 )
 
 
-def run_stubtest(
-    dist: Path, *, verbose: bool = False, specified_platforms_only: bool = False, keep_tmp_dir: bool = False
-) -> bool:
+def run_stubtest(dist: Path, *, verbose: bool = False, ci_platforms_only: bool = False, keep_tmp_dir: bool = False) -> bool:
     """Run stubtest for a single distribution."""
 
     dist_name = dist.name
@@ -44,14 +43,16 @@ def run_stubtest(
 
     stubtest_settings = metadata.stubtest_settings
     if stubtest_settings.skip:
-        print(colored("skipping", "yellow"))
+        print(colored("skipping (skip = true)", "yellow"))
         return True
 
-    if sys.platform not in stubtest_settings.platforms:
-        if specified_platforms_only:
-            print(colored("skipping (platform not specified in METADATA.toml)", "yellow"))
-            return True
-        print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow"))
+    if stubtest_settings.supported_platforms is not None and sys.platform not in stubtest_settings.supported_platforms:
+        print(colored("skipping (platform not supported)", "yellow"))
+        return True
+
+    if ci_platforms_only and sys.platform not in stubtest_settings.ci_platforms:
+        print(colored("skipping (platform skipped in CI)", "yellow"))
+        return True
 
     if not metadata.requires_python.contains(PYTHON_VERSION):
         print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow"))
@@ -189,8 +190,13 @@ def run_stubtest(
             else:
                 print_time(time() - t)
                 print_success_msg()
+
+                if sys.platform not in stubtest_settings.ci_platforms:
+                    print_warning(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI")
+
                 if keep_tmp_dir:
                     print_info(f"Virtual environment kept at: {venv_dir}")
+
     finally:
         if not keep_tmp_dir:
             rmtree(venv_dir)
@@ -398,9 +404,9 @@ def main() -> NoReturn:
     parser.add_argument("--num-shards", type=int, default=1)
     parser.add_argument("--shard-index", type=int, default=0)
     parser.add_argument(
-        "--specified-platforms-only",
+        "--ci-platforms-only",
         action="store_true",
-        help="skip the test if the current platform is not specified in METADATA.toml/tool.stubtest.platforms",
+        help="skip the test if the current platform is not specified in METADATA.toml/tool.stubtest.ci-platforms",
     )
     parser.add_argument("--keep-tmp-dir", action="store_true", help="keep the temporary virtualenv")
     parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE)
@@ -417,7 +423,7 @@ def main() -> NoReturn:
             continue
         try:
             if not run_stubtest(
-                dist, verbose=args.verbose, specified_platforms_only=args.specified_platforms_only, keep_tmp_dir=args.keep_tmp_dir
+                dist, verbose=args.verbose, ci_platforms_only=args.ci_platforms_only, keep_tmp_dir=args.keep_tmp_dir
             ):
                 result = 1
         except NoSuchStubError as e: