Move Python 2-only stubs to @python2 directory (#5660)

CONTRIBUTING.md

@@ -59,8 +59,10 @@ go into `stubs`. Each subdirectory there represents a PyPI distribution, and
 contains the following:
 * `METADATA.toml`, describing the package. See below for details.
 * Stubs (i.e. `*.pyi` files) for packages and modules that are shipped in the
-  source distribution. If the Python 2 version of the stubs must be kept
-  *separate*, they can be put in a `@python2` subdirectory.
+  source distribution.
+* If the stubs are either Python 2-only, or if the Python 2 and Python 3 stubs
+  are separate, the Python 2 stubs are put in a `@python2` subdirectory. In the
+  former case, there are no stubs or package directories on the top level.
 * (Rarely) some docs specific to a given type stub package in `README` file.
 
 When a third party stub is
@@ -89,8 +91,8 @@ supported:
   previous versions are still available on PyPI). Some legacy stubs are
   marked with version `0.1`, indicating that their supported version is
   unknown and needs to be updated.
-* `python2` (default: `False`) and `python3` (default: `True`): These fields
-  indicate whether a package supports Python 2, Python 3, or both.
+* `python2` (default: `false`): If set to `true`, the top-level stubs
+  support both Python 2 and Python 3.
 * `requires` (optional): A list of other stub packages that this package uses.
 * `extra_description` (optional): Can be used to add a custom description to
   the package's long description. It should be a multi-line string in

stubs/enum34/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/fb303/@python2/fb303/FacebookService.pyi

@@ -1,6 +1,6 @@
 from typing import Any, ContextManager
 
-from thrift.Thrift import TProcessor  # type: ignore
+TProcessor = Any  # actually thrift.Thrift.TProcessor
 
 fastbinary: Any
 

stubs/fb303/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/futures/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/ipaddress/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/kazoo/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/pathlib2/METADATA.toml

@@ -1,4 +1,2 @@
 version = "0.1"
-python2 = true
-python3 = false
 requires = []

stubs/pymssql/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

stubs/scribe/@python2/scribe/scribe.pyi

@@ -1,10 +1,11 @@
 from typing import Any
 
 import fb303.FacebookService
-from thrift.Thrift import TProcessor  # type: ignore  # We don't have thrift stubs in typeshed
 
 from .ttypes import *  # noqa: F403
 
+TProcessor = Any  # actually thrift.Thrift.TProcessor
+
 class Iface(fb303.FacebookService.Iface):
     def Log(self, messages): ...
 

stubs/scribe/METADATA.toml

@@ -1,4 +1,2 @@
 version = "0.1"
-python2 = true
-python3 = false
 requires = ["types-fb303"]

stubs/tornado/METADATA.toml

@@ -1,3 +1 @@
 version = "0.1"
-python2 = true
-python3 = false

tests/check_consistent.py

@@ -21,7 +21,7 @@ consistent_files = [
     {"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"},
     {"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"},
 ]
-metadata_keys = {"version", "python2", "python3", "requires", "extra_description", "obsolete_since"}
+metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since"}
 allowed_files = {"README.md"}
 
 
@@ -176,7 +176,6 @@ def check_metadata():
         for key in data:
             assert key in metadata_keys, f"Unexpected key {key} for {distribution}"
         assert isinstance(data.get("python2", False), bool), f"Invalid python2 value for {distribution}"
-        assert isinstance(data.get("python3", True), bool), f"Invalid python3 value for {distribution}"
         assert isinstance(data.get("requires", []), list), f"Invalid requires value for {distribution}"
         for dep in data.get("requires", []):
             assert isinstance(dep, str), f"Invalid dependency {dep} for {distribution}"

tests/mypy_test.py

@@ -18,6 +18,8 @@ import re
 import sys
 import toml
 import tempfile
+from glob import glob
+from pathlib import Path
 from typing import Dict, NamedTuple
 
 PY2_NAMESPACE = "@python2"
@@ -101,13 +103,18 @@ def parse_version(v_str):
 
 
 def is_supported(distribution, major):
-    with open(os.path.join(THIRD_PARTY_NAMESPACE, distribution, "METADATA.toml")) as f:
+    dist_path = Path(THIRD_PARTY_NAMESPACE, distribution)
+    with open(dist_path / "METADATA.toml") as f:
         data = dict(toml.loads(f.read()))
     if major == 2:
         # Python 2 is not supported by default.
-        return bool(data.get("python2", False))
+        return bool(data.get("python2", False)) or (dist_path / "@python2").exists()
     # Python 3 is supported by default.
-    return bool(data.get("python3", True))
+    return has_py3_stubs(dist_path)
+
+
+def has_py3_stubs(dist: Path) -> bool:
+    return len(glob(f"{dist}/*.pyi")) > 0 or len(glob(f"{dist}/[!@]*/__init__.pyi")) > 0
 
 
 def add_files(files, seen, root, name, args, exclude_list):

tests/pytype_exclude_list.txt

@@ -9,8 +9,6 @@ stdlib/builtins.pyi
 stdlib/typing.pyi
 
 # third_party stubs with constructs that pytype doesn't yet support:
-stubs/fb303/fb303/FacebookService.pyi
-stubs/scribe/scribe/scribe.pyi
 stubs/paramiko/paramiko/_winapi.pyi
 stubs/paramiko/paramiko/win_pageant.pyi
 

tests/stubtest_third_party.py

@@ -36,8 +36,8 @@ def run_stubtest(dist: Path) -> None:
     with open(dist / "METADATA.toml") as f:
         metadata = dict(toml.loads(f.read()))
 
-    # Ignore stubs that don't support Python 2
-    if not bool(metadata.get("python3", True)) or not has_py3_stubs(dist):
+    # Ignore stubs that don't support Python 3
+    if not has_py3_stubs(dist):
         return
 
     with tempfile.TemporaryDirectory() as tmp: