From 68dc2d132b8e923672964aabeba4a42345a14849 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Mon, 21 Jun 2021 20:28:25 +0200 Subject: [PATCH] Move Python 2-only stubs to @python2 directory (#5660) --- CONTRIBUTING.md | 10 ++++++---- stubs/enum34/{ => @python2}/enum.pyi | 0 stubs/enum34/METADATA.toml | 2 -- .../fb303/{ => @python2}/fb303/FacebookService.pyi | 2 +- stubs/fb303/{ => @python2}/fb303/__init__.pyi | 0 stubs/fb303/METADATA.toml | 2 -- .../futures/{ => @python2}/concurrent/__init__.pyi | 0 .../{ => @python2}/concurrent/futures/__init__.pyi | 0 .../{ => @python2}/concurrent/futures/_base.pyi | 0 .../{ => @python2}/concurrent/futures/process.pyi | 0 .../{ => @python2}/concurrent/futures/thread.pyi | 0 stubs/futures/METADATA.toml | 2 -- stubs/ipaddress/{ => @python2}/ipaddress.pyi | 0 stubs/ipaddress/METADATA.toml | 2 -- stubs/kazoo/{ => @python2}/kazoo/__init__.pyi | 0 stubs/kazoo/{ => @python2}/kazoo/client.pyi | 0 stubs/kazoo/{ => @python2}/kazoo/exceptions.pyi | 0 .../kazoo/{ => @python2}/kazoo/recipe/__init__.pyi | 0 .../kazoo/{ => @python2}/kazoo/recipe/watchers.pyi | 0 stubs/kazoo/METADATA.toml | 2 -- stubs/pathlib2/{ => @python2}/pathlib2.pyi | 0 stubs/pathlib2/METADATA.toml | 2 -- stubs/pymssql/{ => @python2}/pymssql.pyi | 0 stubs/pymssql/METADATA.toml | 2 -- stubs/scribe/{ => @python2}/scribe/__init__.pyi | 0 stubs/scribe/{ => @python2}/scribe/scribe.pyi | 3 ++- stubs/scribe/{ => @python2}/scribe/ttypes.pyi | 0 stubs/scribe/METADATA.toml | 2 -- stubs/tornado/{ => @python2}/tornado/__init__.pyi | 0 stubs/tornado/{ => @python2}/tornado/concurrent.pyi | 0 stubs/tornado/{ => @python2}/tornado/gen.pyi | 0 stubs/tornado/{ => @python2}/tornado/httpclient.pyi | 0 stubs/tornado/{ => @python2}/tornado/httpserver.pyi | 0 stubs/tornado/{ => @python2}/tornado/httputil.pyi | 0 stubs/tornado/{ => @python2}/tornado/ioloop.pyi | 0 stubs/tornado/{ => @python2}/tornado/locks.pyi | 0 stubs/tornado/{ => @python2}/tornado/netutil.pyi | 0 stubs/tornado/{ => @python2}/tornado/process.pyi | 0 stubs/tornado/{ => @python2}/tornado/tcpserver.pyi | 0 stubs/tornado/{ => @python2}/tornado/testing.pyi | 0 stubs/tornado/{ => @python2}/tornado/util.pyi | 0 stubs/tornado/{ => @python2}/tornado/web.pyi | 0 stubs/tornado/METADATA.toml | 2 -- tests/check_consistent.py | 3 +-- tests/mypy_test.py | 13 ++++++++++--- tests/pytype_exclude_list.txt | 2 -- tests/stubtest_third_party.py | 4 ++-- 47 files changed, 22 insertions(+), 33 deletions(-) rename stubs/enum34/{ => @python2}/enum.pyi (100%) rename stubs/fb303/{ => @python2}/fb303/FacebookService.pyi (99%) rename stubs/fb303/{ => @python2}/fb303/__init__.pyi (100%) rename stubs/futures/{ => @python2}/concurrent/__init__.pyi (100%) rename stubs/futures/{ => @python2}/concurrent/futures/__init__.pyi (100%) rename stubs/futures/{ => @python2}/concurrent/futures/_base.pyi (100%) rename stubs/futures/{ => @python2}/concurrent/futures/process.pyi (100%) rename stubs/futures/{ => @python2}/concurrent/futures/thread.pyi (100%) rename stubs/ipaddress/{ => @python2}/ipaddress.pyi (100%) rename stubs/kazoo/{ => @python2}/kazoo/__init__.pyi (100%) rename stubs/kazoo/{ => @python2}/kazoo/client.pyi (100%) rename stubs/kazoo/{ => @python2}/kazoo/exceptions.pyi (100%) rename stubs/kazoo/{ => @python2}/kazoo/recipe/__init__.pyi (100%) rename stubs/kazoo/{ => @python2}/kazoo/recipe/watchers.pyi (100%) rename stubs/pathlib2/{ => @python2}/pathlib2.pyi (100%) rename stubs/pymssql/{ => @python2}/pymssql.pyi (100%) rename stubs/scribe/{ => @python2}/scribe/__init__.pyi (100%) rename stubs/scribe/{ => @python2}/scribe/scribe.pyi (92%) rename stubs/scribe/{ => @python2}/scribe/ttypes.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/__init__.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/concurrent.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/gen.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/httpclient.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/httpserver.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/httputil.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/ioloop.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/locks.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/netutil.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/process.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/tcpserver.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/testing.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/util.pyi (100%) rename stubs/tornado/{ => @python2}/tornado/web.pyi (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66dbed779..735321e56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,10 @@ go into `stubs`. Each subdirectory there represents a PyPI distribution, and contains the following: * `METADATA.toml`, describing the package. See below for details. * Stubs (i.e. `*.pyi` files) for packages and modules that are shipped in the - source distribution. If the Python 2 version of the stubs must be kept - *separate*, they can be put in a `@python2` subdirectory. + source distribution. +* If the stubs are either Python 2-only, or if the Python 2 and Python 3 stubs + are separate, the Python 2 stubs are put in a `@python2` subdirectory. In the + former case, there are no stubs or package directories on the top level. * (Rarely) some docs specific to a given type stub package in `README` file. When a third party stub is @@ -89,8 +91,8 @@ supported: previous versions are still available on PyPI). Some legacy stubs are marked with version `0.1`, indicating that their supported version is unknown and needs to be updated. -* `python2` (default: `False`) and `python3` (default: `True`): These fields - indicate whether a package supports Python 2, Python 3, or both. +* `python2` (default: `false`): If set to `true`, the top-level stubs + support both Python 2 and Python 3. * `requires` (optional): A list of other stub packages that this package uses. * `extra_description` (optional): Can be used to add a custom description to the package's long description. It should be a multi-line string in diff --git a/stubs/enum34/enum.pyi b/stubs/enum34/@python2/enum.pyi similarity index 100% rename from stubs/enum34/enum.pyi rename to stubs/enum34/@python2/enum.pyi diff --git a/stubs/enum34/METADATA.toml b/stubs/enum34/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/enum34/METADATA.toml +++ b/stubs/enum34/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/fb303/fb303/FacebookService.pyi b/stubs/fb303/@python2/fb303/FacebookService.pyi similarity index 99% rename from stubs/fb303/fb303/FacebookService.pyi rename to stubs/fb303/@python2/fb303/FacebookService.pyi index 19a6a8ec5..4a41a9abf 100644 --- a/stubs/fb303/fb303/FacebookService.pyi +++ b/stubs/fb303/@python2/fb303/FacebookService.pyi @@ -1,6 +1,6 @@ from typing import Any, ContextManager -from thrift.Thrift import TProcessor # type: ignore +TProcessor = Any # actually thrift.Thrift.TProcessor fastbinary: Any diff --git a/stubs/fb303/fb303/__init__.pyi b/stubs/fb303/@python2/fb303/__init__.pyi similarity index 100% rename from stubs/fb303/fb303/__init__.pyi rename to stubs/fb303/@python2/fb303/__init__.pyi diff --git a/stubs/fb303/METADATA.toml b/stubs/fb303/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/fb303/METADATA.toml +++ b/stubs/fb303/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/futures/concurrent/__init__.pyi b/stubs/futures/@python2/concurrent/__init__.pyi similarity index 100% rename from stubs/futures/concurrent/__init__.pyi rename to stubs/futures/@python2/concurrent/__init__.pyi diff --git a/stubs/futures/concurrent/futures/__init__.pyi b/stubs/futures/@python2/concurrent/futures/__init__.pyi similarity index 100% rename from stubs/futures/concurrent/futures/__init__.pyi rename to stubs/futures/@python2/concurrent/futures/__init__.pyi diff --git a/stubs/futures/concurrent/futures/_base.pyi b/stubs/futures/@python2/concurrent/futures/_base.pyi similarity index 100% rename from stubs/futures/concurrent/futures/_base.pyi rename to stubs/futures/@python2/concurrent/futures/_base.pyi diff --git a/stubs/futures/concurrent/futures/process.pyi b/stubs/futures/@python2/concurrent/futures/process.pyi similarity index 100% rename from stubs/futures/concurrent/futures/process.pyi rename to stubs/futures/@python2/concurrent/futures/process.pyi diff --git a/stubs/futures/concurrent/futures/thread.pyi b/stubs/futures/@python2/concurrent/futures/thread.pyi similarity index 100% rename from stubs/futures/concurrent/futures/thread.pyi rename to stubs/futures/@python2/concurrent/futures/thread.pyi diff --git a/stubs/futures/METADATA.toml b/stubs/futures/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/futures/METADATA.toml +++ b/stubs/futures/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/ipaddress/ipaddress.pyi b/stubs/ipaddress/@python2/ipaddress.pyi similarity index 100% rename from stubs/ipaddress/ipaddress.pyi rename to stubs/ipaddress/@python2/ipaddress.pyi diff --git a/stubs/ipaddress/METADATA.toml b/stubs/ipaddress/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/ipaddress/METADATA.toml +++ b/stubs/ipaddress/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/kazoo/kazoo/__init__.pyi b/stubs/kazoo/@python2/kazoo/__init__.pyi similarity index 100% rename from stubs/kazoo/kazoo/__init__.pyi rename to stubs/kazoo/@python2/kazoo/__init__.pyi diff --git a/stubs/kazoo/kazoo/client.pyi b/stubs/kazoo/@python2/kazoo/client.pyi similarity index 100% rename from stubs/kazoo/kazoo/client.pyi rename to stubs/kazoo/@python2/kazoo/client.pyi diff --git a/stubs/kazoo/kazoo/exceptions.pyi b/stubs/kazoo/@python2/kazoo/exceptions.pyi similarity index 100% rename from stubs/kazoo/kazoo/exceptions.pyi rename to stubs/kazoo/@python2/kazoo/exceptions.pyi diff --git a/stubs/kazoo/kazoo/recipe/__init__.pyi b/stubs/kazoo/@python2/kazoo/recipe/__init__.pyi similarity index 100% rename from stubs/kazoo/kazoo/recipe/__init__.pyi rename to stubs/kazoo/@python2/kazoo/recipe/__init__.pyi diff --git a/stubs/kazoo/kazoo/recipe/watchers.pyi b/stubs/kazoo/@python2/kazoo/recipe/watchers.pyi similarity index 100% rename from stubs/kazoo/kazoo/recipe/watchers.pyi rename to stubs/kazoo/@python2/kazoo/recipe/watchers.pyi diff --git a/stubs/kazoo/METADATA.toml b/stubs/kazoo/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/kazoo/METADATA.toml +++ b/stubs/kazoo/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/pathlib2/pathlib2.pyi b/stubs/pathlib2/@python2/pathlib2.pyi similarity index 100% rename from stubs/pathlib2/pathlib2.pyi rename to stubs/pathlib2/@python2/pathlib2.pyi diff --git a/stubs/pathlib2/METADATA.toml b/stubs/pathlib2/METADATA.toml index ae8bccc93..40f3c61c1 100644 --- a/stubs/pathlib2/METADATA.toml +++ b/stubs/pathlib2/METADATA.toml @@ -1,4 +1,2 @@ version = "0.1" -python2 = true -python3 = false requires = [] diff --git a/stubs/pymssql/pymssql.pyi b/stubs/pymssql/@python2/pymssql.pyi similarity index 100% rename from stubs/pymssql/pymssql.pyi rename to stubs/pymssql/@python2/pymssql.pyi diff --git a/stubs/pymssql/METADATA.toml b/stubs/pymssql/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/pymssql/METADATA.toml +++ b/stubs/pymssql/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/stubs/scribe/scribe/__init__.pyi b/stubs/scribe/@python2/scribe/__init__.pyi similarity index 100% rename from stubs/scribe/scribe/__init__.pyi rename to stubs/scribe/@python2/scribe/__init__.pyi diff --git a/stubs/scribe/scribe/scribe.pyi b/stubs/scribe/@python2/scribe/scribe.pyi similarity index 92% rename from stubs/scribe/scribe/scribe.pyi rename to stubs/scribe/@python2/scribe/scribe.pyi index 8988087a0..c1e447476 100644 --- a/stubs/scribe/scribe/scribe.pyi +++ b/stubs/scribe/@python2/scribe/scribe.pyi @@ -1,10 +1,11 @@ from typing import Any import fb303.FacebookService -from thrift.Thrift import TProcessor # type: ignore # We don't have thrift stubs in typeshed from .ttypes import * # noqa: F403 +TProcessor = Any # actually thrift.Thrift.TProcessor + class Iface(fb303.FacebookService.Iface): def Log(self, messages): ... diff --git a/stubs/scribe/scribe/ttypes.pyi b/stubs/scribe/@python2/scribe/ttypes.pyi similarity index 100% rename from stubs/scribe/scribe/ttypes.pyi rename to stubs/scribe/@python2/scribe/ttypes.pyi diff --git a/stubs/scribe/METADATA.toml b/stubs/scribe/METADATA.toml index c5e762bd8..5d762300d 100644 --- a/stubs/scribe/METADATA.toml +++ b/stubs/scribe/METADATA.toml @@ -1,4 +1,2 @@ version = "0.1" -python2 = true -python3 = false requires = ["types-fb303"] diff --git a/stubs/tornado/tornado/__init__.pyi b/stubs/tornado/@python2/tornado/__init__.pyi similarity index 100% rename from stubs/tornado/tornado/__init__.pyi rename to stubs/tornado/@python2/tornado/__init__.pyi diff --git a/stubs/tornado/tornado/concurrent.pyi b/stubs/tornado/@python2/tornado/concurrent.pyi similarity index 100% rename from stubs/tornado/tornado/concurrent.pyi rename to stubs/tornado/@python2/tornado/concurrent.pyi diff --git a/stubs/tornado/tornado/gen.pyi b/stubs/tornado/@python2/tornado/gen.pyi similarity index 100% rename from stubs/tornado/tornado/gen.pyi rename to stubs/tornado/@python2/tornado/gen.pyi diff --git a/stubs/tornado/tornado/httpclient.pyi b/stubs/tornado/@python2/tornado/httpclient.pyi similarity index 100% rename from stubs/tornado/tornado/httpclient.pyi rename to stubs/tornado/@python2/tornado/httpclient.pyi diff --git a/stubs/tornado/tornado/httpserver.pyi b/stubs/tornado/@python2/tornado/httpserver.pyi similarity index 100% rename from stubs/tornado/tornado/httpserver.pyi rename to stubs/tornado/@python2/tornado/httpserver.pyi diff --git a/stubs/tornado/tornado/httputil.pyi b/stubs/tornado/@python2/tornado/httputil.pyi similarity index 100% rename from stubs/tornado/tornado/httputil.pyi rename to stubs/tornado/@python2/tornado/httputil.pyi diff --git a/stubs/tornado/tornado/ioloop.pyi b/stubs/tornado/@python2/tornado/ioloop.pyi similarity index 100% rename from stubs/tornado/tornado/ioloop.pyi rename to stubs/tornado/@python2/tornado/ioloop.pyi diff --git a/stubs/tornado/tornado/locks.pyi b/stubs/tornado/@python2/tornado/locks.pyi similarity index 100% rename from stubs/tornado/tornado/locks.pyi rename to stubs/tornado/@python2/tornado/locks.pyi diff --git a/stubs/tornado/tornado/netutil.pyi b/stubs/tornado/@python2/tornado/netutil.pyi similarity index 100% rename from stubs/tornado/tornado/netutil.pyi rename to stubs/tornado/@python2/tornado/netutil.pyi diff --git a/stubs/tornado/tornado/process.pyi b/stubs/tornado/@python2/tornado/process.pyi similarity index 100% rename from stubs/tornado/tornado/process.pyi rename to stubs/tornado/@python2/tornado/process.pyi diff --git a/stubs/tornado/tornado/tcpserver.pyi b/stubs/tornado/@python2/tornado/tcpserver.pyi similarity index 100% rename from stubs/tornado/tornado/tcpserver.pyi rename to stubs/tornado/@python2/tornado/tcpserver.pyi diff --git a/stubs/tornado/tornado/testing.pyi b/stubs/tornado/@python2/tornado/testing.pyi similarity index 100% rename from stubs/tornado/tornado/testing.pyi rename to stubs/tornado/@python2/tornado/testing.pyi diff --git a/stubs/tornado/tornado/util.pyi b/stubs/tornado/@python2/tornado/util.pyi similarity index 100% rename from stubs/tornado/tornado/util.pyi rename to stubs/tornado/@python2/tornado/util.pyi diff --git a/stubs/tornado/tornado/web.pyi b/stubs/tornado/@python2/tornado/web.pyi similarity index 100% rename from stubs/tornado/tornado/web.pyi rename to stubs/tornado/@python2/tornado/web.pyi diff --git a/stubs/tornado/METADATA.toml b/stubs/tornado/METADATA.toml index ea72961c9..5f1541084 100644 --- a/stubs/tornado/METADATA.toml +++ b/stubs/tornado/METADATA.toml @@ -1,3 +1 @@ version = "0.1" -python2 = true -python3 = false diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 45dca946c..07bf26eb0 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -21,7 +21,7 @@ consistent_files = [ {"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"}, {"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"}, ] -metadata_keys = {"version", "python2", "python3", "requires", "extra_description", "obsolete_since"} +metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since"} allowed_files = {"README.md"} @@ -176,7 +176,6 @@ def check_metadata(): for key in data: assert key in metadata_keys, f"Unexpected key {key} for {distribution}" assert isinstance(data.get("python2", False), bool), f"Invalid python2 value for {distribution}" - assert isinstance(data.get("python3", True), bool), f"Invalid python3 value for {distribution}" assert isinstance(data.get("requires", []), list), f"Invalid requires value for {distribution}" for dep in data.get("requires", []): assert isinstance(dep, str), f"Invalid dependency {dep} for {distribution}" diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 8286d850e..5af495042 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -18,6 +18,8 @@ import re import sys import toml import tempfile +from glob import glob +from pathlib import Path from typing import Dict, NamedTuple PY2_NAMESPACE = "@python2" @@ -101,13 +103,18 @@ def parse_version(v_str): def is_supported(distribution, major): - with open(os.path.join(THIRD_PARTY_NAMESPACE, distribution, "METADATA.toml")) as f: + dist_path = Path(THIRD_PARTY_NAMESPACE, distribution) + with open(dist_path / "METADATA.toml") as f: data = dict(toml.loads(f.read())) if major == 2: # Python 2 is not supported by default. - return bool(data.get("python2", False)) + return bool(data.get("python2", False)) or (dist_path / "@python2").exists() # Python 3 is supported by default. - return bool(data.get("python3", True)) + return has_py3_stubs(dist_path) + + +def has_py3_stubs(dist: Path) -> bool: + return len(glob(f"{dist}/*.pyi")) > 0 or len(glob(f"{dist}/[!@]*/__init__.pyi")) > 0 def add_files(files, seen, root, name, args, exclude_list): diff --git a/tests/pytype_exclude_list.txt b/tests/pytype_exclude_list.txt index e707eb80c..717c8cffb 100644 --- a/tests/pytype_exclude_list.txt +++ b/tests/pytype_exclude_list.txt @@ -9,8 +9,6 @@ stdlib/builtins.pyi stdlib/typing.pyi # third_party stubs with constructs that pytype doesn't yet support: -stubs/fb303/fb303/FacebookService.pyi -stubs/scribe/scribe/scribe.pyi stubs/paramiko/paramiko/_winapi.pyi stubs/paramiko/paramiko/win_pageant.pyi diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index c60c92ac5..9dbc5a0d9 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -36,8 +36,8 @@ def run_stubtest(dist: Path) -> None: with open(dist / "METADATA.toml") as f: metadata = dict(toml.loads(f.read())) - # Ignore stubs that don't support Python 2 - if not bool(metadata.get("python3", True)) or not has_py3_stubs(dist): + # Ignore stubs that don't support Python 3 + if not has_py3_stubs(dist): return with tempfile.TemporaryDirectory() as tmp: