VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2026-01-09 21:12:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Re-organize directory structure (#4971)

Browse Source 
See discussion in #2491

Co-authored-by: Ivan Levkivskyi <ilevkivskyi@dropbox.com>
This commit is contained in:
Ivan Levkivskyi
2021-01-27 12:00:39 +00:00
committed by GitHub
parent 869238e587
commit 16ae4c6120
1399 changed files with 601 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/check_consistent.py
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env python3
# For various reasons we need the contents of certain files to be
# duplicated in two places, for example stdlib/2/builtins.pyi and
# stdlib/2/__builtin__.pyi must be identical. In the past we used
# duplicated in two places, for example stdlib/@python2/builtins.pyi and
# stdlib/@python2/__builtin__.pyi must be identical. In the past we used
# symlinks but that doesn't always work on Windows, so now you must
# manually update both files, and this test verifies that they are
# identical. The list below indicates which sets of files must match.
@@ -11,8 +11,8 @@ import filecmp
import os
consistent_files = [
{"stdlib/2/builtins.pyi", "stdlib/2/__builtin__.pyi"},
{"stdlib/2and3/threading.pyi", "stdlib/2and3/_dummy_threading.pyi"},
{"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"},
{"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"},
]

12
tests/mypy_self_check.py
View File

@@ -1,12 +1,15 @@
#!/usr/bin/env python3
"""Script to run mypy against its own code base."""
import os
import subprocess
import sys
import tempfile
from pathlib import Path
MYPY_VERSION = "0.790"
# Use the current mypy version until a version that supports modular
# typeshed is released on PyPI.
MYPY_VERSION = "git+git://github.com/python/mypy"
if __name__ == "__main__":
@@ -16,8 +19,9 @@ if __name__ == "__main__":
["git", "clone", "--depth", "1", "git://github.com/python/mypy", dirpath],
check=True,
)
os.environ["MYPYPATH"] = str(dirpath)
try:
subprocess.run([sys.executable, "-m", "pip", "install", f"mypy=={MYPY_VERSION}"], check=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-U", MYPY_VERSION], check=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-r", dirpath / "test-requirements.txt"], check=True)
subprocess.run(
[
@@ -26,8 +30,8 @@ if __name__ == "__main__":
dirpath / "mypy_self_check.ini",
"--custom-typeshed-dir",
".",
dirpath / "mypy",
dirpath / "mypyc",
"-p", "mypy",
"-p", "mypyc",
],
check=True,
)

146
tests/mypy_test.py
View File

@@ -16,8 +16,14 @@ import argparse
import os
import re
import sys
import toml
parser = argparse.ArgumentParser(description="Test runner for typeshed. " "Patterns are unanchored regexps on the full path.")
PY2_NAMESPACE = "@python2"
THIRD_PARTY_NAMESPACE = "stubs"
parser = argparse.ArgumentParser(
description="Test runner for typeshed. Patterns are unanchored regexps on the full path."
)
parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
parser.add_argument("-n", "--dry-run", action="store_true", help="Don't actually run mypy")
parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern")
@@ -63,17 +69,53 @@ def match(fn, args, exclude_list):
return True
def libpath(major, minor):
versions = ["%d.%d" % (major, minor) for minor in reversed(range(minor + 1))]
versions.append(str(major))
versions.append("2and3")
paths = []
for v in versions:
for top in ["stdlib", "third_party"]:
p = os.path.join(top, v)
if os.path.isdir(p):
paths.append(p)
return paths
def parse_versions(fname):
with open(fname) as f:
data = f.read().splitlines()
result = {}
for line in data:
# Allow having some comments or empty lines.
if not line.strip() or line.startswith("#"):
continue
mod, ver_str = line.split(": ")
assert ver_str.count(".") == 1
major, minor = ver_str.split(".")
result[mod] = (int(major), int(minor))
return result
def is_supported(distribution, major):
with open(os.path.join(THIRD_PARTY_NAMESPACE, distribution, "METADATA.toml")) as f:
data = dict(toml.loads(f.read()))
if major == 2:
# Python 2 is not supported by default.
return bool(data.get("python2", False))
# Python 3 is supported by default.
return bool(data.get("python3", True))
def add_files(files, seen, root, name, args, exclude_list):
"""Add all files in package or module represented by 'name' located in 'root'."""
full = os.path.join(root, name)
mod, ext = os.path.splitext(name)
if ext in [".pyi", ".py"]:
if match(full, args, exclude_list):
seen.add(mod)
files.append(full)
elif (
os.path.isfile(os.path.join(full, "__init__.pyi")) or
os.path.isfile(os.path.join(full, "__init__.py"))
):
for r, ds, fs in os.walk(full):
ds.sort()
fs.sort()
for f in fs:
m, x = os.path.splitext(f)
if x in [".pyi", ".py"]:
fn = os.path.join(r, f)
if match(fn, args, exclude_list):
seen.add(mod)
files.append(fn)
def main():
@@ -88,7 +130,7 @@ def main():
print("Cannot import mypy. Did you install it?")
sys.exit(1)
versions = [(3, 9), (3, 8), (3, 7), (3, 6), (3, 5), (2, 7)]
versions = [(3, 9), (3, 8), (3, 7), (3, 6), (2, 7)]
if args.python_version:
versions = [v for v in versions if any(("%d.%d" % v).startswith(av) for av in args.python_version)]
if not versions:
@@ -98,44 +140,58 @@ def main():
code = 0
runs = 0
for major, minor in versions:
roots = libpath(major, minor)
files = []
seen = {"__builtin__", "builtins", "typing"} # Always ignore these.
for root in roots:
names = os.listdir(root)
for name in names:
full = os.path.join(root, name)
mod, ext = os.path.splitext(name)
# First add standard library files.
if major == 2:
root = os.path.join("stdlib", PY2_NAMESPACE)
for name in os.listdir(root):
mod, _ = os.path.splitext(name)
if mod in seen or mod.startswith("."):
continue
if ext in [".pyi", ".py"]:
if match(full, args, exclude_list):
seen.add(mod)
files.append(full)
elif os.path.isfile(os.path.join(full, "__init__.pyi")) or os.path.isfile(os.path.join(full, "__init__.py")):
for r, ds, fs in os.walk(full):
ds.sort()
fs.sort()
for f in fs:
m, x = os.path.splitext(f)
if x in [".pyi", ".py"]:
fn = os.path.join(r, f)
if match(fn, args, exclude_list):
seen.add(mod)
files.append(fn)
add_files(files, seen, root, name, args, exclude_list)
else:
supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS"))
root = "stdlib"
for name in os.listdir(root):
if name == PY2_NAMESPACE or name == "VERSIONS":
continue
mod, _ = os.path.splitext(name)
if supported_versions[mod] > (major, minor):
continue
add_files(files, seen, root, name, args, exclude_list)
# Next add files for all third party distributions.
for distribution in os.listdir(THIRD_PARTY_NAMESPACE):
if not is_supported(distribution, major):
continue
if major == 2 and os.path.isdir(os.path.join(THIRD_PARTY_NAMESPACE, distribution, PY2_NAMESPACE)):
root = os.path.join(THIRD_PARTY_NAMESPACE, distribution, PY2_NAMESPACE)
else:
root = os.path.join(THIRD_PARTY_NAMESPACE, distribution)
for name in os.listdir(root):
if name == PY2_NAMESPACE:
continue
mod, _ = os.path.splitext(name)
if mod in seen or mod.startswith("."):
continue
add_files(files, seen, root, name, args, exclude_list)
if files:
runs += 1
flags = ["--python-version", "%d.%d" % (major, minor)]
flags.append("--strict-optional")
flags.append("--no-site-packages")
flags.append("--show-traceback")
flags.append("--no-implicit-optional")
flags.append("--disallow-any-generics")
flags.append("--disallow-subclassing-any")
# Setting custom typeshed dir prevents mypy from falling back to its bundled typeshed in
# case of stub deletions
flags.append("--custom-typeshed-dir")
flags.append(os.path.dirname(os.path.dirname(__file__)))
flags = [
"--python-version", "%d.%d" % (major, minor),
"--strict-optional",
"--no-site-packages",
"--show-traceback",
"--no-implicit-optional",
"--disallow-any-generics",
"--disallow-subclassing-any",
# Setting custom typeshed dir prevents mypy from falling back to its bundled
# typeshed in case of stub deletions
"--custom-typeshed-dir", os.path.dirname(os.path.dirname(__file__)),
]
if args.warn_unused_ignores:
flags.append("--warn-unused-ignores")
if args.platform:

2
tests/mypy_test_suite.py
View File

@@ -16,8 +16,8 @@ if __name__ == "__main__":
subprocess.run(
[sys.executable, "-m", "pip", "install", "-U", "-r", dirpath / "mypy/test-requirements.txt"], check=True
)
shutil.rmtree(dirpath / "mypy/mypy/typeshed/stdlib")
shutil.copytree("stdlib", dirpath / "mypy/mypy/typeshed/stdlib")
shutil.copytree("third_party", dirpath / "mypy/mypy/typeshed/third_party")
try:
subprocess.run([sys.executable, "runtests.py", "typeshed-ci"], cwd=dirpath / "mypy", check=True)
except subprocess.CalledProcessError as e:

73
tests/pytype_exclude_list.txt
View File

@@ -2,47 +2,46 @@
# pytype has its own version of these files, and thus doesn't mind if it
# can't parse the typeshed version:
stdlib/2/__builtin__.pyi
stdlib/2/builtins.pyi
stdlib/2/typing.pyi
stdlib/3/builtins.pyi
stdlib/3/typing.pyi
stdlib/@python2/__builtin__.pyi
stdlib/@python2/builtins.pyi
stdlib/@python2/typing.pyi
stdlib/builtins.pyi
stdlib/typing.pyi
# third_party stubs with constructs that pytype doesn't yet support:
third_party/2/fb303/FacebookService.pyi
third_party/2/scribe/scribe.pyi
third_party/2and3/attr/__init__.pyi
third_party/2and3/attr/converters.pyi
third_party/2and3/attr/filters.pyi
third_party/2and3/attr/validators.pyi
third_party/2and3/paramiko/_winapi.pyi
third_party/2and3/paramiko/win_pageant.pyi
third_party/2and3/pynamodb/models.pyi
stubs/fb303/fb303/FacebookService.pyi
stubs/scribe/scribe/scribe.pyi
stubs/attrs/attr/__init__.pyi
stubs/attrs/attr/converters.pyi
stubs/attrs/attr/filters.pyi
stubs/attrs/attr/validators.pyi
stubs/paramiko/paramiko/_winapi.pyi
stubs/paramiko/paramiko/win_pageant.pyi
# _pb2.pyi have some constructs that break pytype
# Eg
# pytype.pyi.parser.ParseError: File: "/Users/nipunn/src/typeshed/third_party/2and3/google/protobuf/descriptor_pb2.pyi", line 195
# b"TypeValue = typing___NewType('TypeValue', builtin___int)"
third_party/2and3/google/protobuf/any_pb2.pyi
third_party/2and3/google/protobuf/api_pb2.pyi
third_party/2and3/google/protobuf/compiler/plugin_pb2.pyi
third_party/2and3/google/protobuf/descriptor.pyi
third_party/2and3/google/protobuf/descriptor_pb2.pyi
third_party/2and3/google/protobuf/duration_pb2.pyi
third_party/2and3/google/protobuf/empty_pb2.pyi
third_party/2and3/google/protobuf/field_mask_pb2.pyi
third_party/2and3/google/protobuf/internal/containers.pyi
third_party/2and3/google/protobuf/internal/enum_type_wrapper.pyi
third_party/2and3/google/protobuf/internal/extension_dict.pyi
third_party/2and3/google/protobuf/json_format.pyi
third_party/2and3/google/protobuf/message.pyi
third_party/2and3/google/protobuf/message_factory.pyi
third_party/2and3/google/protobuf/service.pyi
third_party/2and3/google/protobuf/source_context_pb2.pyi
third_party/2and3/google/protobuf/struct_pb2.pyi
third_party/2and3/google/protobuf/symbol_database.pyi
third_party/2and3/google/protobuf/timestamp_pb2.pyi
third_party/2and3/google/protobuf/type_pb2.pyi
third_party/2and3/google/protobuf/util/json_format_pb2.pyi
third_party/2and3/google/protobuf/util/json_format_proto3_pb2.pyi
third_party/2and3/google/protobuf/wrappers_pb2.pyi
stubs/protobuf/google/protobuf/any_pb2.pyi
stubs/protobuf/google/protobuf/api_pb2.pyi
stubs/protobuf/google/protobuf/compiler/plugin_pb2.pyi
stubs/protobuf/google/protobuf/descriptor.pyi
stubs/protobuf/google/protobuf/descriptor_pb2.pyi
stubs/protobuf/google/protobuf/duration_pb2.pyi
stubs/protobuf/google/protobuf/empty_pb2.pyi
stubs/protobuf/google/protobuf/field_mask_pb2.pyi
stubs/protobuf/google/protobuf/internal/containers.pyi
stubs/protobuf/google/protobuf/internal/enum_type_wrapper.pyi
stubs/protobuf/google/protobuf/internal/extension_dict.pyi
stubs/protobuf/google/protobuf/json_format.pyi
stubs/protobuf/google/protobuf/message.pyi
stubs/protobuf/google/protobuf/message_factory.pyi
stubs/protobuf/google/protobuf/service.pyi
stubs/protobuf/google/protobuf/source_context_pb2.pyi
stubs/protobuf/google/protobuf/struct_pb2.pyi
stubs/protobuf/google/protobuf/symbol_database.pyi
stubs/protobuf/google/protobuf/timestamp_pb2.pyi
stubs/protobuf/google/protobuf/type_pb2.pyi
stubs/protobuf/google/protobuf/util/json_format_pb2.pyi
stubs/protobuf/google/protobuf/util/json_format_proto3_pb2.pyi
stubs/protobuf/google/protobuf/wrappers_pb2.pyi