tests / scripts: blacken and isort (#4704)

I often run black and isort in typeshed root and then have to undo these
changes.

Co-authored-by: hauntsaninja <>

.flake8

@@ -14,6 +14,7 @@
 
 [flake8]
 per-file-ignores =
+  *.py: E203, W503
   *.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822
   # Since typing.pyi defines "overload" this is not recognized by flake8 as typing.overload.
   # Unfortunately, flake8 does not allow to "noqa" just a specific error inside the file itself.

scripts/migrate_script.py

@@ -7,9 +7,8 @@ import ast
 import os
 import os.path
 import shutil
-
 from dataclasses import dataclass
-from typing import Optional, List, Set, Tuple
+from typing import List, Optional, Set, Tuple
 
 # These names may be still discussed so I make them constants.
 STDLIB_NAMESPACE = "stdlib"
@@ -62,6 +61,7 @@ known_versions = {
 # The latter two are distinguished by is_dir flag.
 class PackageBase:
     """Common attributes for packages/modules"""
+
     path: str  # full initial path like stdlib/2and3/argparse.pyi
     is_dir: bool
 
@@ -79,6 +79,7 @@ class PackageBase:
 @dataclass
 class StdLibPackage(PackageBase):
     """Package/module in standard library."""
+
     path: str
     py_version: Optional[str]  # Can be omitted for Python 2 only packages.
     is_dir: bool
@@ -93,8 +94,7 @@ class ThirdPartyPackage(PackageBase):
     requires: List[str]  # distributions this depends on
 
 
-def add_stdlib_packages_from(subdir: str, packages: List[StdLibPackage],
-                             py_version: Optional[str]) -> None:
+def add_stdlib_packages_from(subdir: str, packages: List[StdLibPackage], py_version: Optional[str]) -> None:
     """Add standard library packages/modules from a given stdlib/xxx subdirectory.
 
     Append to packages list in-place, use py_version as the minimal supported version.
@@ -120,30 +120,27 @@ def collect_stdlib_packages() -> Tuple[List[StdLibPackage], List[StdLibPackage]]
     return stdlib, py2_stdlib
 
 
-def add_third_party_packages_from(subdir: str, packages: List[ThirdPartyPackage],
-                                  py2_compatible: bool, py3_compatible: bool) -> None:
+def add_third_party_packages_from(
+    subdir: str, packages: List[ThirdPartyPackage], py2_compatible: bool, py3_compatible: bool
+) -> None:
     """Add third party packages/modules from a given third_party/xxx subdirectory."""
     for name in os.listdir(subdir):
         path = os.path.join(subdir, name)
-        packages.append(ThirdPartyPackage(path, py2_compatible, py3_compatible,
-                                          requires=[], is_dir=os.path.isdir(path)))
+        packages.append(ThirdPartyPackage(path, py2_compatible, py3_compatible, requires=[], is_dir=os.path.isdir(path)))
 
 
 def collect_third_party_packages() -> Tuple[List[ThirdPartyPackage], List[ThirdPartyPackage]]:
     """Collect third party packages/modules from all current third_party/xxx sub-directories."""
     third_party: List[ThirdPartyPackage] = []
     py2_third_party: List[ThirdPartyPackage] = []
-    add_third_party_packages_from("third_party/3", third_party,
-                                  py2_compatible=False, py3_compatible=True)
-    add_third_party_packages_from("third_party/2and3", third_party,
-                                  py2_compatible=True, py3_compatible=True)
+    add_third_party_packages_from("third_party/3", third_party, py2_compatible=False, py3_compatible=True)
+    add_third_party_packages_from("third_party/2and3", third_party, py2_compatible=True, py3_compatible=True)
     # We special-case Python 2 for third party packages like six.
     subdir = "third_party/2"
     py3_packages = os.listdir("third_party/3")
     for name in os.listdir(subdir):
         path = os.path.join(subdir, name)
-        package = ThirdPartyPackage(path, py2_compatible=True, py3_compatible=False,
-                                    requires=[], is_dir=os.path.isdir(path))
+        package = ThirdPartyPackage(path, py2_compatible=True, py3_compatible=False, requires=[], is_dir=os.path.isdir(path))
         if name in py3_packages:
             # If there is a package with the same name in /2 and /3, we add the former to
             # a separate list, packages from there will be put into /python2 sub-directories.
@@ -168,19 +165,19 @@ def get_top_imported_names(file: str) -> Set[str]:
     for node in ast.walk(parsed):
         if isinstance(node, ast.Import):
             for name in node.names:
-                top_imported.add(name.name.split('.')[0])
+                top_imported.add(name.name.split(".")[0])
         elif isinstance(node, ast.ImportFrom):
             if node.level > 0:
                 # Relative imports always refer to the current package.
                 continue
             assert node.module
-            top_imported.add(node.module.split('.')[0])
+            top_imported.add(node.module.split(".")[0])
     return top_imported
 
 
-def populate_requirements(package: ThirdPartyPackage,
-                          stdlib: List[str], py2_stdlib: List[str],
-                          known_distributions: Set[str]) -> None:
+def populate_requirements(
+    package: ThirdPartyPackage, stdlib: List[str], py2_stdlib: List[str], known_distributions: Set[str]
+) -> None:
     """Generate requirements using imports found in a package."""
     assert not package.requires, "Populate must be called once"
     if not package.is_dir:
@@ -272,14 +269,12 @@ def generate_metadata(package: ThirdPartyPackage, py2_packages: List[str]) -> st
     if not package.py3_compatible:
         lines.append("python3 = false")
     if package.requires:
-        distributions = [f'"types-{package_to_distribution.get(dep, dep)}"'
-                         for dep in package.requires]
+        distributions = [f'"types-{package_to_distribution.get(dep, dep)}"' for dep in package.requires]
         lines.append(f"requires = [{', '.join(distributions)}]")
     return "\n".join(lines)
 
 
-def copy_third_party(packages: List[ThirdPartyPackage],
-                     py2_packages: List[ThirdPartyPackage]) -> None:
+def copy_third_party(packages: List[ThirdPartyPackage], py2_packages: List[ThirdPartyPackage]) -> None:
     """Refactor the third party part using collected metadata."""
     third_party_dir = os.path.join(OUTPUT_DIR, THIRD_PARTY_NAMESPACE)
     os.makedirs(third_party_dir, exist_ok=True)
@@ -325,8 +320,7 @@ def main() -> None:
     py2_stdlib_names += [package.name for package in stdlib if package.py_version == "2.7"]
 
     # Collect all known distributions (for sanity checks).
-    known_distributions = {package_to_distribution.get(package.name, package.name)
-                           for package in third_party + py2_third_party}
+    known_distributions = {package_to_distribution.get(package.name, package.name) for package in third_party + py2_third_party}
 
     # Compute dependencies between third party packages/modules to populate metadata.
     for package in third_party + py2_third_party:

tests/check_consistent.py

@@ -7,21 +7,21 @@
 # manually update both files, and this test verifies that they are
 # identical.  The list below indicates which sets of files must match.
 
-import os
 import filecmp
+import os
 
 consistent_files = [
-    {'stdlib/2/builtins.pyi', 'stdlib/2/__builtin__.pyi'},
-    {'stdlib/2and3/threading.pyi', 'stdlib/2and3/_dummy_threading.pyi'},
+    {"stdlib/2/builtins.pyi", "stdlib/2/__builtin__.pyi"},
+    {"stdlib/2and3/threading.pyi", "stdlib/2and3/_dummy_threading.pyi"},
 ]
 
 
 def main():
-    files = [os.path.join(root, file) for root, dir, files in os.walk('.') for file in files]
-    no_symlink = 'You cannot use symlinks in typeshed, please copy {} to its link.'
+    files = [os.path.join(root, file) for root, dir, files in os.walk(".") for file in files]
+    no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link."
     for file in files:
         _, ext = os.path.splitext(file)
-        if ext == '.pyi' and os.path.islink(file):
+        if ext == ".pyi" and os.path.islink(file):
             raise ValueError(no_symlink.format(file))
     for file1, *others in consistent_files:
         f1 = os.path.join(os.getcwd(), file1)
@@ -34,5 +34,5 @@ def main():
                 )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

tests/mypy_selftest.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 """Script to run mypy against its own code base."""
 
-from pathlib import Path
 import subprocess
 import sys
 import tempfile
+from pathlib import Path
 
 MYPY_VERSION = "0.790"
 

tests/mypy_test.py

@@ -12,26 +12,26 @@ Approach:
 5. Repeat steps 2-4 for other mypy runs (e.g. --py2)
 """
 
+import argparse
 import os
 import re
 import sys
-import argparse
 
-parser = argparse.ArgumentParser(description="Test runner for typeshed. "
-                                             "Patterns are unanchored regexps on the full path.")
-parser.add_argument('-v', '--verbose', action='count', default=0, help="More output")
-parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run mypy")
-parser.add_argument('-x', '--exclude', type=str, nargs='*', help="Exclude pattern")
-parser.add_argument('-p', '--python-version', type=str, nargs='*',
-                    help="These versions only (major[.minor])")
-parser.add_argument('--platform',
-                    help="Run mypy for a certain OS platform (defaults to sys.platform)")
-parser.add_argument('--warn-unused-ignores', action='store_true',
-                    help="Run mypy with --warn-unused-ignores "
-                    "(hint: only get rid of warnings that are "
-                    "unused for all platforms and Python versions)")
+parser = argparse.ArgumentParser(description="Test runner for typeshed. " "Patterns are unanchored regexps on the full path.")
+parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
+parser.add_argument("-n", "--dry-run", action="store_true", help="Don't actually run mypy")
+parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern")
+parser.add_argument("-p", "--python-version", type=str, nargs="*", help="These versions only (major[.minor])")
+parser.add_argument("--platform", help="Run mypy for a certain OS platform (defaults to sys.platform)")
+parser.add_argument(
+    "--warn-unused-ignores",
+    action="store_true",
+    help="Run mypy with --warn-unused-ignores "
+    "(hint: only get rid of warnings that are "
+    "unused for all platforms and Python versions)",
+)
 
-parser.add_argument('filter', type=str, nargs='*', help="Include pattern (default all)")
+parser.add_argument("filter", type=str, nargs="*", help="Include pattern (default all)")
 
 
 def log(args, *varargs):
@@ -41,36 +41,35 @@ def log(args, *varargs):
 
 def match(fn, args, exclude_list):
     if exclude_list.match(fn):
-        log(args, fn, 'exluded by exclude list')
+        log(args, fn, "exluded by exclude list")
         return False
     if not args.filter and not args.exclude:
-        log(args, fn, 'accept by default')
+        log(args, fn, "accept by default")
         return True
     if args.exclude:
         for f in args.exclude:
             if re.search(f, fn):
-                log(args, fn, 'excluded by pattern', f)
+                log(args, fn, "excluded by pattern", f)
                 return False
     if args.filter:
         for f in args.filter:
             if re.search(f, fn):
-                log(args, fn, 'accepted by pattern', f)
+                log(args, fn, "accepted by pattern", f)
                 return True
     if args.filter:
-        log(args, fn, 'rejected (no pattern matches)')
+        log(args, fn, "rejected (no pattern matches)")
         return False
-    log(args, fn, 'accepted (no exclude pattern matches)')
+    log(args, fn, "accepted (no exclude pattern matches)")
     return True
 
 
 def libpath(major, minor):
-    versions = ['%d.%d' % (major, minor)
-                for minor in reversed(range(minor + 1))]
+    versions = ["%d.%d" % (major, minor) for minor in reversed(range(minor + 1))]
     versions.append(str(major))
-    versions.append('2and3')
+    versions.append("2and3")
     paths = []
     for v in versions:
-        for top in ['stdlib', 'third_party']:
+        for top in ["stdlib", "third_party"]:
             p = os.path.join(top, v)
             if os.path.isdir(p):
                 paths.append(p)
@@ -81,8 +80,7 @@ def main():
     args = parser.parse_args()
 
     with open(os.path.join(os.path.dirname(__file__), "mypy_exclude_list.txt")) as f:
-        exclude_list = re.compile("(%s)$" % "|".join(
-            re.findall(r"^\s*([^\s#]+)\s*(?:#.*)?$", f.read(), flags=re.M)))
+        exclude_list = re.compile("(%s)$" % "|".join(re.findall(r"^\s*([^\s#]+)\s*(?:#.*)?$", f.read(), flags=re.M)))
 
     try:
         from mypy.main import main as mypy_main
@@ -92,8 +90,7 @@ def main():
 
     versions = [(3, 9), (3, 8), (3, 7), (3, 6), (3, 5), (2, 7)]
     if args.python_version:
-        versions = [v for v in versions
-                    if any(('%d.%d' % v).startswith(av) for av in args.python_version)]
+        versions = [v for v in versions if any(("%d.%d" % v).startswith(av) for av in args.python_version)]
         if not versions:
             print("--- no versions selected ---")
             sys.exit(1)
@@ -103,51 +100,50 @@ def main():
     for major, minor in versions:
         roots = libpath(major, minor)
         files = []
-        seen = {'__builtin__', 'builtins', 'typing'}  # Always ignore these.
+        seen = {"__builtin__", "builtins", "typing"}  # Always ignore these.
         for root in roots:
             names = os.listdir(root)
             for name in names:
                 full = os.path.join(root, name)
                 mod, ext = os.path.splitext(name)
-                if mod in seen or mod.startswith('.'):
+                if mod in seen or mod.startswith("."):
                     continue
-                if ext in ['.pyi', '.py']:
+                if ext in [".pyi", ".py"]:
                     if match(full, args, exclude_list):
                         seen.add(mod)
                         files.append(full)
-                elif (os.path.isfile(os.path.join(full, '__init__.pyi')) or
-                      os.path.isfile(os.path.join(full, '__init__.py'))):
+                elif os.path.isfile(os.path.join(full, "__init__.pyi")) or os.path.isfile(os.path.join(full, "__init__.py")):
                     for r, ds, fs in os.walk(full):
                         ds.sort()
                         fs.sort()
                         for f in fs:
                             m, x = os.path.splitext(f)
-                            if x in ['.pyi', '.py']:
+                            if x in [".pyi", ".py"]:
                                 fn = os.path.join(r, f)
                                 if match(fn, args, exclude_list):
                                     seen.add(mod)
                                     files.append(fn)
         if files:
             runs += 1
-            flags = ['--python-version', '%d.%d' % (major, minor)]
-            flags.append('--strict-optional')
-            flags.append('--no-site-packages')
-            flags.append('--show-traceback')
-            flags.append('--no-implicit-optional')
-            flags.append('--disallow-any-generics')
-            flags.append('--disallow-subclassing-any')
+            flags = ["--python-version", "%d.%d" % (major, minor)]
+            flags.append("--strict-optional")
+            flags.append("--no-site-packages")
+            flags.append("--show-traceback")
+            flags.append("--no-implicit-optional")
+            flags.append("--disallow-any-generics")
+            flags.append("--disallow-subclassing-any")
             if args.warn_unused_ignores:
-                flags.append('--warn-unused-ignores')
+                flags.append("--warn-unused-ignores")
             if args.platform:
-                flags.extend(['--platform', args.platform])
-            sys.argv = ['mypy'] + flags + files
+                flags.extend(["--platform", args.platform])
+            sys.argv = ["mypy"] + flags + files
             if args.verbose:
-                print("running", ' '.join(sys.argv))
+                print("running", " ".join(sys.argv))
             else:
-                print("running mypy", ' '.join(flags), "# with", len(files), "files")
+                print("running mypy", " ".join(flags), "# with", len(files), "files")
             try:
                 if not args.dry_run:
-                    mypy_main('', sys.stdout, sys.stderr)
+                    mypy_main("", sys.stdout, sys.stderr)
             except SystemExit as err:
                 code = max(code, err.code)
     if code:
@@ -158,5 +154,5 @@ def main():
         sys.exit(1)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

tests/mypy_test_suite.py

@@ -1,28 +1,27 @@
 #!/usr/bin/env python3
 """Script to run mypy's test suite against this version of typeshed."""
 
-from pathlib import Path
 import shutil
 import subprocess
 import sys
 import tempfile
+from pathlib import Path
 
-
-if __name__ == '__main__':
+if __name__ == "__main__":
     with tempfile.TemporaryDirectory() as tempdir:
         dirpath = Path(tempdir)
-        subprocess.run(['python2.7', '-m', 'pip', 'install', '--user', 'typing'], check=True)
-        subprocess.run(['git', 'clone', '--depth', '1', 'git://github.com/python/mypy',
-                        str(dirpath / 'mypy')], check=True)
-        subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', '-r',
-                        str(dirpath / 'mypy/test-requirements.txt')], check=True)
-        shutil.copytree('stdlib', str(dirpath / 'mypy/mypy/typeshed/stdlib'))
-        shutil.copytree('third_party', str(dirpath / 'mypy/mypy/typeshed/third_party'))
+        subprocess.run(["python2.7", "-m", "pip", "install", "--user", "typing"], check=True)
+        subprocess.run(["git", "clone", "--depth", "1", "git://github.com/python/mypy", str(dirpath / "mypy")], check=True)
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "-U", "-r", str(dirpath / "mypy/test-requirements.txt")], check=True
+        )
+        shutil.copytree("stdlib", str(dirpath / "mypy/mypy/typeshed/stdlib"))
+        shutil.copytree("third_party", str(dirpath / "mypy/mypy/typeshed/third_party"))
         try:
-            subprocess.run(['pytest', '-n12'], cwd=str(dirpath / 'mypy'), check=True)
+            subprocess.run(["pytest", "-n12"], cwd=str(dirpath / "mypy"), check=True)
         except subprocess.CalledProcessError as e:
-            print('mypy tests failed', file=sys.stderr)
+            print("mypy tests failed", file=sys.stderr)
             sys.exit(e.returncode)
         else:
-            print('mypy tests succeeded', file=sys.stderr)
+            print("mypy tests succeeded", file=sys.stderr)
             sys.exit(0)

tests/pytype_test.py

@@ -49,7 +49,11 @@ def create_parser() -> argparse.ArgumentParser:
         "--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered."
     )
     parser.add_argument(
-        "files", metavar="FILE", type=str, nargs="*", help="Files or directories to check. (Default: Check all files.)",
+        "files",
+        metavar="FILE",
+        type=str,
+        nargs="*",
+        help="Files or directories to check. (Default: Check all files.)",
     )
     return parser
 
@@ -82,10 +86,8 @@ def load_exclude_list(typeshed_location: str) -> List[str]:
 def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
     options = pytype_config.Options.create(
-        filename,
-        module_name=_get_module_name(filename),
-        parse_pyi=True,
-        python_version=python_version)
+        filename, module_name=_get_module_name(filename), parse_pyi=True, python_version=python_version
+    )
     old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
     os.environ[TYPESHED_HOME] = typeshed_location
     try:
@@ -162,13 +164,7 @@ def find_stubs_in_paths(paths: Sequence[str]) -> List[str]:
     return filenames
 
 
-def run_all_tests(
-    *,
-    files_to_test: Sequence[Tuple[str, int]],
-    typeshed_location: str,
-    print_stderr: bool,
-    dry_run: bool
-) -> None:
+def run_all_tests(*, files_to_test: Sequence[Tuple[str, int]], typeshed_location: str, print_stderr: bool, dry_run: bool) -> None:
     bad = []
     errors = 0
     total_tests = len(files_to_test)

tests/stubtest_test.py

@@ -10,9 +10,9 @@ typeshed CI.
 
 """
 
-from pathlib import Path
 import subprocess
 import sys
+from pathlib import Path
 
 
 def run_stubtest(typeshed_dir: Path) -> int:

tests/stubtest_unused.py

@@ -2,10 +2,10 @@
 
 # Runs stubtest and prints each unused whitelist entry with filename.
 
-from typing import List, Tuple
 import os.path
 import subprocess
 import sys
+from typing import List, Tuple
 
 _UNUSED_NOTE = "note: unused allowlist entry "
 _WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists")
@@ -22,11 +22,9 @@ def main() -> int:
 
 
 def run_stubtest() -> List[str]:
-    proc = subprocess.run(
-        ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    )
+    proc = subprocess.run(["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     output = proc.stdout.decode("utf-8").splitlines()
-    return [line[len(_UNUSED_NOTE):].strip() for line in output if line.startswith(_UNUSED_NOTE)]
+    return [line[len(_UNUSED_NOTE) :].strip() for line in output if line.startswith(_UNUSED_NOTE)]
 
 
 def unused_files(unused: str) -> List[Tuple[str, str]]: