Fix and re-enable pytype_test (#4975)

For now, the test requires changes that have been merged into the pytype master branch but not yet released. I'll update requirements-tests-py3.txt again once I've cut a new release (hopefully later this week).

* Fix pytype_test.
* Re-enable pytype_test.
* Remove continue-on-error from pytype_test.
* Temporarily install pytype from its master branch, as the code to
  support the new directory structure hasn't been released yet.
* Set TYPESHED_HOME before using pytype.pytd.Typeshed().

.github/workflows/tests.yml

@@ -53,8 +53,6 @@ jobs:
           python-version: 3.6
       - run: pip install -r requirements-tests-py3.txt
       - run: ./tests/pytype_test.py
-        # TODO: re-enable when pytype supports new directory structure.
-        continue-on-error: true
 
   mypy:
     name: Run mypy against the stubs

requirements-tests-py3.txt

@@ -5,4 +5,4 @@ flake8==3.8.4
 flake8-bugbear==20.1.4
 flake8-pyi==20.10.0
 isort[pyproject]==5.5.3
-pytype>=2020.09.16
+git+https://github.com/google/pytype.git@master

tests/pytype_test.py

@@ -12,14 +12,14 @@ will also discover incorrect usage of imported modules.
 
 import argparse
 import os
-import re
 import sys
 import traceback
-from typing import List, Match, Optional, Sequence, Tuple
+from typing import List, Optional, Sequence, Tuple
 
 from pytype import config as pytype_config, load_pytd
+from pytype.pytd import typeshed
 
-TYPESHED_SUBDIRS = ["stdlib", "third_party"]
+TYPESHED_SUBDIRS = ["stdlib", "stubs"]
 
 
 TYPESHED_HOME = "TYPESHED_HOME"
@@ -33,6 +33,8 @@ def main() -> None:
     typeshed_location = args.typeshed_location or os.getcwd()
     subdir_paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS]
     check_subdirs_discoverable(subdir_paths)
+    old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
+    os.environ[TYPESHED_HOME] = typeshed_location
     files_to_test = determine_files_to_test(typeshed_location=typeshed_location, paths=args.files or subdir_paths)
     run_all_tests(
         files_to_test=files_to_test,
@@ -40,6 +42,10 @@ def main() -> None:
         print_stderr=args.print_stderr,
         dry_run=args.dry_run,
     )
+    if old_typeshed_home is UNSET:
+        del os.environ[TYPESHED_HOME]
+    else:
+        os.environ[TYPESHED_HOME] = old_typeshed_home
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -61,31 +67,6 @@ def create_parser() -> argparse.ArgumentParser:
     return parser
 
 
-class PathMatcher:
-    def __init__(self, patterns: Sequence[str]) -> None:
-        patterns = [re.escape(os.path.join(*x.split("/"))) for x in patterns]
-        self.matcher = re.compile(r"({})$".format("|".join(patterns))) if patterns else None
-
-    def search(self, path: str) -> Optional[Match[str]]:
-        if not self.matcher:
-            return None
-        return self.matcher.search(path)
-
-
-def load_exclude_list(typeshed_location: str) -> List[str]:
-    filename = os.path.join(typeshed_location, "tests", "pytype_exclude_list.txt")
-    skip_re = re.compile(r"^\s*([^\s#]+)\s*(?:#.*)?$")
-    skip = []
-
-    with open(filename) as f:
-        for line in f:
-            skip_match = skip_re.match(line)
-            if skip_match:
-                skip.append(skip_match.group(1))
-
-    return skip
-
-
 def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> Optional[str]:
     """Runs pytype, returning the stderr if any."""
     if python_version not in _LOADERS:
@@ -94,8 +75,6 @@ def run_pytype(*, filename: str, python_version: str, typeshed_location: str) ->
         loader = load_pytd.create_loader(options)
         _LOADERS[python_version] = (options, loader)
     options, loader = _LOADERS[python_version]
-    old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
-    os.environ[TYPESHED_HOME] = typeshed_location
     try:
         with pytype_config.verbosity_from(options):
             ast = loader.load_file(_get_module_name(filename), filename)
@@ -104,10 +83,6 @@ def run_pytype(*, filename: str, python_version: str, typeshed_location: str) ->
         stderr = traceback.format_exc()
     else:
         stderr = None
-    if old_typeshed_home is UNSET:
-        del os.environ[TYPESHED_HOME]
-    else:
-        os.environ[TYPESHED_HOME] = old_typeshed_home
     return stderr
 
 
@@ -125,7 +100,15 @@ def _get_relative(filename: str) -> str:
 
 def _get_module_name(filename: str) -> str:
     """Converts a filename {subdir}/m.n/module/foo to module.foo."""
-    return ".".join(_get_relative(filename).split(os.path.sep)[2:]).replace(".pyi", "").replace(".__init__", "")
+    parts = _get_relative(filename).split(os.path.sep)
+    if "@python2" in parts:
+        module_parts = parts[parts.index("@python2") + 1:]
+    elif parts[0] == "stdlib":
+        module_parts = parts[1:]
+    else:
+        assert parts[0] == "stubs"
+        module_parts = parts[2:]
+    return ".".join(module_parts).replace(".pyi", "").replace(".__init__", "")
 
 
 def _is_version(path: str, version: str) -> bool:
@@ -142,20 +125,17 @@ def determine_files_to_test(*, typeshed_location: str, paths: Sequence[str]) ->
     """Determine all files to test, checking if it's in the exclude list and which Python versions to use.
 
     Returns a list of pairs of the file path and Python version as an int."""
-    skipped = PathMatcher(load_exclude_list(typeshed_location))
     filenames = find_stubs_in_paths(paths)
+    ts = typeshed.Typeshed()
+    skipped = set(ts.read_blacklist())
     files = []
     for f in sorted(filenames):
         rel = _get_relative(f)
-        if skipped.search(rel):
+        if rel in skipped:
             continue
-        if _is_version(f, "2and3"):
-            files.append((f, 2))
-            files.append((f, 3))
-        elif _is_version(f, "2"):
-            files.append((f, 2))
-        elif _is_version(f, "3"):
-            files.append((f, 3))
+        versions = ts.get_python_major_versions(rel)
+        if versions:
+            files.extend((f, v) for v in versions)
         else:
             print("Unrecognized path: {}".format(f))
     return files
@@ -178,10 +158,11 @@ def run_all_tests(*, files_to_test: Sequence[Tuple[str, int]], typeshed_location
     total_tests = len(files_to_test)
     print("Testing files with pytype...")
     for i, (f, version) in enumerate(files_to_test):
+        python_version = "2.7" if version == 2 else "{0.major}.{0.minor}".format(sys.version_info)
         stderr = (
             run_pytype(
                 filename=f,
-                python_version="2.7" if version == 2 else "{0.major}.{0.minor}".format(sys.version_info),
+                python_version=python_version,
                 typeshed_location=typeshed_location,
             )
             if not dry_run
@@ -192,15 +173,15 @@ def run_all_tests(*, files_to_test: Sequence[Tuple[str, int]], typeshed_location
                 print(stderr)
             errors += 1
             stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1]
-            bad.append((_get_relative(f), stacktrace_final_line))
+            bad.append((_get_relative(f), python_version, stacktrace_final_line))
 
         runs = i + 1
         if runs % 25 == 0:
             print("  {:3d}/{:d} with {:3d} errors".format(runs, total_tests, errors))
 
     print("Ran pytype with {:d} pyis, got {:d} errors.".format(total_tests, errors))
-    for f, err in bad:
-        print("{}: {}".format(f, err))
+    for f, v, err in bad:
+        print("{} ({}): {}".format(f, v, err))
     if errors:
         raise SystemExit("\nRun again with --print-stderr to get the full stacktrace.")