Re-organize directory structure (#4971)

See discussion in #2491

Co-authored-by: Ivan Levkivskyi <ilevkivskyi@dropbox.com>

tests/mypy_test.py

@@ -16,8 +16,14 @@ import argparse
 import os
 import re
 import sys
+import toml
 
-parser = argparse.ArgumentParser(description="Test runner for typeshed. " "Patterns are unanchored regexps on the full path.")
+PY2_NAMESPACE = "@python2"
+THIRD_PARTY_NAMESPACE = "stubs"
+
+parser = argparse.ArgumentParser(
+    description="Test runner for typeshed. Patterns are unanchored regexps on the full path."
+)
 parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
 parser.add_argument("-n", "--dry-run", action="store_true", help="Don't actually run mypy")
 parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern")
@@ -63,17 +69,53 @@ def match(fn, args, exclude_list):
     return True
 
 
-def libpath(major, minor):
-    versions = ["%d.%d" % (major, minor) for minor in reversed(range(minor + 1))]
-    versions.append(str(major))
-    versions.append("2and3")
-    paths = []
-    for v in versions:
-        for top in ["stdlib", "third_party"]:
-            p = os.path.join(top, v)
-            if os.path.isdir(p):
-                paths.append(p)
-    return paths
+def parse_versions(fname):
+    with open(fname) as f:
+        data = f.read().splitlines()
+    result = {}
+    for line in data:
+        # Allow having some comments or empty lines.
+        if not line.strip() or line.startswith("#"):
+            continue
+        mod, ver_str = line.split(": ")
+        assert ver_str.count(".") == 1
+        major, minor = ver_str.split(".")
+        result[mod] = (int(major), int(minor))
+    return result
+
+
+def is_supported(distribution, major):
+    with open(os.path.join(THIRD_PARTY_NAMESPACE, distribution, "METADATA.toml")) as f:
+        data = dict(toml.loads(f.read()))
+    if major == 2:
+        # Python 2 is not supported by default.
+        return bool(data.get("python2", False))
+    # Python 3 is supported by default.
+    return bool(data.get("python3", True))
+
+
+def add_files(files, seen, root, name, args, exclude_list):
+    """Add all files in package or module represented by 'name' located in 'root'."""
+    full = os.path.join(root, name)
+    mod, ext = os.path.splitext(name)
+    if ext in [".pyi", ".py"]:
+        if match(full, args, exclude_list):
+            seen.add(mod)
+            files.append(full)
+    elif (
+        os.path.isfile(os.path.join(full, "__init__.pyi")) or
+        os.path.isfile(os.path.join(full, "__init__.py"))
+    ):
+        for r, ds, fs in os.walk(full):
+            ds.sort()
+            fs.sort()
+            for f in fs:
+                m, x = os.path.splitext(f)
+                if x in [".pyi", ".py"]:
+                    fn = os.path.join(r, f)
+                    if match(fn, args, exclude_list):
+                        seen.add(mod)
+                        files.append(fn)
 
 
 def main():
@@ -88,7 +130,7 @@ def main():
         print("Cannot import mypy. Did you install it?")
         sys.exit(1)
 
-    versions = [(3, 9), (3, 8), (3, 7), (3, 6), (3, 5), (2, 7)]
+    versions = [(3, 9), (3, 8), (3, 7), (3, 6), (2, 7)]
     if args.python_version:
         versions = [v for v in versions if any(("%d.%d" % v).startswith(av) for av in args.python_version)]
         if not versions:
@@ -98,44 +140,58 @@ def main():
     code = 0
     runs = 0
     for major, minor in versions:
-        roots = libpath(major, minor)
         files = []
         seen = {"__builtin__", "builtins", "typing"}  # Always ignore these.
-        for root in roots:
-            names = os.listdir(root)
-            for name in names:
-                full = os.path.join(root, name)
-                mod, ext = os.path.splitext(name)
+
+        # First add standard library files.
+        if major == 2:
+            root = os.path.join("stdlib", PY2_NAMESPACE)
+            for name in os.listdir(root):
+                mod, _ = os.path.splitext(name)
                 if mod in seen or mod.startswith("."):
                     continue
-                if ext in [".pyi", ".py"]:
-                    if match(full, args, exclude_list):
-                        seen.add(mod)
-                        files.append(full)
-                elif os.path.isfile(os.path.join(full, "__init__.pyi")) or os.path.isfile(os.path.join(full, "__init__.py")):
-                    for r, ds, fs in os.walk(full):
-                        ds.sort()
-                        fs.sort()
-                        for f in fs:
-                            m, x = os.path.splitext(f)
-                            if x in [".pyi", ".py"]:
-                                fn = os.path.join(r, f)
-                                if match(fn, args, exclude_list):
-                                    seen.add(mod)
-                                    files.append(fn)
+                add_files(files, seen, root, name, args, exclude_list)
+        else:
+            supported_versions = parse_versions(os.path.join("stdlib", "VERSIONS"))
+            root = "stdlib"
+            for name in os.listdir(root):
+                if name == PY2_NAMESPACE or name == "VERSIONS":
+                    continue
+                mod, _ = os.path.splitext(name)
+                if supported_versions[mod] > (major, minor):
+                    continue
+                add_files(files, seen, root, name, args, exclude_list)
+
+        # Next add files for all third party distributions.
+        for distribution in os.listdir(THIRD_PARTY_NAMESPACE):
+            if not is_supported(distribution, major):
+                continue
+            if major == 2 and os.path.isdir(os.path.join(THIRD_PARTY_NAMESPACE, distribution, PY2_NAMESPACE)):
+                root = os.path.join(THIRD_PARTY_NAMESPACE, distribution, PY2_NAMESPACE)
+            else:
+                root = os.path.join(THIRD_PARTY_NAMESPACE, distribution)
+            for name in os.listdir(root):
+                if name == PY2_NAMESPACE:
+                    continue
+                mod, _ = os.path.splitext(name)
+                if mod in seen or mod.startswith("."):
+                    continue
+                add_files(files, seen, root, name, args, exclude_list)
+
         if files:
             runs += 1
-            flags = ["--python-version", "%d.%d" % (major, minor)]
-            flags.append("--strict-optional")
-            flags.append("--no-site-packages")
-            flags.append("--show-traceback")
-            flags.append("--no-implicit-optional")
-            flags.append("--disallow-any-generics")
-            flags.append("--disallow-subclassing-any")
-            # Setting custom typeshed dir prevents mypy from falling back to its bundled typeshed in
-            # case of stub deletions
-            flags.append("--custom-typeshed-dir")
-            flags.append(os.path.dirname(os.path.dirname(__file__)))
+            flags = [
+                "--python-version", "%d.%d" % (major, minor),
+                "--strict-optional",
+                "--no-site-packages",
+                "--show-traceback",
+                "--no-implicit-optional",
+                "--disallow-any-generics",
+                "--disallow-subclassing-any",
+                # Setting custom typeshed dir prevents mypy from falling back to its bundled
+                # typeshed in case of stub deletions
+                "--custom-typeshed-dir", os.path.dirname(os.path.dirname(__file__)),
+            ]
             if args.warn_unused_ignores:
                 flags.append("--warn-unused-ignores")
             if args.platform: