Add more consistency tests (#4983)

Follow up fo #4971

Co-authored-by: Ivan Levkivskyi <ilevkivskyi@dropbox.com>

.github/workflows/tests.yml

@@ -11,7 +11,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: ./tests/check_consistent.py
+      - run: |
+          pip install toml
+          ./tests/check_consistent.py
 
   flake8:
     name: Lint with flake8

tests/check_consistent.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 
-# For various reasons we need the contents of certain files to be
+# For security (and simplicity) reasons, only a limited kind of files can be
+# present in /stdlib and /stubs directories, see README for detail. Here we
+# verify these constraints.
+
+# In addition, for various reasons we need the contents of certain files to be
 # duplicated in two places, for example stdlib/@python2/builtins.pyi and
 # stdlib/@python2/__builtin__.pyi must be identical.  In the past we used
 # symlinks but that doesn't always work on Windows, so now you must
@@ -10,13 +14,72 @@
 import filecmp
 import os
 
+import toml
+
 consistent_files = [
     {"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"},
     {"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"},
 ]
 
 
-def main():
+def assert_stubs_only(directory):
+    """Check that given directory contains only valid stub files."""
+    top = directory.split(os.sep)[-1]
+    assert top.isidentifier(), f"Bad directory name: {top}"
+    for _, dirs, files in os.walk(directory):
+        for file in files:
+            name, ext = os.path.splitext(file)
+            assert name.isidentifier(), f"Files must be valid modules, got: {name}"
+            assert ext == ".pyi", f"Only stub flies allowed. Got: {file} in {directory}"
+        for subdir in dirs:
+            assert subdir.isidentifier(), f"Directories must be valid packages, got: {subdir}"
+
+
+def check_stdlib():
+    for entry in os.listdir("stdlib"):
+        if os.path.isfile(os.path.join("stdlib", entry)):
+            name, ext = os.path.splitext(entry)
+            if ext != ".pyi":
+                assert entry == "VERSIONS", f"Unexpected file in stdlib root: {entry}"
+            assert name.isidentifier(), "Bad file name in stdlib"
+        else:
+            if entry == "@python2":
+                continue
+            assert_stubs_only(os.path.join("stdlib", entry))
+    for entry in os.listdir("stdlib/@python2"):
+        if os.path.isfile(os.path.join("stdlib/@python2", entry)):
+            name, ext = os.path.splitext(entry)
+            assert name.isidentifier(), "Bad file name in stdlib"
+            assert ext == ".pyi", "Unexpected file in stdlib/@python2 root"
+        else:
+            assert_stubs_only(os.path.join("stdlib/@python2", entry))
+
+
+def check_stubs():
+    for distribution in os.listdir("stubs"):
+        assert not os.path.isfile(distribution), f"Only directories allowed in stubs, got {distribution}"
+        for entry in os.listdir(os.path.join("stubs", distribution)):
+            if os.path.isfile(os.path.join("stubs", distribution, entry)):
+                name, ext = os.path.splitext(entry)
+                if ext != ".pyi":
+                    assert entry in {"METADATA.toml", "README", "README.md", "README.rst"}, entry
+                else:
+                    assert name.isidentifier(), f"Bad file name '{entry}' in stubs"
+            else:
+                if entry == "@python2":
+                    continue
+                assert_stubs_only(os.path.join("stubs", distribution, entry))
+        if os.path.isdir(os.path.join("stubs", distribution, "@python2")):
+            for entry in os.listdir(os.path.join("stubs", distribution, "@python2")):
+                if os.path.isfile(os.path.join("stubs", distribution, "@python2", entry)):
+                    name, ext = os.path.splitext(entry)
+                    assert name.isidentifier(), f"Bad file name '{entry}' in stubs"
+                    assert ext == ".pyi", f"Unexpected file {entry} in @python2 stubs"
+                else:
+                    assert_stubs_only(os.path.join("stubs", distribution, "@python2", entry))
+
+
+def check_same_files():
     files = [os.path.join(root, file) for root, dir, files in os.walk(".") for file in files]
     no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link."
     for file in files:
@@ -34,5 +97,63 @@ def main():
                 )
 
 
+def check_versions():
+    versions = {}
+    with open("stdlib/VERSIONS") as f:
+        data = f.read().splitlines()
+    for line in data:
+        if not line or line.lstrip().startswith("#"):
+            continue
+        assert ": " in line, f"Bad line in VERSIONS: {line}"
+        module, version = line.split(": ")
+        msg = f"Unsupported Python version{version}"
+        assert version.count(".") == 1, msg
+        major, minor = version.split(".")
+        assert major in {"2", "3"}, msg
+        assert minor.isdigit(), msg
+        assert module not in versions, f"Duplicate module {module} in VERSIONS"
+        versions[module] = (int(major), int(minor))
+    modules = set()
+    for entry in os.listdir("stdlib"):
+        if entry == "@python2" or entry == "VERSIONS":
+            continue
+        if os.path.isfile(os.path.join("stdlib", entry)):
+            mod, _ = os.path.splitext(entry)
+            modules.add(mod)
+        else:
+            modules.add(entry)
+    extra = modules - set(versions)
+    assert not extra, f"Modules not in versions: {extra}"
+    extra = set(versions) - modules
+    assert not extra, f"Versions not in modules: {extra}"
+
+
+def check_metadata():
+    for distribution in os.listdir("stubs"):
+        with open(os.path.join("stubs", distribution, "METADATA.toml")) as f:
+            data = toml.loads(f.read())
+        assert "version" in data, f"Missing version for {distribution}"
+        version = data["version"]
+        msg = f"Unsupported Python version {version}"
+        assert version.count(".") == 1, msg
+        major, minor = version.split(".")
+        assert major.isdigit() and minor.isdigit(), msg
+        for key in data:
+            assert key in {
+                "version", "python2", "python3", "requires"
+            }, f"Unexpected key {key} for {distribution}"
+        assert isinstance(data.get("python2", False), bool), f"Invalid python2 value for {distribution}"
+        assert isinstance(data.get("python3", True), bool), f"Invalid python3 value for {distribution}"
+        assert isinstance(data.get("requires", []), list), f"Invalid requires value for {distribution}"
+        for dep in data.get("requires", []):
+            # TODO: add more validation here.
+            assert isinstance(dep, str), f"Invalid dependency {dep} for {distribution}"
+            assert dep.startswith("types-"), f"Only stub dependencies supported, got {dep}"
+
+
 if __name__ == "__main__":
-    main()
+    check_stdlib()
+    check_versions()
+    check_stubs()
+    check_metadata()
+    check_same_files()