Use packaging to parse versions (#8314)

packaging is the standard tool for dealing with versions and
requirements. This allows us to easily support all PEP 508 specs.
packaging also has good error messages, so I don't create our own here.
This lets us get rid of our handrolled half parser and sanity checks.
I kept the whitespace consistency check, since that's just a formatting
lint and isn't testing the correctness of parsing (which we now
definitely do correctly).

.github/workflows/tests.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x
-      - run: pip install $(grep tomli== requirements-tests.txt)
+      - run: pip install -r requirements-tests.txt
       - run: ./tests/check_consistent.py
 
   new-syntax:
@@ -73,7 +73,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x
-      - run: pip install $(grep tomli== requirements-tests.txt) $(grep mypy== requirements-tests.txt) termcolor
+      - run: pip install -r requirements-tests.txt
       - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }}
 
   pyright:

CONTRIBUTING.md

@@ -80,16 +80,6 @@ terminal to install all non-pytype requirements:
 (.venv) > pip install -r requirements-tests.txt
 ```
 
-### Optional dependencies
-
-Several tests also have `termcolor` as an optional dependency. Installing this
-is not essential to run the tests, but can make the output of some of the tests
-slightly prettier and easier to read. To install `termcolor`, run:
-
-```
-pip install termcolor
-```
-
 ## Code formatting
 
 The code is formatted using `black` and `isort`.

requirements-tests.txt

@@ -9,3 +9,5 @@ flake8-pyi==22.5.1
 # must match .pre-commit-config.yaml
 isort==5.10.1
 tomli==1.2.2
+packaging==21.3
+termcolor

tests/README.md

@@ -13,11 +13,11 @@ objects at runtime.
 objects at runtime.
 
 To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
-in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.8+.
+in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.9+.
 
 ## mypy\_test.py
 
-This test requires Python 3.8+. Run using:
+Run using:
 ```
 (.venv3)$ python3 tests/mypy_test.py
 ```
@@ -37,7 +37,6 @@ for this test script.
 
 ## pytype\_test.py
 
-This test requires Python 3.7+.
 Note: this test cannot be run on Windows
 systems unless you are using Windows Subsystem for Linux.
 
@@ -73,7 +72,6 @@ python3 tests/check_consistent.py
 
 ## stubtest\_stdlib.py
 
-This test requires Python 3.6 or higher.
 Run using
 ```
 (.venv3)$ python3 tests/stubtest_stdlib.py
@@ -102,10 +100,8 @@ directly, with
 ```
 stubtest can also help you find things missing from the stubs.
 
-
 ## stubtest\_third\_party.py
 
-This test requires Python 3.7+.
 Run using
 ```
 (.venv3)$ python3 tests/stubtest_third_party.py

tests/check_consistent.py

@@ -7,8 +7,11 @@ from __future__ import annotations
 
 import os
 import re
+import sys
 
 import tomli
+from packaging.requirements import Requirement
+from packaging.version import Version
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "ignore_missing_stub"}}
@@ -104,49 +107,25 @@ def _find_stdlib_modules() -> set[str]:
     return modules
 
 
-def _strip_dep_version(dependency: str) -> tuple[str, str, str]:
-    dep_version_pos = len(dependency)
-    for pos, c in enumerate(dependency):
-        if c in "=<>":
-            dep_version_pos = pos
-            break
-    stripped = dependency[:dep_version_pos]
-    rest = dependency[dep_version_pos:]
-    if not rest:
-        return stripped, "", ""
-    number_pos = 0
-    for pos, c in enumerate(rest):
-        if c not in "=<>":
-            number_pos = pos
-            break
-    relation = rest[:number_pos]
-    version = rest[number_pos:]
-    return stripped, relation, version
-
-
 def check_metadata() -> None:
     for distribution in os.listdir("stubs"):
         with open(os.path.join("stubs", distribution, "METADATA.toml")) as f:
             data = tomli.loads(f.read())
         assert "version" in data, f"Missing version for {distribution}"
         version = data["version"]
-        msg = f"Unsupported Python version {version}"
+        msg = f"Unsupported version {repr(version)}"
         assert isinstance(version, str), msg
-        assert re.fullmatch(r"\d+(\.\d+)+|\d+(\.\d+)*\.\*", version), msg
+        # Check that the version parses
+        Version(version.removesuffix(".*"))
         for key in data:
             assert key in metadata_keys, f"Unexpected key {key} for {distribution}"
         assert isinstance(data.get("requires", []), list), f"Invalid requires value for {distribution}"
         for dep in data.get("requires", []):
-            assert isinstance(dep, str), f"Invalid dependency {dep} for {distribution}"
+            assert isinstance(dep, str), f"Invalid requirement {repr(dep)} for {distribution}"
             for space in " \t\n":
-                assert space not in dep, f"For consistency dependency should not have whitespace: {dep}"
-            assert ";" not in dep, f"Semicolons in dependencies are not supported, got {dep}"
-            stripped, relation, dep_version = _strip_dep_version(dep)
-            if relation:
-                assert relation in {"==", ">", ">=", "<", "<="}, f"Bad relation '{relation}' in dependency {dep}"
-                assert dep_version.count(".") <= 2, f"Bad version '{dep_version}' in dependency {dep}"
-                for part in dep_version.split("."):
-                    assert part.isnumeric(), f"Bad version '{part}' in dependency {dep}"
+                assert space not in dep, f"For consistency, requirement should not have whitespace: {dep}"
+            # Check that the requirement parses
+            Requirement(dep)
 
         assert set(data.get("tool", [])).issubset(tool_keys.keys()), f"Unrecognised tool for {distribution}"
         for tool, tk in tool_keys.items():
@@ -155,6 +134,7 @@ def check_metadata() -> None:
 
 
 if __name__ == "__main__":
+    assert sys.version_info >= (3, 9), "Python 3.9+ is required to run this test"
     check_stdlib()
     check_versions()
     check_stubs()