diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py
index 5360054fa..f6148e6e3 100644
--- a/tests/parse_metadata.py
+++ b/tests/parse_metadata.py
@@ -20,6 +20,7 @@ from packaging.version import Version
 from utils import cache
 
 __all__ = [
+    "NoSuchStubError",
     "StubMetadata",
     "PackageDependencies",
     "StubtestSettings",
@@ -160,6 +161,10 @@ _KNOWN_METADATA_TOOL_FIELDS: Final = {
 _DIST_NAME_RE: Final = re.compile(r"^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$", re.IGNORECASE)
 
 
+class NoSuchStubError(ValueError):
+    """Raise NoSuchStubError to indicate that a stubs/{distribution} directory doesn't exist"""
+
+
 @cache
 def read_metadata(distribution: str) -> StubMetadata:
     """Return an object describing the metadata of a stub as given in the METADATA.toml file.
@@ -169,8 +174,11 @@ def read_metadata(distribution: str) -> StubMetadata:
     Use `read_dependencies` if you need to parse the dependencies
     given in the `requires` field, for example.
     """
-    with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
-        data: dict[str, object] = tomli.load(f)
+    try:
+        with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
+            data: dict[str, object] = tomli.load(f)
+    except FileNotFoundError:
+        raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None
 
     unknown_metadata_fields = data.keys() - _KNOWN_METADATA_FIELDS
     assert not unknown_metadata_fields, f"Unexpected keys in METADATA.toml for {distribution!r}: {unknown_metadata_fields}"
diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py
index 0a13050c5..af393c35f 100755
--- a/tests/stubtest_third_party.py
+++ b/tests/stubtest_third_party.py
@@ -12,13 +12,18 @@ from pathlib import Path
 from textwrap import dedent
 from typing import NoReturn
 
-from parse_metadata import get_recursive_requirements, read_metadata
+from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata
 from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg
 
 
-def run_stubtest(dist: Path, *, verbose: bool = False, specified_platforms_only: bool = False) -> bool:
+def run_stubtest(
+    dist: Path, *, parser: argparse.ArgumentParser, verbose: bool = False, specified_platforms_only: bool = False
+) -> bool:
     dist_name = dist.name
-    metadata = read_metadata(dist_name)
+    try:
+        metadata = read_metadata(dist_name)
+    except NoSuchStubError as e:
+        parser.error(str(e))
     print(f"{dist_name}... ", end="")
 
     stubtest_settings = metadata.stubtest_settings
@@ -110,6 +115,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_platforms_only:
             print_commands(dist, pip_cmd, stubtest_cmd, mypypath)
             print_command_output(e)
 
+            print("Python version: ", file=sys.stderr)
+            ret = subprocess.run([sys.executable, "-VV"], capture_output=True)
+            print_command_output(ret)
+
             print("Ran with the following environment:", file=sys.stderr)
             ret = subprocess.run([pip_exe, "freeze", "--all"], capture_output=True)
             print_command_output(ret)
@@ -253,7 +262,7 @@ def main() -> NoReturn:
     for i, dist in enumerate(dists):
         if i % args.num_shards != args.shard_index:
             continue
-        if not run_stubtest(dist, verbose=args.verbose, specified_platforms_only=args.specified_platforms_only):
+        if not run_stubtest(dist, parser=parser, verbose=args.verbose, specified_platforms_only=args.specified_platforms_only):
             result = 1
     sys.exit(result)