Fix stdlib/disutils testing (#9734)

tests/stubtest_allowlists/py310.txt

@@ -86,7 +86,6 @@ _collections_abc.MappingView.__class_getitem__
 _csv.Reader
 _csv.Writer
 bdb.Breakpoint.clearBreakpoints
-distutils.util.get_host_platform
 inspect.Signature.from_builtin  # Removed in 3.11, can add if someone needs this
 inspect.Signature.from_function  # Removed in 3.11, can add if someone needs this
 multiprocessing.managers.SharedMemoryServer.create
@@ -169,7 +168,3 @@ ast.ImportFrom.level  # None on the class, but never None on instances
 
 # White lies around defaults
 dataclasses.KW_ONLY
-
-# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410),
-# and the whole directory is going to be removed in 3.12 anyway
-distutils\..*

tests/stubtest_allowlists/py311.txt

@@ -139,7 +139,3 @@ typing._TypedDict.values
 
 # White lies around defaults
 dataclasses.KW_ONLY
-
-# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410),
-# and the whole directory is going to be removed in 3.12 anyway
-distutils\..*

tests/stubtest_allowlists/py37.txt

@@ -75,16 +75,6 @@ tkinter.Tk.__init__
 # Exists at runtime, but missing from stubs
 contextvars.ContextVar.__class_getitem__
 datetime.datetime_CAPI
-distutils.sysconfig.expand_makefile_vars
-distutils.sysconfig.get_python_version
-distutils.cygwinccompiler.RE_VERSION
-distutils.dist.command_re
-distutils.fancy_getopt.longopt_re
-distutils.fancy_getopt.neg_alias_re
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
 dummy_threading.Lock
 dummy_threading.RLock
 html.parser.HTMLParser.unescape
@@ -162,54 +152,3 @@ types.GetSetDescriptorType.__get__
 types.MemberDescriptorType.__get__
 types.MethodDescriptorType.__get__
 types.WrapperDescriptorType.__get__
-
-# Missing from distutils (deprecated, to be removed in 3.12)
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
-distutils.core.Command.dump_options
-distutils.core.Command.ensure_finalized
-distutils.core.Distribution.announce
-distutils.core.Distribution.common_usage
-distutils.core.Distribution.display_option_names
-distutils.core.Distribution.display_options
-distutils.core.Distribution.dump_option_dicts
-distutils.core.Distribution.find_config_files
-distutils.core.Distribution.get_command_packages
-distutils.core.Distribution.global_options
-distutils.core.Distribution.has_c_libraries
-distutils.core.Distribution.has_data_files
-distutils.core.Distribution.has_ext_modules
-distutils.core.Distribution.has_headers
-distutils.core.Distribution.has_modules
-distutils.core.Distribution.has_pure_modules
-distutils.core.Distribution.has_scripts
-distutils.core.Distribution.is_pure
-distutils.core.Distribution.negative_opt
-distutils.core.Distribution.parse_command_line
-distutils.core.Distribution.print_command_list
-distutils.core.Distribution.reinitialize_command
-distutils.core.Distribution.run_commands
-distutils.cygwinccompiler.is_cygwingcc
-distutils.dist.Distribution.announce
-distutils.dist.Distribution.common_usage
-distutils.dist.Distribution.display_option_names
-distutils.dist.Distribution.display_options
-distutils.dist.Distribution.dump_option_dicts
-distutils.dist.Distribution.find_config_files
-distutils.dist.Distribution.get_command_packages
-distutils.dist.Distribution.global_options
-distutils.dist.Distribution.has_c_libraries
-distutils.dist.Distribution.has_data_files
-distutils.dist.Distribution.has_ext_modules
-distutils.dist.Distribution.has_headers
-distutils.dist.Distribution.has_modules
-distutils.dist.Distribution.has_pure_modules
-distutils.dist.Distribution.has_scripts
-distutils.dist.Distribution.is_pure
-distutils.dist.Distribution.negative_opt
-distutils.dist.Distribution.parse_command_line
-distutils.dist.Distribution.print_command_list
-distutils.dist.Distribution.reinitialize_command
-distutils.dist.Distribution.run_commands

tests/stubtest_allowlists/py38.txt

@@ -35,10 +35,6 @@ collections.KeysView.__reversed__
 collections.ValuesView.__reversed__
 collections.Mapping.__reversed__  # Set to None at runtime for a better error message
 distutils.command.bdist_wininst  # see #6523
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
 dummy_threading.Condition.acquire
 dummy_threading.Condition.release
 dummy_threading.Event.isSet
@@ -97,13 +93,6 @@ tkinter.Tk.__init__
 # Exists at runtime, but missing from stubs
 contextvars.ContextVar.__class_getitem__
 datetime.datetime_CAPI
-distutils.sysconfig.expand_makefile_vars
-distutils.sysconfig.get_python_version
-distutils.util.get_host_platform
-distutils.cygwinccompiler.RE_VERSION
-distutils.dist.command_re
-distutils.fancy_getopt.longopt_re
-distutils.fancy_getopt.neg_alias_re
 dummy_threading.ExceptHookArgs
 dummy_threading.Lock
 dummy_threading.RLock
@@ -180,54 +169,3 @@ types.GetSetDescriptorType.__get__
 types.MemberDescriptorType.__get__
 types.MethodDescriptorType.__get__
 types.WrapperDescriptorType.__get__
-
-# Missing from distutils (deprecated, to be removed in 3.12)
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
-distutils.core.Command.dump_options
-distutils.core.Command.ensure_finalized
-distutils.core.Distribution.announce
-distutils.core.Distribution.common_usage
-distutils.core.Distribution.display_option_names
-distutils.core.Distribution.display_options
-distutils.core.Distribution.dump_option_dicts
-distutils.core.Distribution.find_config_files
-distutils.core.Distribution.get_command_packages
-distutils.core.Distribution.global_options
-distutils.core.Distribution.has_c_libraries
-distutils.core.Distribution.has_data_files
-distutils.core.Distribution.has_ext_modules
-distutils.core.Distribution.has_headers
-distutils.core.Distribution.has_modules
-distutils.core.Distribution.has_pure_modules
-distutils.core.Distribution.has_scripts
-distutils.core.Distribution.is_pure
-distutils.core.Distribution.negative_opt
-distutils.core.Distribution.parse_command_line
-distutils.core.Distribution.print_command_list
-distutils.core.Distribution.reinitialize_command
-distutils.core.Distribution.run_commands
-distutils.cygwinccompiler.is_cygwingcc
-distutils.dist.Distribution.announce
-distutils.dist.Distribution.common_usage
-distutils.dist.Distribution.display_option_names
-distutils.dist.Distribution.display_options
-distutils.dist.Distribution.dump_option_dicts
-distutils.dist.Distribution.find_config_files
-distutils.dist.Distribution.get_command_packages
-distutils.dist.Distribution.global_options
-distutils.dist.Distribution.has_c_libraries
-distutils.dist.Distribution.has_data_files
-distutils.dist.Distribution.has_ext_modules
-distutils.dist.Distribution.has_headers
-distutils.dist.Distribution.has_modules
-distutils.dist.Distribution.has_pure_modules
-distutils.dist.Distribution.has_scripts
-distutils.dist.Distribution.is_pure
-distutils.dist.Distribution.negative_opt
-distutils.dist.Distribution.parse_command_line
-distutils.dist.Distribution.print_command_list
-distutils.dist.Distribution.reinitialize_command
-distutils.dist.Distribution.run_commands

tests/stubtest_allowlists/py39.txt

@@ -35,10 +35,6 @@ collections.ValuesView.__reversed__
 contextlib.AbstractAsyncContextManager.__class_getitem__
 contextlib.AbstractContextManager.__class_getitem__
 distutils.command.bdist_wininst  # see #6523
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
 fractions.Fraction.__new__  # overload is too complicated for stubtest to resolve
 functools.cached_property.__set__  # Stub is a while lie; see comments in the stub
 gettext.install
@@ -97,13 +93,6 @@ collections.Awaitable.__class_getitem__
 collections.Container.__class_getitem__
 collections.Iterable.__class_getitem__
 collections.MappingView.__class_getitem__
-distutils.sysconfig.expand_makefile_vars
-distutils.sysconfig.get_python_version
-distutils.util.get_host_platform
-distutils.cygwinccompiler.RE_VERSION
-distutils.dist.command_re
-distutils.fancy_getopt.longopt_re
-distutils.fancy_getopt.neg_alias_re
 hmac.HMAC.digest_cons
 hmac.HMAC.inner
 hmac.HMAC.outer
@@ -175,54 +164,3 @@ types.GetSetDescriptorType.__get__
 types.MemberDescriptorType.__get__
 types.MethodDescriptorType.__get__
 types.WrapperDescriptorType.__get__
-
-# Missing from distutils (deprecated, to be removed in 3.12)
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
-distutils.core.Command.dump_options
-distutils.core.Command.ensure_finalized
-distutils.core.Distribution.announce
-distutils.core.Distribution.common_usage
-distutils.core.Distribution.display_option_names
-distutils.core.Distribution.display_options
-distutils.core.Distribution.dump_option_dicts
-distutils.core.Distribution.find_config_files
-distutils.core.Distribution.get_command_packages
-distutils.core.Distribution.global_options
-distutils.core.Distribution.has_c_libraries
-distutils.core.Distribution.has_data_files
-distutils.core.Distribution.has_ext_modules
-distutils.core.Distribution.has_headers
-distutils.core.Distribution.has_modules
-distutils.core.Distribution.has_pure_modules
-distutils.core.Distribution.has_scripts
-distutils.core.Distribution.is_pure
-distutils.core.Distribution.negative_opt
-distutils.core.Distribution.parse_command_line
-distutils.core.Distribution.print_command_list
-distutils.core.Distribution.reinitialize_command
-distutils.core.Distribution.run_commands
-distutils.cygwinccompiler.is_cygwingcc
-distutils.dist.Distribution.announce
-distutils.dist.Distribution.common_usage
-distutils.dist.Distribution.display_option_names
-distutils.dist.Distribution.display_options
-distutils.dist.Distribution.dump_option_dicts
-distutils.dist.Distribution.find_config_files
-distutils.dist.Distribution.get_command_packages
-distutils.dist.Distribution.global_options
-distutils.dist.Distribution.has_c_libraries
-distutils.dist.Distribution.has_data_files
-distutils.dist.Distribution.has_ext_modules
-distutils.dist.Distribution.has_headers
-distutils.dist.Distribution.has_modules
-distutils.dist.Distribution.has_pure_modules
-distutils.dist.Distribution.has_scripts
-distutils.dist.Distribution.is_pure
-distutils.dist.Distribution.negative_opt
-distutils.dist.Distribution.parse_command_line
-distutils.dist.Distribution.print_command_list
-distutils.dist.Distribution.reinitialize_command
-distutils.dist.Distribution.run_commands

tests/stubtest_allowlists/py3_common.txt

@@ -633,19 +633,8 @@ distutils.ccompiler.CCompiler.src_extensions
 distutils.ccompiler.CCompiler.static_lib_extension
 distutils.ccompiler.CCompiler.static_lib_format
 distutils.ccompiler.compiler_class
-distutils.cmd.Command.dump_options
-distutils.cmd.Command.ensure_finalized
 distutils.command.bdist
 distutils.command.install.*
-distutils.core.Distribution.finalize_options
-distutils.core.Distribution.get_command_class
-distutils.core.Distribution.get_command_list
-distutils.core.Distribution.handle_display_options
-distutils.core.Distribution.print_commands
-distutils.core.Distribution.run_command
-distutils.cygwinccompiler.CONFIG_H_NOTOK
-distutils.cygwinccompiler.CONFIG_H_OK
-distutils.cygwinccompiler.CONFIG_H_UNCERTAIN
 distutils.cygwinccompiler.CygwinCCompiler.compiler_type
 distutils.cygwinccompiler.CygwinCCompiler.exe_extension
 distutils.cygwinccompiler.CygwinCCompiler.obj_extension
@@ -654,22 +643,12 @@ distutils.cygwinccompiler.CygwinCCompiler.shared_lib_format
 distutils.cygwinccompiler.CygwinCCompiler.static_lib_extension
 distutils.cygwinccompiler.CygwinCCompiler.static_lib_format
 distutils.cygwinccompiler.Mingw32CCompiler.compiler_type
-distutils.cygwinccompiler.check_config_h
-distutils.cygwinccompiler.get_msvcr
-distutils.cygwinccompiler.get_versions
 distutils.dir_util.ensure_relative
-distutils.dist.Distribution.finalize_options
-distutils.dist.Distribution.get_command_class
-distutils.dist.Distribution.get_command_list
-distutils.dist.Distribution.handle_display_options
-distutils.dist.Distribution.print_commands
-distutils.dist.Distribution.run_command
 distutils.dist.DistributionMetadata.set_classifiers
 distutils.dist.DistributionMetadata.set_keywords
 distutils.dist.DistributionMetadata.set_platforms
 distutils.dist.fix_help_options
 distutils.extension.read_setup_file
-distutils.fancy_getopt.WS_TRANS
 distutils.fancy_getopt.FancyGetopt.add_option
 distutils.fancy_getopt.FancyGetopt.get_attr_name
 distutils.fancy_getopt.FancyGetopt.has_option
@@ -677,9 +656,6 @@ distutils.fancy_getopt.FancyGetopt.print_help
 distutils.fancy_getopt.FancyGetopt.set_aliases
 distutils.fancy_getopt.FancyGetopt.set_negative_aliases
 distutils.fancy_getopt.FancyGetopt.set_option_table
-distutils.fancy_getopt.longopt_pat
-distutils.fancy_getopt.longopt_xlate
-distutils.fancy_getopt.translate_longopt
 distutils.msvccompiler.MSVCCompiler.compiler_type
 distutils.msvccompiler.MSVCCompiler.exe_extension
 distutils.msvccompiler.MSVCCompiler.executables
@@ -698,13 +674,9 @@ distutils.msvccompiler.get_build_version
 distutils.msvccompiler.normalize_and_reduce_paths
 distutils.msvccompiler.read_keys
 distutils.msvccompiler.read_values
-distutils.sysconfig.BASE_EXEC_PREFIX
-distutils.sysconfig.BASE_PREFIX
 distutils.sysconfig.build_flags
 distutils.sysconfig.parse_config_h
 distutils.sysconfig.parse_makefile
-distutils.sysconfig.project_base
-distutils.sysconfig.python_build
 distutils.text_file.TextFile.default_options
 distutils.text_file.TextFile.error
 distutils.text_file.TextFile.gen_error

tests/stubtest_stdlib.py

@@ -6,55 +6,72 @@ Note that therefore the output of stubtest depends on which Python version it is
 In typeshed CI, we run stubtest with each currently supported Python minor version.
 
 """
+from __future__ import annotations
 
 import subprocess
 import sys
+import tempfile
 from pathlib import Path
 
+from utils import get_mypy_req, make_venv, print_error
+
 
 def run_stubtest(typeshed_dir: Path) -> int:
     allowlist_dir = typeshed_dir / "tests" / "stubtest_allowlists"
     version_allowlist = f"py{sys.version_info.major}{sys.version_info.minor}.txt"
     platform_allowlist = f"{sys.platform}.txt"
     combined_allowlist = f"{sys.platform}-py{sys.version_info.major}{sys.version_info.minor}.txt"
+    with tempfile.TemporaryDirectory() as tmp:
+        venv_dir = Path(tmp)
+        try:
+            pip_exe, python_exe = make_venv(venv_dir)
+        except Exception:
+            print_error("fail")
+            raise
 
-    cmd = [
-        sys.executable,
-        "-m",
-        "mypy.stubtest",
-        "--check-typeshed",
-        "--custom-typeshed-dir",
-        str(typeshed_dir),
-        "--allowlist",
-        str(allowlist_dir / "py3_common.txt"),
-        "--allowlist",
-        str(allowlist_dir / version_allowlist),
-    ]
-    if (allowlist_dir / platform_allowlist).exists():
-        cmd += ["--allowlist", str(allowlist_dir / platform_allowlist)]
-    if (allowlist_dir / combined_allowlist).exists():
-        cmd += ["--allowlist", str(allowlist_dir / combined_allowlist)]
-    if sys.version_info < (3, 10):
-        # As discussed in https://github.com/python/typeshed/issues/3693, we only aim for
-        # positional-only arg accuracy for the latest Python version.
-        cmd += ["--ignore-positional-only"]
-    print(" ".join(cmd), file=sys.stderr)
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError as e:
-        print(
-            "\nNB: stubtest output depends on the Python version (and system) it is run with. "
-            "See README.md for more details.\n"
-            "NB: We only check positional-only arg accuracy for Python 3.10.\n"
-            f"\nCommand run was: {' '.join(cmd)}\n",
-            file=sys.stderr,
-        )
-        print("\n\n", file=sys.stderr)
-        print(f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_dir}', file=sys.stderr)
-        return e.returncode
-    else:
-        print("stubtest succeeded", file=sys.stderr)
-        return 0
+        # Install the same mypy version as in "requirements-tests.txt"
+        subprocess.run([pip_exe, "install", get_mypy_req()], check=True)
+
+        # Uninstall setuptools from the venv so we can test stdlib's distutils
+        subprocess.run([pip_exe, "uninstall", "-y", "setuptools"], check=True)
+
+        cmd = [
+            python_exe,
+            "-m",
+            "mypy.stubtest",
+            "--check-typeshed",
+            "--custom-typeshed-dir",
+            str(typeshed_dir),
+            "--allowlist",
+            str(allowlist_dir / "py3_common.txt"),
+            "--allowlist",
+            str(allowlist_dir / version_allowlist),
+        ]
+        if (allowlist_dir / platform_allowlist).exists():
+            cmd += ["--allowlist", str(allowlist_dir / platform_allowlist)]
+        if (allowlist_dir / combined_allowlist).exists():
+            cmd += ["--allowlist", str(allowlist_dir / combined_allowlist)]
+        if sys.version_info < (3, 10):
+            # As discussed in https://github.com/python/typeshed/issues/3693, we only aim for
+            # positional-only arg accuracy for python 3.10 and above.
+            cmd += ["--ignore-positional-only"]
+        print(" ".join(cmd), file=sys.stderr)
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError as e:
+            print(
+                "\nNB: stubtest output depends on the Python version (and system) it is run with. "
+                + "See README.md for more details.\n"
+                + "NB: We only check positional-only arg accuracy for Python 3.10.\n"
+                + f"\nCommand run was: {' '.join(cmd)}\n",
+                file=sys.stderr,
+            )
+            print("\n\n", file=sys.stderr)
+            print(f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_dir}', file=sys.stderr)
+            return e.returncode
+        else:
+            print("stubtest succeeded", file=sys.stderr)
+            return 0
 
 
 if __name__ == "__main__":

tests/utils.py

@@ -81,8 +81,8 @@ def make_venv(venv_dir: Path) -> VenvInfo:
 
 @cache
 def get_mypy_req() -> str:
-    with open("requirements-tests.txt", encoding="UTF-8") as f:
-        return next(line.strip() for line in f if "mypy" in line)
+    with open("requirements-tests.txt", encoding="UTF-8") as requirements_file:
+        return next(strip_comments(line) for line in requirements_file if "mypy" in line)
 
 
 # ====================================================================