Fix distutils and difflib stubtest exceptions (#5182)

stdlib/distutils/archive_util.pyi

@@ -7,6 +7,16 @@ def make_archive(
     base_dir: Optional[str] = ...,
     verbose: int = ...,
     dry_run: int = ...,
+    owner: Optional[str] = ...,
+    group: Optional[str] = ...,
+) -> str: ...
+def make_tarball(
+    base_name: str,
+    base_dir: str,
+    compress: Optional[str] = ...,
+    verbose: int = ...,
+    dry_run: int = ...,
+    owner: Optional[str] = ...,
+    group: Optional[str] = ...,
 ) -> str: ...
-def make_tarball(base_name: str, base_dir: str, compress: Optional[str] = ..., verbose: int = ..., dry_run: int = ...) -> str: ...
 def make_zipfile(base_name: str, base_dir: str, verbose: int = ..., dry_run: int = ...) -> str: ...

stdlib/distutils/debug.pyi

@@ -1 +1,3 @@
-DEBUG: bool
+from typing import Optional
+
+DEBUG: Optional[bool]

stdlib/distutils/extension.pyi

@@ -5,18 +5,18 @@ class Extension:
         self,
         name: str,
         sources: List[str],
-        include_dirs: List[str] = ...,
-        define_macros: List[Tuple[str, Optional[str]]] = ...,
-        undef_macros: List[str] = ...,
-        library_dirs: List[str] = ...,
-        libraries: List[str] = ...,
-        runtime_library_dirs: List[str] = ...,
-        extra_objects: List[str] = ...,
-        extra_compile_args: List[str] = ...,
-        extra_link_args: List[str] = ...,
-        export_symbols: List[str] = ...,
+        include_dirs: Optional[List[str]] = ...,
+        define_macros: Optional[List[Tuple[str, Optional[str]]]] = ...,
+        undef_macros: Optional[List[str]] = ...,
+        library_dirs: Optional[List[str]] = ...,
+        libraries: Optional[List[str]] = ...,
+        runtime_library_dirs: Optional[List[str]] = ...,
+        extra_objects: Optional[List[str]] = ...,
+        extra_compile_args: Optional[List[str]] = ...,
+        extra_link_args: Optional[List[str]] = ...,
+        export_symbols: Optional[List[str]] = ...,
         swig_opts: Optional[str] = ...,  # undocumented
-        depends: List[str] = ...,
-        language: str = ...,
-        optional: bool = ...,
+        depends: Optional[List[str]] = ...,
+        language: Optional[str] = ...,
+        optional: Optional[bool] = ...,
     ) -> None: ...

stdlib/distutils/fancy_getopt.pyi

@@ -1,4 +1,4 @@
-from typing import Any, List, Mapping, Optional, Tuple, Union, overload
+from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union, overload
 
 _Option = Tuple[str, Optional[str], str]
 _GR = Tuple[List[str], OptionDummy]
@@ -18,4 +18,5 @@ class FancyGetopt:
     def get_option_order(self) -> List[Tuple[str, str]]: ...
     def generate_help(self, header: Optional[str] = ...) -> List[str]: ...
 
-class OptionDummy: ...
+class OptionDummy:
+    def __init__(self, options: Iterable[str] = ...) -> None: ...

stdlib/distutils/filelist.pyi

@@ -1 +1,68 @@
-class FileList: ...
+from typing import Iterable, List, Optional, Pattern, Union, overload
+from typing_extensions import Literal
+
+# class is entirely undocumented
+class FileList:
+    allfiles: Optional[Iterable[str]] = ...
+    files: List[str] = ...
+    def __init__(self, warn: None = ..., debug_print: None = ...) -> None: ...
+    def set_allfiles(self, allfiles: Iterable[str]) -> None: ...
+    def findall(self, dir: str = ...) -> None: ...
+    def debug_print(self, msg: str) -> None: ...
+    def append(self, item: str) -> None: ...
+    def extend(self, items: Iterable[str]) -> None: ...
+    def sort(self) -> None: ...
+    def remove_duplicates(self) -> None: ...
+    def process_template_line(self, line: str) -> None: ...
+    @overload
+    def include_pattern(
+        self,
+        pattern: str,
+        anchor: Union[int, bool] = ...,
+        prefix: Optional[str] = ...,
+        is_regex: Literal[0, False] = ...,
+    ) -> bool: ...
+    @overload
+    def include_pattern(self, pattern: Union[str, Pattern[str]], *, is_regex: Literal[True, 1] = ...) -> bool: ...
+    @overload
+    def include_pattern(
+        self,
+        pattern: Union[str, Pattern[str]],
+        anchor: Union[int, bool] = ...,
+        prefix: Optional[str] = ...,
+        is_regex: Union[int, bool] = ...,
+    ) -> bool: ...
+    @overload
+    def exclude_pattern(
+        self,
+        pattern: str,
+        anchor: Union[int, bool] = ...,
+        prefix: Optional[str] = ...,
+        is_regex: Literal[0, False] = ...,
+    ) -> bool: ...
+    @overload
+    def exclude_pattern(self, pattern: Union[str, Pattern[str]], *, is_regex: Literal[True, 1] = ...) -> bool: ...
+    @overload
+    def exclude_pattern(
+        self,
+        pattern: Union[str, Pattern[str]],
+        anchor: Union[int, bool] = ...,
+        prefix: Optional[str] = ...,
+        is_regex: Union[int, bool] = ...,
+    ) -> bool: ...
+
+def findall(dir: str = ...) -> List[str]: ...
+def glob_to_re(pattern: str) -> str: ...
+@overload
+def translate_pattern(
+    pattern: str, anchor: Union[int, bool] = ..., prefix: Optional[str] = ..., is_regex: Literal[False, 0] = ...
+) -> Pattern[str]: ...
+@overload
+def translate_pattern(pattern: Union[str, Pattern[str]], *, is_regex: Literal[True, 1] = ...) -> Pattern[str]: ...
+@overload
+def translate_pattern(
+    pattern: Union[str, Pattern[str]],
+    anchor: Union[int, bool] = ...,
+    prefix: Optional[str] = ...,
+    is_regex: Union[int, bool] = ...,
+) -> Pattern[str]: ...

stdlib/distutils/sysconfig.pyi

@@ -11,4 +11,3 @@ def get_makefile_filename() -> str: ...
 def get_python_inc(plat_specific: bool = ..., prefix: Optional[str] = ...) -> str: ...
 def get_python_lib(plat_specific: bool = ..., standard_lib: bool = ..., prefix: Optional[str] = ...) -> str: ...
 def customize_compiler(compiler: CCompiler) -> None: ...
-def set_python_build() -> None: ...

stdlib/distutils/text_file.pyi

@@ -15,7 +15,7 @@ class TextFile:
     ) -> None: ...
     def open(self, filename: str) -> None: ...
     def close(self) -> None: ...
-    def warn(self, msg: str, line: Union[List[int], Tuple[int, int], int] = ...) -> None: ...
+    def warn(self, msg: str, line: Optional[Union[List[int], Tuple[int, int], int]] = ...) -> None: ...
     def readline(self) -> Optional[str]: ...
     def readlines(self) -> List[str]: ...
     def unreadline(self, line: str) -> str: ...

tests/stubtest_whitelists/linux.txt

@@ -5,7 +5,7 @@ curses.COLORS
 curses.COLOR_PAIRS
 curses.COLS
 curses.LINES
-distutils.command.bdist_msi
+distutils.command.bdist_msi  # msi is only available on windows
 grp.getgrgid
 grp.struct_group._asdict
 grp.struct_group._make

tests/stubtest_whitelists/py3_common.txt

@@ -139,19 +139,10 @@ ctypes.pointer
 ctypes.string_at
 ctypes.wstring_at
 dbm.error
-difflib.SequenceMatcher.__init__
-distutils.archive_util.make_archive
-distutils.archive_util.make_tarball
-distutils.command.bdist_packager
-distutils.core.Extension.__init__
-distutils.debug.DEBUG
-distutils.extension.Extension.__init__
-distutils.fancy_getopt.OptionDummy.__init__
-distutils.filelist.FileList.__init__
-distutils.sysconfig.set_python_build
-distutils.text_file.TextFile.warn
-distutils.version.Version._cmp
-distutils.version.Version.parse
+difflib.SequenceMatcher.__init__  # mypy default value for generic parameter issues. See https://github.com/python/mypy/issues/3737
+distutils.command.bdist_packager  # It exists in docs as package name but not in code except as a mention in a comment.
+distutils.version.Version._cmp  # class should have declared this
+distutils.version.Version.parse  # class should have declared this
 email.errors.MessageDefect.__init__
 email.generator.BytesGenerator.__init__
 email.generator.DecodedGenerator.__init__