Bump mypy to 1.9, add to json.encoder, small fixups (#11549)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

requirements-tests.txt

@@ -5,7 +5,7 @@ black==24.1.1            # must match .pre-commit-config.yaml
 flake8==7.0.0            # must match .pre-commit-config.yaml
 flake8-noqa==1.4.0       # must match .pre-commit-config.yaml
 flake8-pyi==24.1.0       # must match .pre-commit-config.yaml
-mypy==1.8.0
+mypy==1.9.0
 pre-commit-hooks==4.5.0  # must match .pre-commit-config.yaml
 pytype==2024.2.27; platform_system != "Windows" and python_version < "3.12"
 ruff==0.3.0              # must match .pre-commit-config.yaml

stdlib/json/encoder.pyi

@@ -10,6 +10,8 @@ INFINITY: float
 
 def py_encode_basestring(s: str) -> str: ...  # undocumented
 def py_encode_basestring_ascii(s: str) -> str: ...  # undocumented
+def encode_basestring(s: str) -> str: ...  # undocumented
+def encode_basestring_ascii(s: str) -> str: ...  # undocumented
 
 class JSONEncoder:
     item_separator: str

stdlib/ssl.pyi

@@ -15,6 +15,8 @@ _PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes |
 
 _SrvnmeCbType: TypeAlias = Callable[[SSLSocket | SSLObject, str | None, SSLSocket], int | None]
 
+socket_error = OSError
+
 class _Cipher(TypedDict):
     aead: bool
     alg_bits: int

stubs/aiofiles/aiofiles/ospath.pyi

@@ -17,3 +17,5 @@ async def samefile(
     f1: FileDescriptorOrPath, f2: FileDescriptorOrPath, *, loop: AbstractEventLoop | None = ..., executor: Any = ...
 ) -> bool: ...
 async def sameopenfile(fp1: int, fp2: int, *, loop: AbstractEventLoop | None = ..., executor: Any = ...) -> bool: ...
+async def islink(path: FileDescriptorOrPath) -> bool: ...
+async def ismount(path: FileDescriptorOrPath) -> bool: ...

stubs/boltons/@tests/stubtest_allowlist.txt

@@ -5,6 +5,18 @@ boltons.strutils.int_list_to_int_tuples
 # Internal compatibility aliases
 boltons.cacheutils.basestring
 boltons.funcutils.basestring
-boltons.funcutils.inspect_formatargspec
 boltons.funcutils.make_method
 boltons.iterutils.basestring
+
+boltons.cacheutils.bytes
+boltons.cacheutils.str
+boltons.cacheutils.unicode
+boltons.cacheutils.xrange
+boltons.deprutils.ModuleType.__dict__
+boltons.deprutils.ModuleType.__getattr__
+boltons.iterutils.unicode
+boltons.iterutils.xrange
+boltons.mathutils.unicode
+boltons.urlutils.OMD
+boltons.urlutils.unichr
+boltons.urlutils.unicode

stubs/chevron/chevron/renderer.pyi

@@ -4,6 +4,8 @@ from typing import Any, Literal
 
 g_token_cache: dict[str, list[tuple[str, str]]]  # undocumented
 python3: Literal[True]
+string_type = str
+unicode_type = str
 
 def unicode(x: str, y: str) -> str: ...
 def render(

stubs/colorama/@tests/stubtest_allowlist_linux.txt

@@ -2,6 +2,3 @@
 colorama.winterm.WinColor
 colorama.winterm.WinStyle
 colorama.winterm.WinTerm
-
-# A re-export that's an implementation detail:
-colorama.winterm.get_osfhandle

stubs/decorator/@tests/stubtest_allowlist.txt

@@ -7,3 +7,4 @@ decorator.FunctionMaker.varkw
 decorator.decorate
 decorator.decorator
 decorator.get_init
+decorator.EMPTY

stubs/fanstatic/@tests/stubtest_allowlist.txt

@@ -14,6 +14,8 @@ fanstatic.core.DummyNeededResources.clear
 fanstatic.core.DummyNeededResources.library_url
 fanstatic.core.DummyNeededResources.resources
 
+fanstatic.compiler.mtime
+
 # Error: is inconsistent
 # ======================
 # The core API for Dependable is a bit annoying, since the base class

stubs/fpdf2/@tests/stubtest_allowlist.txt

@@ -12,8 +12,3 @@ fpdf.fonts.HarfBuzzFont
 # fixed in later versions.
 fpdf.FPDF.set_encryption
 fpdf.fpdf.FPDF.set_encryption
-
-# Runtime has some internal arguments.
-fpdf.syntax.build_obj_dict
-fpdf.ViewerPreferences.serialize
-fpdf\.[a-z]+\.[A-Za-z]+\.serialize

stubs/gevent/@tests/stubtest_allowlist.txt

@@ -74,7 +74,6 @@ gevent.ssl.base64
 gevent.ssl.create_connection
 gevent.ssl.errno
 gevent.ssl.os
-gevent.ssl.socket_error
 gevent.ssl.warnings
 gevent\.subprocess\._[A-Za-z0-9]\w*
 
@@ -147,11 +146,7 @@ gevent.Greenlet.link_value
 gevent.greenlet.Greenlet.link
 gevent.greenlet.Greenlet.link_exception
 gevent.greenlet.Greenlet.link_value
-gevent._ffi.watcher.IoMixin.__init__
 gevent._threading.Queue.qsize
-gevent.monkey.patch_module
-gevent.monkey.patch_ssl
-gevent.monkey.patch_thread
 
 # removed deprecated argument
 gevent._hub_primitives.wait_readwrite
@@ -195,22 +190,10 @@ gevent.ssl.get_server_certificate
 # for now we ignore that fact, most of the ways to call Popen should be compatible
 gevent.subprocess.Popen.__init__
 
-# we exclude the undocumented internal argument _raise_exc from the stubs
-gevent.subprocess.Popen.wait
-
-# we exclude the undocumented internal argument _format_hub from the stubs
-gevent.threadpool.ThreadPool.__repr__
-
 # gevent overwrites with a named parameter for fd, but we're fine with only
 # supporting the API of the superclass
 gevent.threadpool.ThreadPoolExecutor.submit
 
-# we exclude the undocumented internal argument _one_shot from the stubs
-gevent.Timeout.__init__
-gevent.Timeout.start_new
-gevent.timeout.Timeout.__init__
-gevent.timeout.Timeout.start_new
-
 # Error: is not a type/function
 # =====================
 # zope.interface related errors, these shouldn't matter

stubs/hdbcli/@tests/stubtest_allowlist.txt

@@ -4,3 +4,7 @@ hdbcli.dbapi.Error.errortext
 hdbcli.dbapi.Warning.errorcode
 hdbcli.dbapi.Warning.errortext
 hdbcli.dbapi.ExecuteManyErrorEntry.rownumber
+
+hdbcli.dbapi.buffer
+hdbcli.dbapi.long
+hdbcli.dbapi.unicode

stubs/httplib2/httplib2/auth.pyi

@@ -13,3 +13,4 @@ scheme: Any
 challenge: Any
 authentication_info: Any
 www_authenticate: Any
+downcaseTokens: Any

stubs/inifile/@tests/stubtest_allowlist.txt

@@ -6,6 +6,7 @@ inifile.iter_from_file
 inifile.iteritems
 inifile.reraise
 inifile.string_types
+inifile.text_type
 
 # Attributes that should be treated as read-only and thus are annotated
 # with @property

stubs/keyboard/@tests/stubtest_allowlist.txt

@@ -3,3 +3,6 @@ keyboard.KeyboardEvent.scan_code
 keyboard._keyboard_event.KeyboardEvent.scan_code
 # TODO: Should this be allowlisted?
 keyboard.__main__
+
+keyboard._canonical_names.basestring
+keyboard._keyboard_event.basestring

stubs/pyjks/@tests/stubtest_allowlist.txt

@@ -26,35 +26,16 @@ jks.MAGIC_NUMBER_JCEKS
 jks.MAGIC_NUMBER_JKS
 jks.RSA_ENCRYPTION_OID
 jks.SIGNATURE_WHITENING
-jks.b1
-jks.b2
-jks.b4
-jks.b8
 jks.py23basestring
 jks.bks.DSA_OID
 jks.bks.DSA_WITH_SHA1_OID
 jks.bks.RSA_ENCRYPTION_OID
-jks.bks.b1
-jks.bks.b2
-jks.bks.b4
-jks.bks.b8
 jks.bks.py23basestring
 jks.jks.DSA_OID
 jks.jks.DSA_WITH_SHA1_OID
 jks.jks.RSA_ENCRYPTION_OID
-jks.jks.b1
-jks.jks.b2
-jks.jks.b4
-jks.jks.b8
 jks.jks.py23basestring
 jks.sun_crypto.DSA_OID
 jks.sun_crypto.DSA_WITH_SHA1_OID
 jks.sun_crypto.RSA_ENCRYPTION_OID
-jks.sun_crypto.b1
-jks.sun_crypto.b2
-jks.sun_crypto.b4
-jks.sun_crypto.b8
 jks.sun_crypto.py23basestring
-
-# Unintended reexports due to `from .utils import *` imports at runtime
-jks\..*print_function

stubs/pyserial/@tests/stubtest_allowlist.txt

@@ -26,6 +26,11 @@ serial.basestring
 serial.serialutil.basestring
 serial.serialutil.iterbytes
 serial.serialutil.to_bytes
+serial.serialutil.unicode
+serial.tools.hexlify_codec.unicode
+serial.tools.miniterm.raw_input
+serial.tools.miniterm.unichr
+serial.urlhandler.protocol_hwgrep.basestring
 
 # Deprecated aliases
 serial.SerialBase.applySettingsDict

stubs/pytest-lazy-fixture/@tests/stubtest_allowlist.txt

@@ -10,3 +10,4 @@ pytest_lazyfixture.copy_metafunc
 
 # Compat:
 pytest_lazyfixture.PY3
+pytest_lazyfixture.string_type

stubs/python-slugify/@tests/stubtest_allowlist.txt

@@ -1,2 +1 @@
 slugify.__main__
-slugify.annotations  # __future__.annotations is not public API

stubs/pywin32/@tests/stubtest_allowlist_win32.txt

@@ -73,6 +73,3 @@ win32com.axdebug.gateways
 win32comext.axdebug.gateways
 # failed to import, ModuleNotFoundError: No module named 'win32comext.axdebug.axdebug'
 win32comext.axdebug.axdebug
-
-# Stubtest doesn't agree with win32.lib.pywintypes.error.__init__ being positional-only
-.+?\.error.__init__

stubs/requests-oauthlib/requests_oauthlib/oauth1_auth.pyi

@@ -8,6 +8,8 @@ CONTENT_TYPE_FORM_URLENCODED: str
 CONTENT_TYPE_MULTI_PART: str
 log: Logger
 
+unicode = str
+
 class OAuth1(AuthBase):
     client_class: type[Client]
     client: Client

stubs/requests/requests/sessions.pyi

@@ -36,6 +36,7 @@ get_auth_from_url = utils.get_auth_from_url
 codes = status_codes.codes
 REDIRECT_STATI = models.REDIRECT_STATI
 
+def preferred_clock() -> float: ...
 def merge_setting(request_setting, session_setting, dict_class=...): ...
 def merge_hooks(request_hooks, session_hooks, dict_class=...): ...
 

stubs/seaborn/@tests/stubtest_allowlist.txt

@@ -3,3 +3,5 @@ seaborn.external.docscrape.ClassDoc.__init__  # stubtest doesn't like ABC class
 seaborn.external.docscrape.NumpyDocString.__str__  # weird signature
 
 seaborn.axisgrid.Grid.tight_layout  # the method doesn't really take pos args but runtime has *args
+
+seaborn.external.appdirs.unicode

stubs/setuptools/@tests/stubtest_allowlist_linux.txt

@@ -1,2 +0,0 @@
-# Mock
-setuptools.msvc.winreg

stubs/simplejson/@tests/stubtest_allowlist.txt

@@ -2,6 +2,9 @@
 simplejson.scanner.make_scanner
 simplejson.scanner.JSONDecodeError.__init__
 simplejson.JSONDecodeError.__init__
+simplejson.encoder.c_make_encoder
+simplejson.encoder.c_encode_basestring_ascii
+simplejson.encoder.py_encode_basestring_ascii
 
 # Tests are not included:
 simplejson.tests.*

stubs/simplejson/simplejson/encoder.pyi

@@ -57,4 +57,4 @@ class JSONEncoder:
 class JSONEncoderForHTML(JSONEncoder): ...
 
 def encode_basestring(s: str | bytes, _PY3: Literal[True] = ..., _q: str = ...) -> str: ...
-def py_encode_basestring_ascii(s: str | bytes, _PY3: Literal[True] = ...) -> str: ...
+def encode_basestring_ascii(s: str | bytes, _PY3: Literal[True] = ...) -> str: ...

stubs/six/six/__init__.pyi

@@ -114,3 +114,5 @@ class MovedAttribute(_LazyDescr):
 
 def add_move(move: MovedModule | MovedAttribute) -> None: ...
 def remove_move(name: str) -> None: ...
+
+advance_iterator = next

stubs/toml/@tests/stubtest_allowlist.txt

@@ -0,0 +1,4 @@
+toml.decoder.basestring
+toml.decoder.unichr
+toml.decoder.unicode
+toml.encoder.unicode

stubs/vobject/@tests/stubtest_allowlist.txt

@@ -8,7 +8,6 @@ vobject.win32tz
 
 # dependencies
 vobject.icalendar.Pytz
-vobject.icalendar.pytz
 
 # python2 compat
 vobject.base.basestring
@@ -16,6 +15,7 @@ vobject.base.str_
 vobject.base.to_unicode
 vobject.base.to_basestring
 vobject.vcard.basestring
+vobject.base.unicode_type
 
 # implementation details that users shouldn't depend on
 vobject.base.formatter

tests/stubtest_allowlists/darwin.txt

@@ -65,8 +65,3 @@ curses.COLORS  # Initialized after start_color
 curses.COLOR_PAIRS  # Initialized after start_color
 curses.COLS  # Initialized only after initscr call
 curses.LINES  # Initialized only after initscr call
-
-# Takes a "_warn" parameter at runtime
-# for technical reasons; no need to expose it in the stub.
-asyncio.ThreadedChildWatcher.__del__
-asyncio.unix_events.ThreadedChildWatcher.__del__

tests/stubtest_allowlists/linux.txt

@@ -53,8 +53,3 @@ curses.COLORS  # Initialized after start_color
 curses.COLOR_PAIRS  # Initialized after start_color
 curses.COLS  # Initialized only after initscr call
 curses.LINES  # Initialized only after initscr call
-
-# Takes a "_warn" parameter at runtime
-# for technical reasons; no need to expose it in the stub.
-asyncio.ThreadedChildWatcher.__del__
-asyncio.unix_events.ThreadedChildWatcher.__del__

tests/stubtest_allowlists/py310.txt

@@ -7,6 +7,7 @@ bz2.BZ2Decompressor.__init__  # function does not accept parameters but C signat
 configparser.ParsingError.filename
 contextlib.AbstractAsyncContextManager.__class_getitem__
 contextlib.AbstractContextManager.__class_getitem__
+email.contentmanager.typ
 enum.Enum._generate_next_value_
 functools._lru_cache_wrapper.cache_parameters  # Cannot be detected statically
 gettext.install

tests/stubtest_allowlists/py312.txt

@@ -125,7 +125,3 @@ typing.SupportsAbs.__type_params__
 typing.SupportsRound.__type_params__
 typing_extensions.SupportsAbs.__type_params__
 typing_extensions.SupportsRound.__type_params__
-
-# Dunder that exists on all runtime-checkable protocols on py312+
-# TODO teach stubtest about this so we don't have to allowlist it
-.+\.__non_callable_proto_members__

tests/stubtest_allowlists/py38.txt

@@ -35,13 +35,13 @@ dummy_threading.Condition.release
 dummy_threading.Event.isSet
 dummy_threading.Thread.native_id
 dummy_threading.local.__new__
+email.contentmanager.typ
 enum.Enum._generate_next_value_
 gettext.install  # codeset default value is ['unspecified'] so can't be specified
 gettext.translation  # codeset default value is ['unspecified'] so can't be specified
 importlib.abc.Finder.find_module
 inspect.Signature.from_builtin  # Removed in 3.11, can add if someone needs this
 inspect.Signature.from_function  # Removed in 3.11, can add if someone needs this
-random.Random.randrange  # missing undocumented arg _int
 sched.Event.__doc__  # __slots__ is overridden
 typing.NamedTuple.__new__
 typing.NamedTuple._asdict

tests/stubtest_allowlists/py39.txt

@@ -21,6 +21,7 @@ collections.ValuesView.__reversed__
 configparser.ParsingError.filename
 contextlib.AbstractAsyncContextManager.__class_getitem__
 contextlib.AbstractContextManager.__class_getitem__
+email.contentmanager.typ
 enum.Enum._generate_next_value_
 functools._lru_cache_wrapper.cache_parameters  # Cannot be detected statically
 gettext.install

tests/stubtest_allowlists/py3_common.txt

@@ -371,10 +371,6 @@ threading.Condition.release
 multiprocessing.dummy.Condition.acquire
 multiprocessing.dummy.Condition.release
 
-# Takes a "_warn" parameter at runtime
-# for technical reasons; no need to expose it in the stub.
-asyncio.proactor_events._ProactorBasePipeTransport.__del__
-
 # Weird special builtins that are typed as functions, but aren't functions
 builtins.copyright
 builtins.credits
@@ -450,6 +446,8 @@ multiprocessing.(dummy|managers).Namespace.__[gs]etattr__  # Any field can be se
 # mypy infers the variable has type `(*args) -> ForkingPickler`
 # but stubtest infers the runtime type as <class ForkingPickler>
 multiprocessing.reduction.AbstractReducer.ForkingPickler
+# Non-private parameter on __del__
+multiprocessing.pool.Pool.__del__
 
 # C signature is broader than what is actually accepted
 queue.SimpleQueue.__init__
@@ -519,9 +517,6 @@ _typeshed.*  # Utility types for typeshed, doesn't exist at runtime
 typing._SpecialForm.__call__
 typing._SpecialForm.__init__
 
-# Should go away with a future version of stubtest
-typing_extensions\..*\.__non_callable_proto_members__
-
 # Pretend typing.ByteString is a Union, to better match its documented semantics.
 # As a side effect, this changes the definition of collections.abc.ByteString, which is okay,
 # because it's not an ABC that makes any sense and was deprecated in 3.12
@@ -567,6 +562,10 @@ turtle.ScrolledCanvas.find_all
 turtle.ScrolledCanvas.select_clear
 turtle.ScrolledCanvas.select_item
 
+# Attributes that are intended to be private
+uuid.bytes_
+uuid.int_
+
 # Always overridden in inheriting class
 webbrowser.UnixBrowser.remote_action
 webbrowser.UnixBrowser.remote_action_newtab
@@ -705,13 +704,3 @@ typing(_extensions)?\.IO\.__iter__  # See https://github.com/python/typeshed/com
 (tarfile.AbsolutePathError)?
 (tarfile.AbsoluteLinkError)?
 (shutil.unpack_archive)?
-
-# Omit internal _KEEP argument
-tarfile.TarInfo.replace
-
-# Private parameters are deliberately omitted in the stubs (see https://github.com/python/mypy/issues/16443)
-asyncio.(base_events.)?BaseEventLoop.__del__
-asyncio.(base_subprocess.)?BaseSubprocessTransport.__del__
-asyncio.(sslproto.)?_SSLProtocolTransport.__del__
-multiprocessing.pool.Pool.__del__
-subprocess.Popen.__del__

tests/stubtest_allowlists/win32-py310.txt

@@ -1,3 +1,5 @@
+tarfile.TarInfo.replace
+
 # pathlib methods that exist on Windows, but always raise NotImplementedError,
 # so are omitted from the stub
 pathlib.Path.is_mount

tests/stubtest_allowlists/win32-py38.txt

@@ -1,3 +1,5 @@
+tarfile.TarInfo.replace
+
 # ==========
 # Allowlist entries that cannot or should not be fixed
 # ==========

tests/stubtest_allowlists/win32-py39.txt

@@ -1,3 +1,5 @@
+tarfile.TarInfo.replace
+
 # pathlib methods that exist on Windows, but always raise NotImplementedError,
 # so are omitted from the stub
 pathlib.Path.is_mount

tests/stubtest_allowlists/win32.txt

@@ -60,7 +60,3 @@ pathlib.Path.group
 # but have yet to find their way to all GitHub Actions images
 (sys.get_int_max_str_digits)?
 (sys.set_int_max_str_digits)?
-
-# Takes a "_warn" parameter at runtime
-# for technical reasons; no need to expose it in the stub.
-asyncio.windows_utils.PipeHandle.__del__