a close reading of the pickle module (#12971)

stdlib/@tests/stubtest_allowlists/common.txt

@@ -31,7 +31,6 @@ importlib.abc.MetaPathFinder.find_spec  # Not defined on the actual class, but e
 importlib.abc.PathEntryFinder.find_spec  # Not defined on the actual class, but expected to exist.
 numbers.Number.__hash__  # typeshed marks this as abstract but code just sets this as None
 optparse.Values.__getattr__  # Some attributes are set in __init__ using setattr
-pickle.Pickler.reducer_override  # implemented in C pickler
 select.poll  # Depends on configuration
 selectors.DevpollSelector  # Depends on configuration
 shutil.rmtree  # stubtest doesn't like that we have this as an instance of a callback protocol instead of a function
@@ -408,10 +407,6 @@ os._wrap_close.write  # Methods that come from __getattr__() at runtime
 os._wrap_close.writelines  # Methods that come from __getattr__() at runtime
 pickle._Pickler\..*  # Best effort typing for undocumented internals
 pickle._Unpickler\..*  # Best effort typing for undocumented internals
-pickle.Pickler.memo  # undocumented implementation detail, has different type in C/Python implementations
-pickle.Pickler.persistent_id  # C pickler persistent_id is an attribute
-pickle.Unpickler.memo  # undocumented implementation detail, has different type in C/Python implementations
-pickle.Unpickler.persistent_load  # C unpickler persistent_load is an attribute
 _?queue.SimpleQueue.__init__  # C signature is broader than what is actually accepted
 re.Pattern.scanner  # Undocumented and not useful. #6405
 ssl.PROTOCOL_SSLv2  # Depends on the existence and flags of SSL

stdlib/VERSIONS

@@ -57,6 +57,7 @@ _msi: 3.0-3.12
 _multibytecodec: 3.0-
 _operator: 3.4-
 _osx_support: 3.0-
+_pickle: 3.0-
 _posixsubprocess: 3.2-
 _py_abc: 3.7-
 _pydecimal: 3.5-

stdlib/_pickle.pyi

@@ -0,0 +1,108 @@
+import sys
+from _typeshed import ReadableBuffer, SupportsWrite
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from pickle import PickleBuffer as PickleBuffer
+from typing import Any, Protocol, type_check_only
+from typing_extensions import TypeAlias
+
+class _ReadableFileobj(Protocol):
+    def read(self, n: int, /) -> bytes: ...
+    def readline(self) -> bytes: ...
+
+_BufferCallback: TypeAlias = Callable[[PickleBuffer], Any] | None
+
+_ReducedType: TypeAlias = (
+    str
+    | tuple[Callable[..., Any], tuple[Any, ...]]
+    | tuple[Callable[..., Any], tuple[Any, ...], Any]
+    | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None]
+    | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None, Iterator[Any] | None]
+)
+
+def dump(
+    obj: Any,
+    file: SupportsWrite[bytes],
+    protocol: int | None = None,
+    *,
+    fix_imports: bool = True,
+    buffer_callback: _BufferCallback = None,
+) -> None: ...
+def dumps(
+    obj: Any, protocol: int | None = None, *, fix_imports: bool = True, buffer_callback: _BufferCallback = None
+) -> bytes: ...
+def load(
+    file: _ReadableFileobj,
+    *,
+    fix_imports: bool = True,
+    encoding: str = "ASCII",
+    errors: str = "strict",
+    buffers: Iterable[Any] | None = (),
+) -> Any: ...
+def loads(
+    data: ReadableBuffer,
+    /,
+    *,
+    fix_imports: bool = True,
+    encoding: str = "ASCII",
+    errors: str = "strict",
+    buffers: Iterable[Any] | None = (),
+) -> Any: ...
+
+class PickleError(Exception): ...
+class PicklingError(PickleError): ...
+class UnpicklingError(PickleError): ...
+
+@type_check_only
+class PicklerMemoProxy:
+    def clear(self, /) -> None: ...
+    def copy(self, /) -> dict[int, tuple[int, Any]]: ...
+
+class Pickler:
+    fast: bool
+    dispatch_table: Mapping[type, Callable[[Any], _ReducedType]]
+    reducer_override: Callable[[Any], Any]
+    bin: bool  # undocumented
+    def __init__(
+        self,
+        file: SupportsWrite[bytes],
+        protocol: int | None = None,
+        *,
+        fix_imports: bool = True,
+        buffer_callback: _BufferCallback = None,
+    ) -> None: ...
+    @property
+    def memo(self) -> PicklerMemoProxy: ...
+    @memo.setter
+    def memo(self, value: PicklerMemoProxy | dict[int, tuple[int, Any]]) -> None: ...
+    def dump(self, obj: Any, /) -> None: ...
+    def clear_memo(self) -> None: ...
+    if sys.version_info >= (3, 13):
+        def persistent_id(self, obj: Any, /) -> Any: ...
+    else:
+        persistent_id: Callable[[Any], Any]
+
+@type_check_only
+class UnpicklerMemoProxy:
+    def clear(self, /) -> None: ...
+    def copy(self, /) -> dict[int, tuple[int, Any]]: ...
+
+class Unpickler:
+    def __init__(
+        self,
+        file: _ReadableFileobj,
+        *,
+        fix_imports: bool = True,
+        encoding: str = "ASCII",
+        errors: str = "strict",
+        buffers: Iterable[Any] | None = (),
+    ) -> None: ...
+    @property
+    def memo(self) -> UnpicklerMemoProxy: ...
+    @memo.setter
+    def memo(self, value: UnpicklerMemoProxy | dict[int, tuple[int, Any]]) -> None: ...
+    def load(self) -> Any: ...
+    def find_class(self, module_name: str, global_name: str, /) -> Any: ...
+    if sys.version_info >= (3, 13):
+        def persistent_load(self, pid: Any, /) -> Any: ...
+    else:
+        persistent_load: Callable[[Any], Any]

stdlib/multiprocessing/reduction.pyi

@@ -1,12 +1,12 @@
 import pickle
 import sys
+from _pickle import _ReducedType
 from _typeshed import HasFileno, SupportsWrite, Unused
 from abc import ABCMeta
 from builtins import type as Type  # alias to avoid name clash
 from collections.abc import Callable
 from copyreg import _DispatchTableType
 from multiprocessing import connection
-from pickle import _ReducedType
 from socket import socket
 from typing import Any, Final
 

stdlib/pickle.pyi

@@ -1,7 +1,20 @@
+from _pickle import (
+    PickleError as PickleError,
+    Pickler as Pickler,
+    PicklingError as PicklingError,
+    Unpickler as Unpickler,
+    UnpicklingError as UnpicklingError,
+    _BufferCallback,
+    _ReadableFileobj,
+    _ReducedType,
+    dump as dump,
+    dumps as dumps,
+    load as load,
+    loads as loads,
+)
 from _typeshed import ReadableBuffer, SupportsWrite
-from collections.abc import Callable, Iterable, Iterator, Mapping
-from typing import Any, ClassVar, Protocol, SupportsBytes, SupportsIndex, final
-from typing_extensions import TypeAlias
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any, ClassVar, SupportsBytes, SupportsIndex, final
 
 __all__ = [
     "PickleBuffer",
@@ -93,10 +106,6 @@ DEFAULT_PROTOCOL: int
 
 bytes_types: tuple[type[Any], ...]  # undocumented
 
-class _ReadableFileobj(Protocol):
-    def read(self, n: int, /) -> bytes: ...
-    def readline(self) -> bytes: ...
-
 @final
 class PickleBuffer:
     def __init__(self, buffer: ReadableBuffer) -> None: ...
@@ -105,84 +114,6 @@ class PickleBuffer:
     def __buffer__(self, flags: int, /) -> memoryview: ...
     def __release_buffer__(self, buffer: memoryview, /) -> None: ...
 
-_BufferCallback: TypeAlias = Callable[[PickleBuffer], Any] | None
-
-def dump(
-    obj: Any,
-    file: SupportsWrite[bytes],
-    protocol: int | None = None,
-    *,
-    fix_imports: bool = True,
-    buffer_callback: _BufferCallback = None,
-) -> None: ...
-def dumps(
-    obj: Any, protocol: int | None = None, *, fix_imports: bool = True, buffer_callback: _BufferCallback = None
-) -> bytes: ...
-def load(
-    file: _ReadableFileobj,
-    *,
-    fix_imports: bool = True,
-    encoding: str = "ASCII",
-    errors: str = "strict",
-    buffers: Iterable[Any] | None = (),
-) -> Any: ...
-def loads(
-    data: ReadableBuffer,
-    /,
-    *,
-    fix_imports: bool = True,
-    encoding: str = "ASCII",
-    errors: str = "strict",
-    buffers: Iterable[Any] | None = (),
-) -> Any: ...
-
-class PickleError(Exception): ...
-class PicklingError(PickleError): ...
-class UnpicklingError(PickleError): ...
-
-_ReducedType: TypeAlias = (
-    str
-    | tuple[Callable[..., Any], tuple[Any, ...]]
-    | tuple[Callable[..., Any], tuple[Any, ...], Any]
-    | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None]
-    | tuple[Callable[..., Any], tuple[Any, ...], Any, Iterator[Any] | None, Iterator[Any] | None]
-)
-
-class Pickler:
-    fast: bool
-    dispatch_table: Mapping[type, Callable[[Any], _ReducedType]]
-    bin: bool  # undocumented
-    dispatch: ClassVar[dict[type, Callable[[Unpickler, Any], None]]]  # undocumented, _Pickler only
-
-    def __init__(
-        self,
-        file: SupportsWrite[bytes],
-        protocol: int | None = None,
-        *,
-        fix_imports: bool = True,
-        buffer_callback: _BufferCallback = None,
-    ) -> None: ...
-    def reducer_override(self, obj: Any) -> Any: ...
-    def dump(self, obj: Any, /) -> None: ...
-    def clear_memo(self) -> None: ...
-    def persistent_id(self, obj: Any) -> Any: ...
-
-class Unpickler:
-    dispatch: ClassVar[dict[int, Callable[[Unpickler], None]]]  # undocumented, _Unpickler only
-
-    def __init__(
-        self,
-        file: _ReadableFileobj,
-        *,
-        fix_imports: bool = True,
-        encoding: str = "ASCII",
-        errors: str = "strict",
-        buffers: Iterable[Any] | None = (),
-    ) -> None: ...
-    def load(self) -> Any: ...
-    def find_class(self, module_name: str, global_name: str, /) -> Any: ...
-    def persistent_load(self, pid: Any) -> Any: ...
-
 MARK: bytes
 STOP: bytes
 POP: bytes
@@ -266,6 +197,36 @@ READONLY_BUFFER: bytes
 def encode_long(x: int) -> bytes: ...  # undocumented
 def decode_long(data: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer) -> int: ...  # undocumented
 
-# pure-Python implementations
-_Pickler = Pickler  # undocumented
-_Unpickler = Unpickler  # undocumented
+# undocumented pure-Python implementations
+class _Pickler:
+    fast: bool
+    dispatch_table: Mapping[type, Callable[[Any], _ReducedType]]
+    bin: bool  # undocumented
+    dispatch: ClassVar[dict[type, Callable[[Unpickler, Any], None]]]  # undocumented, _Pickler only
+    reducer_override: Callable[[Any], Any]
+    def __init__(
+        self,
+        file: SupportsWrite[bytes],
+        protocol: int | None = None,
+        *,
+        fix_imports: bool = True,
+        buffer_callback: _BufferCallback = None,
+    ) -> None: ...
+    def dump(self, obj: Any) -> None: ...
+    def clear_memo(self) -> None: ...
+    def persistent_id(self, obj: Any) -> Any: ...
+
+class _Unpickler:
+    dispatch: ClassVar[dict[int, Callable[[Unpickler], None]]]  # undocumented, _Unpickler only
+    def __init__(
+        self,
+        file: _ReadableFileobj,
+        *,
+        fix_imports: bool = True,
+        encoding: str = "ASCII",
+        errors: str = "strict",
+        buffers: Iterable[Any] | None = None,
+    ) -> None: ...
+    def load(self) -> Any: ...
+    def find_class(self, module: str, name: str) -> Any: ...
+    def persistent_load(self, pid: Any) -> Any: ...