Document _typeshed (#5501)

* Document _typeshed

Closes: #5486

* Allow README.md files

README.md

@@ -38,6 +38,13 @@ type stub packages (if found on PyPI).
 PyCharm, pytype etc. work in a similar way, for more details see documentation
 for the type-checking tool you are using.
 
+### The `_typeshed` package
+
+typeshed includes a package `_typeshed` as part of the standard library.
+This package and its submodules contains utility types, but is not
+available at runtime. For more information about how to use this package,
+[see the `stdlib/_typeshed` directory](https://github.com/python/typeshed/tree/master/stdlib/_typeshed).
+
 ## Discussion
 
 If you've run into behavior in the type checker that suggests the type

stdlib/_typeshed/README.md

@@ -0,0 +1,34 @@
+# Utility types for typeshed
+
+This package and its submodules contains various common types used by
+typeshed. It can also be used by packages outside typeshed, but beware
+the API stability guarantees below.
+
+## Usage
+
+The `_typeshed` package and its types do not exist at runtime, but can be
+used freely in stubs (`.pyi`) files. To import the types from this package in
+implementation (`.py`) files, use the following construct:
+
+```python
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from _typeshed import ...
+```
+
+Types can then be used in annotations by either quoting them or
+using:
+
+```python
+from __future__ import annotations
+```
+
+## API Stability
+
+You can use this package and its submodules outside of typeshed, but we
+guarantee only limited API stability. Items marked as "stable" will not be
+removed or changed in an incompatible way for at least one year.
+Before making such a change, the "stable" moniker will be removed
+and we will mark the type in question as deprecated. No guarantees
+are made about unmarked types.

stdlib/_typeshed/__init__.pyi

@@ -1,16 +1,6 @@
 # Utility types for typeshed
-
-# This module contains various common types to be used by typeshed. The
-# module and its types do not exist at runtime. You can use this module
-# outside of typeshed, but no API stability guarantees are made. To use
-# it in implementation (.py) files, the following construct must be used:
 #
-#     from typing import TYPE_CHECKING
-#     if TYPE_CHECKING:
-#         from _typeshed import ...
-#
-# If on Python versions < 3.10 and "from __future__ import annotations"
-# is not used, types from this module must be quoted.
+# See the README.md file in this directory for more information.
 
 import array
 import mmap
@@ -28,6 +18,7 @@ _T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
 _T_contra = TypeVar("_T_contra", contravariant=True)
 
+# stable
 class IdentityFunction(Protocol):
     def __call__(self, __x: _T) -> _T: ...
 
@@ -44,24 +35,28 @@ class SupportsRDivMod(Protocol[_T_contra, _T_co]):
 
 # Mapping-like protocols
 
+# stable
 class SupportsItems(Protocol[_KT_co, _VT_co]):
     def items(self) -> AbstractSet[Tuple[_KT_co, _VT_co]]: ...
 
+# stable
 class SupportsKeysAndGetItem(Protocol[_KT, _VT_co]):
     def keys(self) -> Iterable[_KT]: ...
     def __getitem__(self, __k: _KT) -> _VT_co: ...
 
+# stable
 class SupportsGetItem(Container[_KT_contra], Protocol[_KT_contra, _VT_co]):
     def __getitem__(self, __k: _KT_contra) -> _VT_co: ...
 
+# stable
 class SupportsItemAccess(SupportsGetItem[_KT_contra, _VT], Protocol[_KT_contra, _VT]):
     def __setitem__(self, __k: _KT_contra, __v: _VT) -> None: ...
     def __delitem__(self, __v: _KT_contra) -> None: ...
 
 # These aliases are simple strings in Python 2.
-StrPath = Union[str, PathLike[str]]
-BytesPath = Union[bytes, PathLike[bytes]]
-StrOrBytesPath = Union[str, bytes, PathLike[str], PathLike[bytes]]
+StrPath = Union[str, PathLike[str]]  # stable
+BytesPath = Union[bytes, PathLike[bytes]]  # stable
+StrOrBytesPath = Union[str, bytes, PathLike[str], PathLike[bytes]]  # stable
 AnyPath = StrOrBytesPath  # obsolete, will be removed soon
 
 OpenTextModeUpdating = Literal[
@@ -131,27 +126,33 @@ OpenBinaryModeWriting = Literal["wb", "bw", "ab", "ba", "xb", "bx"]
 OpenBinaryModeReading = Literal["rb", "br", "rbU", "rUb", "Urb", "brU", "bUr", "Ubr"]
 OpenBinaryMode = Union[OpenBinaryModeUpdating, OpenBinaryModeReading, OpenBinaryModeWriting]
 
+# stable
 class HasFileno(Protocol):
     def fileno(self) -> int: ...
 
-FileDescriptor = int
-FileDescriptorLike = Union[int, HasFileno]
+FileDescriptor = int  # stable
+FileDescriptorLike = Union[int, HasFileno]  # stable
 
+# stable
 class SupportsRead(Protocol[_T_co]):
     def read(self, __length: int = ...) -> _T_co: ...
 
+# stable
 class SupportsReadline(Protocol[_T_co]):
     def readline(self, __length: int = ...) -> _T_co: ...
 
+# stable
 class SupportsNoArgReadline(Protocol[_T_co]):
     def readline(self) -> _T_co: ...
 
+# stable
 class SupportsWrite(Protocol[_T_contra]):
     def write(self, __s: _T_contra) -> Any: ...
 
-ReadableBuffer = Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]
-WriteableBuffer = Union[bytearray, memoryview, array.array[Any], mmap.mmap]
+ReadableBuffer = Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]  # stable
+WriteableBuffer = Union[bytearray, memoryview, array.array[Any], mmap.mmap]  # stable
 
+# stable
 if sys.version_info >= (3, 10):
     from types import NoneType as NoneType
 else:

stdlib/_typeshed/tkinter.pyi

@@ -1,3 +1,5 @@
+# See the README.md file in this directory for more information.
+
 from tkinter import Event, Misc, Widget
 from typing import Optional, Protocol
 

stdlib/_typeshed/wsgi.pyi

@@ -1,27 +1,27 @@
 # Types to support PEP 3333 (WSGI)
 #
-# This module doesn't exist at runtime and neither do the types defined in this
-# file. They are provided for type checking purposes.
+# See the README.md file in this directory for more information.
 
 from sys import _OptExcInfo
 from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Tuple
 
+# stable
 class StartResponse(Protocol):
     def __call__(
         self, status: str, headers: List[Tuple[str, str]], exc_info: Optional[_OptExcInfo] = ...
     ) -> Callable[[bytes], Any]: ...
 
-WSGIEnvironment = Dict[str, Any]
-WSGIApplication = Callable[[WSGIEnvironment, StartResponse], Iterable[bytes]]
+WSGIEnvironment = Dict[str, Any]  # stable
+WSGIApplication = Callable[[WSGIEnvironment, StartResponse], Iterable[bytes]]  # stable
 
-# WSGI input streams per PEP 3333
+# WSGI input streams per PEP 3333, stable
 class InputStream(Protocol):
     def read(self, size: int = ...) -> bytes: ...
     def readline(self, size: int = ...) -> bytes: ...
     def readlines(self, hint: int = ...) -> List[bytes]: ...
     def __iter__(self) -> Iterable[bytes]: ...
 
-# WSGI error streams per PEP 3333
+# WSGI error streams per PEP 3333, stable
 class ErrorStream(Protocol):
     def flush(self) -> None: ...
     def write(self, s: str) -> None: ...

stdlib/_typeshed/xml.pyi

@@ -1,4 +1,4 @@
-# Stub-only types. This module does not exist at runtime.
+# See the README.md file in this directory for more information.
 
 from typing import Any, Optional
 from typing_extensions import Protocol

tests/check_consistent.py

@@ -22,6 +22,7 @@ consistent_files = [
     {"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"},
 ]
 metadata_keys = {"version", "python2", "python3", "requires", "extra_description", "obsolete_since"}
+allowed_files = {"README.md"}
 
 
 def assert_stubs_only(directory):
@@ -30,6 +31,8 @@ def assert_stubs_only(directory):
     assert top.isidentifier(), f"Bad directory name: {top}"
     for _, dirs, files in os.walk(directory):
         for file in files:
+            if file in allowed_files:
+                continue
             name, ext = os.path.splitext(file)
             assert name.isidentifier(), f"Files must be valid modules, got: {name}"
             assert ext == ".pyi", f"Only stub flies allowed. Got: {file} in {directory}"