Major update for the xml module (#13349)

stdlib/xml/etree/ElementInclude.pyi

@@ -1,9 +1,14 @@
 import sys
 from _typeshed import FileDescriptorOrPath
-from collections.abc import Callable
-from typing import Final
+from typing import Final, Literal, Protocol, overload
 from xml.etree.ElementTree import Element
 
+class _Loader(Protocol):
+    @overload
+    def __call__(self, href: FileDescriptorOrPath, parse: Literal["xml"], encoding: str | None = None) -> Element: ...
+    @overload
+    def __call__(self, href: FileDescriptorOrPath, parse: Literal["text"], encoding: str | None = None) -> str: ...
+
 XINCLUDE: Final[str]
 XINCLUDE_INCLUDE: Final[str]
 XINCLUDE_FALLBACK: Final[str]
@@ -13,17 +18,15 @@ if sys.version_info >= (3, 9):
 
 class FatalIncludeError(SyntaxError): ...
 
-def default_loader(href: FileDescriptorOrPath, parse: str, encoding: str | None = None) -> str | Element: ...
+@overload
+def default_loader(href: FileDescriptorOrPath, parse: Literal["xml"], encoding: str | None = None) -> Element: ...
+@overload
+def default_loader(href: FileDescriptorOrPath, parse: Literal["text"], encoding: str | None = None) -> str: ...
 
-# TODO: loader is of type default_loader ie it takes a callable that has the
-# same signature as default_loader. But default_loader has a keyword argument
-# Which can't be represented using Callable...
 if sys.version_info >= (3, 9):
-    def include(
-        elem: Element, loader: Callable[..., str | Element] | None = None, base_url: str | None = None, max_depth: int | None = 6
-    ) -> None: ...
+    def include(elem: Element, loader: _Loader | None = None, base_url: str | None = None, max_depth: int | None = 6) -> None: ...
 
     class LimitedRecursiveIncludeError(FatalIncludeError): ...
 
 else:
-    def include(elem: Element, loader: Callable[..., str | Element] | None = None) -> None: ...
+    def include(elem: Element, loader: _Loader | None = None) -> None: ...

stdlib/xml/etree/ElementPath.pyi

@@ -1,6 +1,6 @@
-from collections.abc import Callable, Generator
+from collections.abc import Callable, Generator, Iterable
 from re import Pattern
-from typing import TypeVar
+from typing import Any, Literal, TypeVar, overload
 from typing_extensions import TypeAlias
 from xml.etree.ElementTree import Element
 
@@ -8,27 +8,34 @@ xpath_tokenizer_re: Pattern[str]
 
 _Token: TypeAlias = tuple[str, str]
 _Next: TypeAlias = Callable[[], _Token]
-_Callback: TypeAlias = Callable[[_SelectorContext, list[Element]], Generator[Element, None, None]]
+_Callback: TypeAlias = Callable[[_SelectorContext, Iterable[Element]], Generator[Element, None, None]]
+_T = TypeVar("_T")
 
 def xpath_tokenizer(pattern: str, namespaces: dict[str, str] | None = None) -> Generator[_Token, None, None]: ...
 def get_parent_map(context: _SelectorContext) -> dict[Element, Element]: ...
 def prepare_child(next: _Next, token: _Token) -> _Callback: ...
 def prepare_star(next: _Next, token: _Token) -> _Callback: ...
 def prepare_self(next: _Next, token: _Token) -> _Callback: ...
-def prepare_descendant(next: _Next, token: _Token) -> _Callback: ...
+def prepare_descendant(next: _Next, token: _Token) -> _Callback | None: ...
 def prepare_parent(next: _Next, token: _Token) -> _Callback: ...
-def prepare_predicate(next: _Next, token: _Token) -> _Callback: ...
+def prepare_predicate(next: _Next, token: _Token) -> _Callback | None: ...
 
-ops: dict[str, Callable[[_Next, _Token], _Callback]]
+ops: dict[str, Callable[[_Next, _Token], _Callback | None]]
 
 class _SelectorContext:
     parent_map: dict[Element, Element] | None
     root: Element
     def __init__(self, root: Element) -> None: ...
 
-_T = TypeVar("_T")
-
-def iterfind(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ...
-def find(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ...
-def findall(elem: Element, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ...
-def findtext(elem: Element, path: str, default: _T | None = None, namespaces: dict[str, str] | None = None) -> _T | str: ...
+@overload
+def iterfind(  # type: ignore[overload-overlap]
+    elem: Element[Any], path: Literal[""], namespaces: dict[str, str] | None = None
+) -> None: ...
+@overload
+def iterfind(elem: Element[Any], path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ...
+def find(elem: Element[Any], path: str, namespaces: dict[str, str] | None = None) -> Element | None: ...
+def findall(elem: Element[Any], path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ...
+@overload
+def findtext(elem: Element[Any], path: str, default: None = None, namespaces: dict[str, str] | None = None) -> str | None: ...
+@overload
+def findtext(elem: Element[Any], path: str, default: _T, namespaces: dict[str, str] | None = None) -> _T | str: ...

stdlib/xml/etree/ElementTree.pyi

@@ -2,8 +2,9 @@ import sys
 from _collections_abc import dict_keys
 from _typeshed import FileDescriptorOrPath, ReadableBuffer, SupportsRead, SupportsWrite
 from collections.abc import Callable, Generator, ItemsView, Iterable, Iterator, Mapping, Sequence
-from typing import Any, Final, Literal, SupportsIndex, TypeVar, overload
+from typing import Any, Final, Generic, Literal, Protocol, SupportsIndex, TypeVar, overload, type_check_only
 from typing_extensions import TypeAlias, TypeGuard, deprecated
+from xml.parsers.expat import XMLParserType
 
 __all__ = [
     "C14NWriterTarget",
@@ -78,13 +79,22 @@ def canonicalize(
     exclude_tags: Iterable[str] | None = None,
 ) -> None: ...
 
-class Element:
-    tag: str
+# The tag for Element can be set to the Comment or ProcessingInstruction
+# functions defined in this module. _ElementCallable could be a recursive
+# type, but defining it that way uncovered a bug in pytype.
+_ElementCallable: TypeAlias = Callable[..., Element[Any]]
+_CallableElement: TypeAlias = Element[_ElementCallable]
+
+_Tag = TypeVar("_Tag", default=str, bound=str | _ElementCallable)
+_OtherTag = TypeVar("_OtherTag", default=str, bound=str | _ElementCallable)
+
+class Element(Generic[_Tag]):
+    tag: _Tag
     attrib: dict[str, str]
     text: str | None
     tail: str | None
-    def __init__(self, tag: str, attrib: dict[str, str] = ..., **extra: str) -> None: ...
-    def append(self, subelement: Element, /) -> None: ...
+    def __init__(self, tag: _Tag, attrib: dict[str, str] = {}, **extra: str) -> None: ...
+    def append(self, subelement: Element[Any], /) -> None: ...
     def clear(self) -> None: ...
     def extend(self, elements: Iterable[Element], /) -> None: ...
     def find(self, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ...
@@ -100,14 +110,17 @@ class Element:
     def insert(self, index: int, subelement: Element, /) -> None: ...
     def items(self) -> ItemsView[str, str]: ...
     def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ...
+    @overload
+    def iterfind(self, path: Literal[""], namespaces: dict[str, str] | None = None) -> None: ...  # type: ignore[overload-overlap]
+    @overload
     def iterfind(self, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ...
     def itertext(self) -> Generator[str, None, None]: ...
     def keys(self) -> dict_keys[str, str]: ...
     # makeelement returns the type of self in Python impl, but not in C impl
-    def makeelement(self, tag: str, attrib: dict[str, str], /) -> Element: ...
+    def makeelement(self, tag: _OtherTag, attrib: dict[str, str], /) -> Element[_OtherTag]: ...
     def remove(self, subelement: Element, /) -> None: ...
     def set(self, key: str, value: str, /) -> None: ...
-    def __copy__(self) -> Element: ...  # returns the type of self in Python impl, but not in C impl
+    def __copy__(self) -> Element[_Tag]: ...  # returns the type of self in Python impl, but not in C impl
     def __deepcopy__(self, memo: Any, /) -> Element: ...  # Only exists in C impl
     def __delitem__(self, key: SupportsIndex | slice, /) -> None: ...
     @overload
@@ -130,8 +143,8 @@ class Element:
         def getiterator(self, tag: str | None = None) -> list[Element]: ...
 
 def SubElement(parent: Element, tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ...
-def Comment(text: str | None = None) -> Element: ...
-def ProcessingInstruction(target: str, text: str | None = None) -> Element: ...
+def Comment(text: str | None = None) -> _CallableElement: ...
+def ProcessingInstruction(target: str, text: str | None = None) -> _CallableElement: ...
 
 PI = ProcessingInstruction
 
@@ -145,9 +158,11 @@ class QName:
     def __eq__(self, other: object) -> bool: ...
     def __hash__(self) -> int: ...
 
-class ElementTree:
+_Root = TypeVar("_Root", Element, Element | None, default=Element | None)
+
+class ElementTree(Generic[_Root]):
     def __init__(self, element: Element | None = None, file: _FileRead | None = None) -> None: ...
-    def getroot(self) -> Element | Any: ...
+    def getroot(self) -> _Root: ...
     def parse(self, source: _FileRead, parser: XMLParser | None = None) -> Element: ...
     def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ...
     if sys.version_info < (3, 9):
@@ -159,6 +174,9 @@ class ElementTree:
     @overload
     def findtext(self, path: str, default: _T, namespaces: dict[str, str] | None = None) -> _T | str: ...
     def findall(self, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ...
+    @overload
+    def iterfind(self, path: Literal[""], namespaces: dict[str, str] | None = None) -> None: ...  # type: ignore[overload-overlap]
+    @overload
     def iterfind(self, path: str, namespaces: dict[str, str] | None = None) -> Generator[Element, None, None]: ...
     def write(
         self,
@@ -166,18 +184,20 @@ class ElementTree:
         encoding: str | None = None,
         xml_declaration: bool | None = None,
         default_namespace: str | None = None,
-        method: str | None = None,
+        method: Literal["xml", "html", "text", "c14n"] | None = None,
         *,
         short_empty_elements: bool = True,
     ) -> None: ...
     def write_c14n(self, file: _FileWriteC14N) -> None: ...
 
+HTML_EMPTY: set[str]
+
 def register_namespace(prefix: str, uri: str) -> None: ...
 @overload
 def tostring(
     element: Element,
     encoding: None = None,
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
@@ -187,7 +207,7 @@ def tostring(
 def tostring(
     element: Element,
     encoding: Literal["unicode"],
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
@@ -197,7 +217,7 @@ def tostring(
 def tostring(
     element: Element,
     encoding: str,
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
@@ -207,7 +227,7 @@ def tostring(
 def tostringlist(
     element: Element,
     encoding: None = None,
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
@@ -217,7 +237,7 @@ def tostringlist(
 def tostringlist(
     element: Element,
     encoding: Literal["unicode"],
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
@@ -227,21 +247,23 @@ def tostringlist(
 def tostringlist(
     element: Element,
     encoding: str,
-    method: str | None = None,
+    method: Literal["xml", "html", "text", "c14n"] | None = None,
     *,
     xml_declaration: bool | None = None,
     default_namespace: str | None = None,
     short_empty_elements: bool = True,
 ) -> list[Any]: ...
-def dump(elem: Element) -> None: ...
+def dump(elem: Element | ElementTree[Any]) -> None: ...
 
 if sys.version_info >= (3, 9):
-    def indent(tree: Element | ElementTree, space: str = "  ", level: int = 0) -> None: ...
+    def indent(tree: Element | ElementTree[Any], space: str = "  ", level: int = 0) -> None: ...
 
-def parse(source: _FileRead, parser: XMLParser | None = None) -> ElementTree: ...
+def parse(source: _FileRead, parser: XMLParser[Any] | None = None) -> ElementTree[Element]: ...
 
-class _IterParseIterator(Iterator[tuple[str, Any]]):
-    def __next__(self) -> tuple[str, Any]: ...
+# This class is defined inside the body of iterparse
+@type_check_only
+class _IterParseIterator(Iterator[tuple[str, Element]], Protocol):
+    def __next__(self) -> tuple[str, Element]: ...
     if sys.version_info >= (3, 13):
         def close(self) -> None: ...
     if sys.version_info >= (3, 11):
@@ -249,13 +271,13 @@ class _IterParseIterator(Iterator[tuple[str, Any]]):
 
 def iterparse(source: _FileRead, events: Sequence[str] | None = None, parser: XMLParser | None = None) -> _IterParseIterator: ...
 
-class XMLPullParser:
-    def __init__(self, events: Sequence[str] | None = None, *, _parser: XMLParser | None = None) -> None: ...
+_EventQueue: TypeAlias = tuple[str] | tuple[str, tuple[str, str]] | tuple[str, None]
+
+class XMLPullParser(Generic[_E]):
+    def __init__(self, events: Sequence[str] | None = None, *, _parser: XMLParser[_E] | None = None) -> None: ...
     def feed(self, data: str | ReadableBuffer) -> None: ...
     def close(self) -> None: ...
-    # Second element in the tuple could be `Element`, `tuple[str, str]` or `None`.
-    # Use `Any` to avoid false-positive errors.
-    def read_events(self) -> Iterator[tuple[str, Any]]: ...
+    def read_events(self) -> Iterator[_EventQueue | tuple[str, _E]]: ...
     def flush(self) -> None: ...
 
 def XML(text: str | ReadableBuffer, parser: XMLParser | None = None) -> Element: ...
@@ -281,12 +303,12 @@ class TreeBuilder:
     # comment_factory can take None because passing None to Comment is not an error
     def __init__(
         self,
-        element_factory: _ElementFactory | None = ...,
+        element_factory: _ElementFactory | None = None,
         *,
-        comment_factory: Callable[[str | None], Element] | None = ...,
-        pi_factory: Callable[[str, str | None], Element] | None = ...,
-        insert_comments: bool = ...,
-        insert_pis: bool = ...,
+        comment_factory: Callable[[str | None], Element[Any]] | None = None,
+        pi_factory: Callable[[str, str | None], Element[Any]] | None = None,
+        insert_comments: bool = False,
+        insert_pis: bool = False,
     ) -> None: ...
     insert_comments: bool
     insert_pis: bool
@@ -298,8 +320,8 @@ class TreeBuilder:
     def start(self, tag: Any, attrs: dict[Any, Any], /) -> Element: ...
     def end(self, tag: str, /) -> Element: ...
     # These two methods have pos-only parameters in the C implementation
-    def comment(self, text: str | None, /) -> Element: ...
-    def pi(self, target: str, text: str | None = None, /) -> Element: ...
+    def comment(self, text: str | None, /) -> Element[Any]: ...
+    def pi(self, target: str, text: str | None = None, /) -> Element[Any]: ...
 
 class C14NWriterTarget:
     def __init__(
@@ -321,13 +343,33 @@ class C14NWriterTarget:
     def comment(self, text: str) -> None: ...
     def pi(self, target: str, data: str) -> None: ...
 
-class XMLParser:
-    parser: Any
-    target: Any
+# The target type is tricky, because the implementation doesn't
+# require any particular attribute to be present. This documents the attributes
+# that can be present, but uncommenting any of them would require them.
+class _Target(Protocol):
+    # start: Callable[str, dict[str, str], Any] | None
+    # end: Callable[[str], Any] | None
+    # start_ns: Callable[[str, str], Any] | None
+    # end_ns: Callable[[str], Any] | None
+    # data: Callable[[str], Any] | None
+    # comment: Callable[[str], Any]
+    # pi: Callable[[str, str], Any] | None
+    # close: Callable[[], Any] | None
+    ...
+
+_E = TypeVar("_E", default=Element)
+
+# This is generic because the return type of close() depends on the target.
+# The default target is TreeBuilder, which returns Element.
+# C14NWriterTarget does not implement a close method, so using it results
+# in a type of XMLParser[None].
+class XMLParser(Generic[_E]):
+    parser: XMLParserType
+    target: _Target
     # TODO-what is entity used for???
-    entity: Any
+    entity: dict[str, str]
     version: str
-    def __init__(self, *, target: Any = ..., encoding: str | None = ...) -> None: ...
-    def close(self) -> Any: ...
+    def __init__(self, *, target: _Target | None = None, encoding: str | None = None) -> None: ...
+    def close(self) -> _E: ...
     def feed(self, data: str | ReadableBuffer, /) -> None: ...
     def flush(self) -> None: ...