From b4dc047307797dcade84de9b3dc75d99f5ffc70b Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 8 Oct 2025 07:48:18 -0400 Subject: [PATCH] [yt-dlp] Check common code in extractor subpackage with stubtest (#14682) --- stubs/yt-dlp/@tests/stubtest_allowlist.txt | 2 +- stubs/yt-dlp/yt_dlp/extractor/common.pyi | 80 +++++++++++-------- .../yt_dlp/extractor/commonmistakes.pyi | 8 +- .../yt_dlp/extractor/commonprotocols.pyi | 8 +- 4 files changed, 59 insertions(+), 39 deletions(-) diff --git a/stubs/yt-dlp/@tests/stubtest_allowlist.txt b/stubs/yt-dlp/@tests/stubtest_allowlist.txt index eec37a338..293acf692 100644 --- a/stubs/yt-dlp/@tests/stubtest_allowlist.txt +++ b/stubs/yt-dlp/@tests/stubtest_allowlist.txt @@ -1,5 +1,5 @@ # Extractors will not be stubbed at this time. -yt_dlp.extractor.* +yt_dlp.extractor\.(?!__init__|common*).* # Postprocessors will not be stubbed at this time. yt_dlp.postprocessor.* # Won't be covered. diff --git a/stubs/yt-dlp/yt_dlp/extractor/common.pyi b/stubs/yt-dlp/yt_dlp/extractor/common.pyi index 5cd06e6d0..d0c6b3523 100644 --- a/stubs/yt-dlp/yt_dlp/extractor/common.pyi +++ b/stubs/yt-dlp/yt_dlp/extractor/common.pyi @@ -3,8 +3,8 @@ from _typeshed import Unused from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence from functools import cached_property from json.decoder import JSONDecoder -from typing import Any, Literal, TypedDict, TypeVar, type_check_only -from typing_extensions import Required, TypeAlias +from typing import Any, ClassVar, Literal, TypedDict, TypeVar, overload, type_check_only +from typing_extensions import Never, Required, TypeAlias, deprecated from urllib.request import Request, _DataType from xml.etree import ElementTree as ET @@ -12,7 +12,7 @@ from ..cache import Cache from ..cookies import LenientSimpleCookie, YoutubeDLCookieJar from ..networking.common import Response, _RequestData from ..networking.impersonate import ImpersonateTarget -from ..utils._utils import NO_DEFAULT, RetryManager as _RetryManager +from ..utils._utils import NO_DEFAULT, ExtractorError, FormatSorter, RetryManager as _RetryManager, classproperty from ..YoutubeDL import YoutubeDL @type_check_only @@ -37,9 +37,9 @@ _StrNoDefaultOrNone: TypeAlias = str | None | type[NO_DEFAULT] _T = TypeVar("_T") class InfoExtractor: - IE_DESC: str | bool - SEARCH_KEY: str - def _login_hint(self, method: _StrNoDefaultOrNone, netrc: str | None = None) -> dict[str, str]: ... + IE_DESC: ClassVar[str | bool | None] + SEARCH_KEY: ClassVar[str | None] + def _login_hint(self, method: _StrNoDefaultOrNone = ..., netrc: str | None = None) -> dict[str, str]: ... def __init__(self, downloader: YoutubeDL | None = None) -> None: ... @classmethod def _match_valid_url(cls, url: str) -> re.Match[str] | None: ... @@ -82,8 +82,8 @@ class InfoExtractor: fatal: bool = True, encoding: str | None = None, data: _DataType | None = None, - headers: Mapping[str, str] | None = None, - query: str | Mapping[str, str] | None = None, + headers: Mapping[str, str] | None = {}, + query: str | Mapping[str, str] | None = {}, expected_status: int | None = None, impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, @@ -128,14 +128,21 @@ class InfoExtractor: parse_constant: Callable[[str], Any] | None = None, object_pairs_hook: Callable[[list[tuple[Any, Any]]], Any] | None = None, ) -> Any: ... - def report_warning(self, msg: str, video_id: str | None = None, only_once: bool = False) -> None: ... + # *args and **kwargs are passed to self._downloader.report_warning(). + def report_warning( + self, msg: str, video_id: str | None = None, *args: Any, only_once: bool = False, **kwargs: Any + ) -> None: ... def to_screen( self, msg: str, message: str, skip_eol: bool = False, quiet: bool | None = None, only_once: bool = False ) -> None: ... def write_debug(self, msg: str, only_once: bool = False) -> None: ... # *args and **kwargs are passed to .params.get() where params is normally a mapping but is not required to be. def get_param(self, name: str, default: Any = None, *args: Any, **kwargs: Any) -> Any: ... - def report_drm(self, video_id: str) -> None: ... + @overload + def report_drm(self, video_id: str, partial: type[NO_DEFAULT] = ...) -> None: ... + @overload + @deprecated("InfoExtractor.report_drm no longer accepts the argument partial") + def report_drm(self, video_id: str, partial: bool) -> None: ... def report_extraction(self, id_or_name: str) -> None: ... def report_download_webpage(self, video_id: str) -> None: ... def report_age_confirmation(self) -> None: ... @@ -149,7 +156,12 @@ class InfoExtractor: def raise_geo_restricted( self, msg: str = ..., countries: Collection[str] | None = None, metadata_available: bool = False ) -> None: ... - def raise_no_formats(self, msg: str, expected: bool = False, video_id: str | None = None) -> None: ... + @overload + def raise_no_formats( + self, msg: str | ExtractorError, expected: Literal[False] = False, video_id: str | None = None + ) -> Never: ... + @overload + def raise_no_formats(self, msg: str | ExtractorError, expected: Literal[True], video_id: str | None = None) -> None: ... @staticmethod def url_result( url: str, @@ -169,7 +181,6 @@ class InfoExtractor: getter: Callable[..., Any] = ..., ie: InfoExtractor | None = None, video_kwargs: Mapping[str, Any] | None = None, - multi_video: bool = False, **kwargs: Any, # Added to the dict return value. ) -> dict[str, Any]: ... @staticmethod @@ -212,7 +223,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> tuple[ET.ElementTree, Response]: ... def _download_xml( @@ -228,7 +239,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> ET.ElementTree: ... def _download_socket_json_handle( @@ -244,7 +255,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> tuple[dict[str, Any], Response]: ... def _download_socket_json( @@ -260,7 +271,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> dict[str, Any]: ... def _download_json_handle( @@ -276,7 +287,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> tuple[dict[str, Any], Response]: ... def _download_json( @@ -292,7 +303,7 @@ class InfoExtractor: headers: Mapping[str, str] = {}, query: Mapping[str, str] = {}, expected_status: int | None = None, - impersonate: str | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, require_impersonation: bool = False, ) -> dict[str, Any]: ... def _download_webpage( @@ -301,14 +312,17 @@ class InfoExtractor: video_id: str, note: str | None = None, errnote: str | None = None, - transform_source: Callable[..., str] | None = ..., fatal: bool = True, + tries: int = 1, + timeout: float | type[NO_DEFAULT] = ..., + # Remaining arguments are collected with *args, **kwargs and + # forwarded to _download_webpage_handle(). encoding: str | None = ..., data: _DataType | None = ..., headers: Mapping[str, str] = ..., query: Mapping[str, str] = ..., expected_status: int | None = ..., - impersonate: str | None = ..., + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = ..., require_impersonation: bool = ..., ) -> str: ... def _parse_xml( @@ -333,7 +347,7 @@ class InfoExtractor: ) -> Literal["needs_auth", "premium_only", "private", "public", "subscriber_only", "unlisted"] | None: ... def _request_webpage( self, - url_or_req: str | Request, + url_or_request: str | Request, video_id: str, note: str | None = None, errnote: str | None = None, @@ -342,6 +356,8 @@ class InfoExtractor: headers: Mapping[str, str] | None = None, query: Mapping[str, str] | None = None, expected_status: int | None = None, + impersonate: ImpersonateTarget | str | bool | Collection[str | ImpersonateTarget] | None = None, + require_impersonation: bool = False, ) -> Response | Literal[False]: ... @classmethod def _match_id(cls, url: str) -> str: ... @@ -418,15 +434,7 @@ class InfoExtractor: group: tuple[int, ...] | list[int] | None = None, fatal: bool = False, ) -> str | None: ... - def _html_search_meta( - self, - name: str, - html: str, - display_name: str | None = None, - fatal: bool = False, - flags: int = 0, - group: tuple[int, ...] | list[int] | None = None, - ) -> str | None: ... + def _html_search_meta(self, name: str, html: str, display_name: str | None = None, fatal: bool = False) -> str | None: ... def _dc_search_uploader(self, html: str) -> str | None: ... @staticmethod def _rta_search(html: str) -> int: ... @@ -464,6 +472,11 @@ class InfoExtractor: @staticmethod def _hidden_inputs(html: str) -> dict[str, Any]: ... def _form_hidden_inputs(self, form_id: str, html: str) -> dict[str, Any]: ... + @classproperty + @deprecated( + "yt_dlp.InfoExtractor.FormatSort is deprecated and may be removed in the future. Use yt_dlp.utils.FormatSorter instead" + ) + def FormatSort(self) -> FormatSorter: ... def _check_formats(self, formats: list[dict[str, Any]], video_id: str) -> None: ... @staticmethod def _remove_duplicate_formats(formats: list[dict[str, Any]]) -> None: ... @@ -792,7 +805,7 @@ class InfoExtractor: def _live_title(self, name: _T) -> _T: ... def _get_cookies(self, url: str) -> LenientSimpleCookie: ... def _apply_first_set_cookie_header(self, url_handle: Response, cookie: str) -> None: ... - @property + @classproperty def _RETURN_TYPE(cls) -> str: ... def _get_subtitles(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]: ... # Not implemented here. # Passes *args and **kwargs to _get_comments. @@ -864,5 +877,8 @@ class SearchInfoExtractor(InfoExtractor): def _real_extract(self, query: str) -> _InfoDict: ... def _get_n_results(self, query: str, n: int) -> list[_InfoDict]: ... def _search_results(self, query: str) -> list[_InfoDict]: ... + @classproperty + def SEARCH_KEY(self) -> str | None: ... -class UnsupportedURLIE(InfoExtractor): ... +class UnsupportedURLIE(InfoExtractor): + IE_DESC: ClassVar[bool] diff --git a/stubs/yt-dlp/yt_dlp/extractor/commonmistakes.pyi b/stubs/yt-dlp/yt_dlp/extractor/commonmistakes.pyi index 0b33ccd54..2645cf3ac 100644 --- a/stubs/yt-dlp/yt_dlp/extractor/commonmistakes.pyi +++ b/stubs/yt-dlp/yt_dlp/extractor/commonmistakes.pyi @@ -1,10 +1,12 @@ +from typing import ClassVar + from .common import InfoExtractor class CommonMistakesIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool] class UnicodeBOMIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool] class BlobIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool] diff --git a/stubs/yt-dlp/yt_dlp/extractor/commonprotocols.pyi b/stubs/yt-dlp/yt_dlp/extractor/commonprotocols.pyi index 495cdd496..a22cf411f 100644 --- a/stubs/yt-dlp/yt_dlp/extractor/commonprotocols.pyi +++ b/stubs/yt-dlp/yt_dlp/extractor/commonprotocols.pyi @@ -1,10 +1,12 @@ +from typing import ClassVar + from .common import InfoExtractor class RtmpIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool] class MmsIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool] class ViewSourceIE(InfoExtractor): - IE_DESC: bool + IE_DESC: ClassVar[bool]