mirror of
https://github.com/davidhalter/typeshed.git
synced 2025-12-07 20:54:28 +08:00
Expand tokenize stub to include Intnumber etc (#3839)
The all uppercase tokens, as well as tok_name mentioned in the comment actually come from the `from token import *`.
This commit is contained in:
@@ -1,13 +1,8 @@
|
||||
from typing import Any, Callable, Generator, Iterable, List, NamedTuple, Optional, Pattern, Union, Sequence, TextIO, Tuple
|
||||
from typing import Any, Callable, Dict, Generator, Iterable, List, NamedTuple, Optional, Pattern, Union, Sequence, Set, TextIO, Tuple
|
||||
from builtins import open as _builtin_open
|
||||
import sys
|
||||
from token import * # noqa: F403
|
||||
|
||||
if sys.version_info < (3, 7):
|
||||
COMMENT: int
|
||||
NL: int
|
||||
ENCODING: int
|
||||
|
||||
cookie_re: Pattern[str]
|
||||
blank_re: Pattern[bytes]
|
||||
|
||||
@@ -28,7 +23,7 @@ class TokenInfo(_TokenInfo):
|
||||
_Token = Union[TokenInfo, Sequence[Union[int, str, _Position]]]
|
||||
|
||||
class TokenError(Exception): ...
|
||||
class StopTokenizing(Exception): ...
|
||||
class StopTokenizing(Exception): ... # undocumented
|
||||
|
||||
class Untokenizer:
|
||||
tokens: List[str]
|
||||
@@ -40,6 +35,8 @@ class Untokenizer:
|
||||
def untokenize(self, iterable: Iterable[_Token]) -> str: ...
|
||||
def compat(self, token: Sequence[Union[int, str]], iterable: Iterable[_Token]) -> None: ...
|
||||
|
||||
# the docstring says "returns bytes" but is incorrect --
|
||||
# if the ENCODING token is missing, it skips the encode
|
||||
def untokenize(iterable: Iterable[_Token]) -> Any: ...
|
||||
def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, Sequence[bytes]]: ...
|
||||
def tokenize(readline: Callable[[], bytes]) -> Generator[TokenInfo, None, None]: ...
|
||||
@@ -48,70 +45,61 @@ def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, N
|
||||
if sys.version_info >= (3, 6):
|
||||
from os import PathLike
|
||||
def open(filename: Union[str, bytes, int, PathLike[Any]]) -> TextIO: ...
|
||||
else:
|
||||
elif sys.version_info >= (3, 2):
|
||||
def open(filename: Union[str, bytes, int]) -> TextIO: ...
|
||||
|
||||
# Names in __all__ with no definition:
|
||||
# AMPER
|
||||
# AMPEREQUAL
|
||||
# ASYNC
|
||||
# AT
|
||||
# ATEQUAL
|
||||
# AWAIT
|
||||
# CIRCUMFLEX
|
||||
# CIRCUMFLEXEQUAL
|
||||
# COLON
|
||||
# COMMA
|
||||
# DEDENT
|
||||
# DOT
|
||||
# DOUBLESLASH
|
||||
# DOUBLESLASHEQUAL
|
||||
# DOUBLESTAR
|
||||
# DOUBLESTAREQUAL
|
||||
# ELLIPSIS
|
||||
# ENDMARKER
|
||||
# EQEQUAL
|
||||
# EQUAL
|
||||
# ERRORTOKEN
|
||||
# GREATER
|
||||
# GREATEREQUAL
|
||||
# INDENT
|
||||
# ISEOF
|
||||
# ISNONTERMINAL
|
||||
# ISTERMINAL
|
||||
# LBRACE
|
||||
# LEFTSHIFT
|
||||
# LEFTSHIFTEQUAL
|
||||
# LESS
|
||||
# LESSEQUAL
|
||||
# LPAR
|
||||
# LSQB
|
||||
# MINEQUAL
|
||||
# MINUS
|
||||
# NAME
|
||||
# NEWLINE
|
||||
# NOTEQUAL
|
||||
# NT_OFFSET
|
||||
# NUMBER
|
||||
# N_TOKENS
|
||||
# OP
|
||||
# PERCENT
|
||||
# PERCENTEQUAL
|
||||
# PLUS
|
||||
# PLUSEQUAL
|
||||
# RARROW
|
||||
# RBRACE
|
||||
# RIGHTSHIFT
|
||||
# RIGHTSHIFTEQUAL
|
||||
# RPAR
|
||||
# RSQB
|
||||
# SEMI
|
||||
# SLASH
|
||||
# SLASHEQUAL
|
||||
# STAR
|
||||
# STAREQUAL
|
||||
# STRING
|
||||
# TILDE
|
||||
# VBAR
|
||||
# VBAREQUAL
|
||||
# tok_name
|
||||
def group(*choices: str) -> str: ... # undocumented
|
||||
def any(*choices: str) -> str: ... # undocumented
|
||||
def maybe(*choices: str) -> str: ... # undocumented
|
||||
|
||||
Whitespace: str # undocumented
|
||||
Comment: str # undocumented
|
||||
Ignore: str # undocumented
|
||||
Name: str # undocumented
|
||||
|
||||
Hexnumber: str # undocumented
|
||||
Binnumber: str # undocumented
|
||||
Octnumber: str # undocumented
|
||||
Decnumber: str # undocumented
|
||||
Intnumber: str # undocumented
|
||||
Exponent: str # undocumented
|
||||
Pointfloat: str # undocumented
|
||||
Expfloat: str # undocumented
|
||||
Floatnumber: str # undocumented
|
||||
Imagnumber: str # undocumented
|
||||
Number: str # undocumented
|
||||
|
||||
def _all_string_prefixes() -> Set[str]: ... # undocumented
|
||||
|
||||
StringPrefix: str # undocumented
|
||||
|
||||
Single: str # undocumented
|
||||
Double: str # undocumented
|
||||
Single3: str # undocumented
|
||||
Double3: str # undocumented
|
||||
Triple: str # undocumented
|
||||
String: str # undocumented
|
||||
|
||||
if sys.version_info < (3, 7):
|
||||
Operator: str # undocumented
|
||||
Bracket: str # undocumented
|
||||
|
||||
Special: str # undocumented
|
||||
Funny: str # undocumented
|
||||
|
||||
PlainToken: str # undocumented
|
||||
Token: str # undocumented
|
||||
|
||||
ContStr: str # undocumented
|
||||
PseudoExtras: str # undocumented
|
||||
PseudoToken: str # undocumented
|
||||
|
||||
endpats: Dict[str, str] # undocumented
|
||||
if sys.version_info < (3, 6):
|
||||
single_quoted: Dict[str, str] # undocumented
|
||||
triple_quoted: Dict[str, str] # undocumented
|
||||
else:
|
||||
single_quoted: Set[str] # undocumented
|
||||
triple_quoted: Set[str] # undocumented
|
||||
|
||||
tabsize: int # undocumented
|
||||
|
||||
Reference in New Issue
Block a user