From 03119a237d6b9a35e1c7d3d034f3eafc89debf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 13 Apr 2017 20:14:35 -0700 Subject: [PATCH] Partial lib2to3 stubs (#1096) * Partial lib2to3 stubs * Use typing.Text instead of str * Add a _Path shortcut for managing PathLike * Respond to review, pgen2/driver * Respond to review, pgen2/grammar * Respond to review, pgen2/parse * Respond to review, pgen2/pgen * Respond to review, pgen2/token * Respond to review, pgen2/tokenize * Respond to review, pytree * Move to 2and3 * Make pytype happy * Respond to review nits * Move _RawNode, _Convert to pytree and make the latter return None * Make concrete Symbols subclasses for python and pattern * Add missing Callable import --- stdlib/2and3/lib2to3/__init__.pyi | 1 + stdlib/2and3/lib2to3/pgen2/__init__.pyi | 10 ++ stdlib/2and3/lib2to3/pgen2/driver.pyi | 24 +++++ stdlib/2and3/lib2to3/pgen2/grammar.pyi | 29 ++++++ stdlib/2and3/lib2to3/pgen2/literals.pyi | 9 ++ stdlib/2and3/lib2to3/pgen2/parse.pyi | 29 ++++++ stdlib/2and3/lib2to3/pgen2/pgen.pyi | 49 ++++++++++ stdlib/2and3/lib2to3/pgen2/token.pyi | 73 +++++++++++++++ stdlib/2and3/lib2to3/pgen2/tokenize.pyi | 30 ++++++ stdlib/2and3/lib2to3/pygram.pyi | 116 ++++++++++++++++++++++++ stdlib/2and3/lib2to3/pytree.pyi | 86 ++++++++++++++++++ 11 files changed, 456 insertions(+) create mode 100644 stdlib/2and3/lib2to3/__init__.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/__init__.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/driver.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/grammar.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/literals.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/parse.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/pgen.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/token.pyi create mode 100644 stdlib/2and3/lib2to3/pgen2/tokenize.pyi create mode 100644 stdlib/2and3/lib2to3/pygram.pyi create mode 100644 stdlib/2and3/lib2to3/pytree.pyi diff --git a/stdlib/2and3/lib2to3/__init__.pyi b/stdlib/2and3/lib2to3/__init__.pyi new file mode 100644 index 000000000..145e31b1d --- /dev/null +++ b/stdlib/2and3/lib2to3/__init__.pyi @@ -0,0 +1 @@ +# Stubs for lib2to3 (Python 3.6) diff --git a/stdlib/2and3/lib2to3/pgen2/__init__.pyi b/stdlib/2and3/lib2to3/pgen2/__init__.pyi new file mode 100644 index 000000000..1adc82a6e --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/__init__.pyi @@ -0,0 +1,10 @@ +# Stubs for lib2to3.pgen2 (Python 3.6) + +import os +import sys +from typing import Text, Union + +if sys.version_info >= (3, 6): + _Path = Union[Text, os.PathLike] +else: + _Path = Text diff --git a/stdlib/2and3/lib2to3/pgen2/driver.pyi b/stdlib/2and3/lib2to3/pgen2/driver.pyi new file mode 100644 index 000000000..56785f087 --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/driver.pyi @@ -0,0 +1,24 @@ +# Stubs for lib2to3.pgen2.driver (Python 3.6) + +import os +import sys +from typing import Any, Callable, IO, Iterable, List, Optional, Text, Tuple, Union + +from logging import Logger +from lib2to3.pytree import _Convert, _NL +from lib2to3.pgen2 import _Path +from lib2to3.pgen2.grammar import Grammar + + +class Driver: + grammar: Grammar + logger: Logger + convert: _Convert + def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ..., logger: Optional[Logger] = ...) -> None: ... + def parse_tokens(self, tokens: Iterable[Any], debug: bool = ...) -> _NL: ... + def parse_stream_raw(self, stream: IO[Text], debug: bool = ...) -> _NL: ... + def parse_stream(self, stream: IO[Text], debug: bool = ...) -> _NL: ... + def parse_file(self, filename: _Path, encoding: Optional[Text] = ..., debug: bool = ...) -> _NL: ... + def parse_string(self, text: Text, debug: bool = ...) -> _NL: ... + +def load_grammar(gt: Text = ..., gp: Optional[Text] = ..., save: bool = ..., force: bool = ..., logger: Optional[Logger] = ...) -> Grammar: ... diff --git a/stdlib/2and3/lib2to3/pgen2/grammar.pyi b/stdlib/2and3/lib2to3/pgen2/grammar.pyi new file mode 100644 index 000000000..122d771db --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/grammar.pyi @@ -0,0 +1,29 @@ +# Stubs for lib2to3.pgen2.grammar (Python 3.6) + +from lib2to3.pgen2 import _Path + +from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar + +_P = TypeVar('_P') +_Label = Tuple[int, Optional[Text]] +_DFA = List[List[Tuple[int, int]]] +_DFAS = Tuple[_DFA, Dict[int, int]] + +class Grammar: + symbol2number: Dict[Text, int] + number2symbol: Dict[int, Text] + states: List[_DFA] + dfas: Dict[int, _DFAS] + labels: List[_Label] + keywords: Dict[Text, int] + tokens: Dict[int, int] + symbol2label: Dict[Text, int] + start: int + def __init__(self) -> None: ... + def dump(self, filename: _Path) -> None: ... + def load(self, filename: _Path) -> None: ... + def copy(self: _P) -> _P: ... + def report(self) -> None: ... + +opmap_raw: Text +opmap: Dict[Text, Text] diff --git a/stdlib/2and3/lib2to3/pgen2/literals.pyi b/stdlib/2and3/lib2to3/pgen2/literals.pyi new file mode 100644 index 000000000..8719500da --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/literals.pyi @@ -0,0 +1,9 @@ +# Stubs for lib2to3.pgen2.literals (Python 3.6) + +from typing import Dict, Match, Text + +simple_escapes: Dict[Text, Text] + +def escape(m: Match) -> Text: ... +def evalString(s: Text) -> Text: ... +def test() -> None: ... diff --git a/stdlib/2and3/lib2to3/pgen2/parse.pyi b/stdlib/2and3/lib2to3/pgen2/parse.pyi new file mode 100644 index 000000000..101d4760a --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/parse.pyi @@ -0,0 +1,29 @@ +# Stubs for lib2to3.pgen2.parse (Python 3.6) + +from typing import Any, Dict, List, Optional, Sequence, Set, Text, Tuple + +from lib2to3.pgen2.grammar import Grammar, _DFAS +from lib2to3.pytree import _NL, _Convert, _RawNode + +_Context = Sequence[Any] + +class ParseError(Exception): + msg: Text + type: int + value: Optional[Text] + context: _Context + def __init__(self, msg: Text, type: int, value: Optional[Text], context: _Context) -> None: ... + +class Parser: + grammar: Grammar + convert: _Convert + stack: List[Tuple[_DFAS, int, _RawNode]] + rootnode: Optional[_NL] + used_names: Set[Text] + def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ...) -> None: ... + def setup(self, start: Optional[int] = ...) -> None: ... + def addtoken(self, type: int, value: Optional[Text], context: _Context) -> bool: ... + def classify(self, type: int, value: Optional[Text], context: _Context) -> int: ... + def shift(self, type: int, value: Optional[Text], newstate: int, context: _Context) -> None: ... + def push(self, type: int, newdfa: _DFAS, newstate: int, context: _Context) -> None: ... + def pop(self) -> None: ... diff --git a/stdlib/2and3/lib2to3/pgen2/pgen.pyi b/stdlib/2and3/lib2to3/pgen2/pgen.pyi new file mode 100644 index 000000000..4180a230a --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/pgen.pyi @@ -0,0 +1,49 @@ +# Stubs for lib2to3.pgen2.pgen (Python 3.6) + +from typing import Any, Dict, IO, Iterable, Iterator, List, Optional, Text, Tuple +from mypy_extensions import NoReturn + +from lib2to3.pgen2 import _Path, grammar +from lib2to3.pgen2.tokenize import _TokenInfo + +class PgenGrammar(grammar.Grammar): ... + +class ParserGenerator: + filename: _Path + stream: IO[Text] + generator: Iterator[_TokenInfo] + first: Dict[Text, Dict[Text, int]] + def __init__(self, filename: _Path, stream: Optional[IO[Text]] = ...) -> None: ... + def make_grammar(self) -> PgenGrammar: ... + def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: ... + def make_label(self, c: PgenGrammar, label: Text) -> int: ... + def addfirstsets(self) -> None: ... + def calcfirst(self, name: Text) -> None: ... + def parse(self) -> Tuple[Dict[Text, List[DFAState]], Text]: ... + def make_dfa(self, start: NFAState, finish: NFAState) -> List[DFAState]: ... + def dump_nfa(self, name: Text, start: NFAState, finish: NFAState) -> List[DFAState]: ... + def dump_dfa(self, name: Text, dfa: Iterable[DFAState]) -> None: ... + def simplify_dfa(self, dfa: List[DFAState]) -> None: ... + def parse_rhs(self) -> Tuple[NFAState, NFAState]: ... + def parse_alt(self) -> Tuple[NFAState, NFAState]: ... + def parse_item(self) -> Tuple[NFAState, NFAState]: ... + def parse_atom(self) -> Tuple[NFAState, NFAState]: ... + def expect(self, type: int, value: Optional[Any] = ...) -> Text: ... + def gettoken(self) -> None: ... + def raise_error(self, msg: str, *args: Any) -> NoReturn: ... + +class NFAState: + arcs: List[Tuple[Optional[Text], NFAState]] + def __init__(self) -> None: ... + def addarc(self, next: NFAState, label: Optional[Text] = ...) -> None: ... + +class DFAState: + nfaset: Dict[NFAState, Any] + isfinal: bool + arcs: Dict[Text, DFAState] + def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: ... + def addarc(self, next: DFAState, label: Text) -> None: ... + def unifystate(self, old: DFAState, new: DFAState) -> None: ... + def __eq__(self, other: Any) -> bool: ... + +def generate_grammar(filename: _Path = ...) -> PgenGrammar: ... diff --git a/stdlib/2and3/lib2to3/pgen2/token.pyi b/stdlib/2and3/lib2to3/pgen2/token.pyi new file mode 100644 index 000000000..c256af8f2 --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/token.pyi @@ -0,0 +1,73 @@ +# Stubs for lib2to3.pgen2.token (Python 3.6) + +import sys +from typing import Dict, Text + +ENDMARKER: int +NAME: int +NUMBER: int +STRING: int +NEWLINE: int +INDENT: int +DEDENT: int +LPAR: int +RPAR: int +LSQB: int +RSQB: int +COLON: int +COMMA: int +SEMI: int +PLUS: int +MINUS: int +STAR: int +SLASH: int +VBAR: int +AMPER: int +LESS: int +GREATER: int +EQUAL: int +DOT: int +PERCENT: int +BACKQUOTE: int +LBRACE: int +RBRACE: int +EQEQUAL: int +NOTEQUAL: int +LESSEQUAL: int +GREATEREQUAL: int +TILDE: int +CIRCUMFLEX: int +LEFTSHIFT: int +RIGHTSHIFT: int +DOUBLESTAR: int +PLUSEQUAL: int +MINEQUAL: int +STAREQUAL: int +SLASHEQUAL: int +PERCENTEQUAL: int +AMPEREQUAL: int +VBAREQUAL: int +CIRCUMFLEXEQUAL: int +LEFTSHIFTEQUAL: int +RIGHTSHIFTEQUAL: int +DOUBLESTAREQUAL: int +DOUBLESLASH: int +DOUBLESLASHEQUAL: int +OP: int +COMMENT: int +NL: int +if sys.version_info >= (3,): + RARROW: int +if sys.version_info >= (3, 5): + AT: int + ATEQUAL: int + AWAIT: int + ASYNC: int +ERRORTOKEN: int +N_TOKENS: int +NT_OFFSET: int +tok_name: Dict[int, Text] + +def ISTERMINAL(x: int) -> bool: ... +def ISNONTERMINAL(x: int) -> bool: ... +def ISEOF(x: int) -> bool: ... diff --git a/stdlib/2and3/lib2to3/pgen2/tokenize.pyi b/stdlib/2and3/lib2to3/pgen2/tokenize.pyi new file mode 100644 index 000000000..c10305ffd --- /dev/null +++ b/stdlib/2and3/lib2to3/pgen2/tokenize.pyi @@ -0,0 +1,30 @@ +# Stubs for lib2to3.pgen2.tokenize (Python 3.6) +# NOTE: Only elements from __all__ are present. + +from typing import Callable, Iterable, Iterator, List, Text, Tuple +from lib2to3.pgen2.token import * # noqa + + +_Coord = Tuple[int, int] +_TokenEater = Callable[[int, Text, _Coord, _Coord, Text], None] +_TokenInfo = Tuple[int, Text, _Coord, _Coord, Text] + + +class TokenError(Exception): ... +class StopTokenizing(Exception): ... + +def tokenize(readline: Callable[[], Text], tokeneater: _TokenEater = ...) -> None: ... + +class Untokenizer: + tokens: List[Text] + prev_row: int + prev_col: int + def __init__(self) -> None: ... + def add_whitespace(self, start: _Coord) -> None: ... + def untokenize(self, iterable: Iterable[_TokenInfo]) -> Text: ... + def compat(self, token: Tuple[int, Text], iterable: Iterable[_TokenInfo]) -> None: ... + +def untokenize(iterable: Iterable[_TokenInfo]) -> Text: ... +def generate_tokens( + readline: Callable[[], Text] +) -> Iterator[_TokenInfo]: ... diff --git a/stdlib/2and3/lib2to3/pygram.pyi b/stdlib/2and3/lib2to3/pygram.pyi new file mode 100644 index 000000000..aeb7b93bf --- /dev/null +++ b/stdlib/2and3/lib2to3/pygram.pyi @@ -0,0 +1,116 @@ +# Stubs for lib2to3.pygram (Python 3.6) + +from typing import Any +from lib2to3.pgen2.grammar import Grammar + +class Symbols: + def __init__(self, grammar: Grammar) -> None: ... + +class python_symbols(Symbols): + and_expr: int + and_test: int + annassign: int + arglist: int + argument: int + arith_expr: int + assert_stmt: int + async_funcdef: int + async_stmt: int + atom: int + augassign: int + break_stmt: int + classdef: int + comp_for: int + comp_if: int + comp_iter: int + comp_op: int + comparison: int + compound_stmt: int + continue_stmt: int + decorated: int + decorator: int + decorators: int + del_stmt: int + dictsetmaker: int + dotted_as_name: int + dotted_as_names: int + dotted_name: int + encoding_decl: int + eval_input: int + except_clause: int + exec_stmt: int + expr: int + expr_stmt: int + exprlist: int + factor: int + file_input: int + flow_stmt: int + for_stmt: int + funcdef: int + global_stmt: int + if_stmt: int + import_as_name: int + import_as_names: int + import_from: int + import_name: int + import_stmt: int + lambdef: int + listmaker: int + not_test: int + old_lambdef: int + old_test: int + or_test: int + parameters: int + pass_stmt: int + power: int + print_stmt: int + raise_stmt: int + return_stmt: int + shift_expr: int + simple_stmt: int + single_input: int + sliceop: int + small_stmt: int + star_expr: int + stmt: int + subscript: int + subscriptlist: int + suite: int + term: int + test: int + testlist: int + testlist1: int + testlist_gexp: int + testlist_safe: int + testlist_star_expr: int + tfpdef: int + tfplist: int + tname: int + trailer: int + try_stmt: int + typedargslist: int + varargslist: int + vfpdef: int + vfplist: int + vname: int + while_stmt: int + with_item: int + with_stmt: int + with_var: int + xor_expr: int + yield_arg: int + yield_expr: int + yield_stmt: int + +class pattern_symbols(Symbols): + Alternative: int + Alternatives: int + Details: int + Matcher: int + NegatedUnit: int + Repeater: int + Unit: int + +python_grammar: Grammar +python_grammar_no_print_statement: Grammar +pattern_grammar: Grammar diff --git a/stdlib/2and3/lib2to3/pytree.pyi b/stdlib/2and3/lib2to3/pytree.pyi new file mode 100644 index 000000000..06a7c1229 --- /dev/null +++ b/stdlib/2and3/lib2to3/pytree.pyi @@ -0,0 +1,86 @@ +# Stubs for lib2to3.pytree (Python 3.6) + +import sys +from typing import Any, Callable, Dict, Iterator, List, Optional, Text, Tuple, TypeVar, Union + +from lib2to3.pgen2.grammar import Grammar + +_P = TypeVar('_P') +_NL = Union[Node, Leaf] +_Context = Tuple[Text, int, int] +_Results = Dict[Text, _NL] +_RawNode = Tuple[int, Text, _Context, Optional[List[_NL]]] +_Convert = Callable[[Grammar, _RawNode], Any] + +HUGE: int + +def type_repr(type_num: int) -> Text: ... + +class Base: + type: int + parent: Optional[Node] + prefix: Text + children: List[_NL] + was_changed: bool + was_checked: bool + def __eq__(self, other: Any) -> bool: ... + def _eq(self: _P, other: _P) -> bool: ... + def clone(self: _P) -> _P: ... + def post_order(self) -> Iterator[_NL]: ... + def pre_order(self) -> Iterator[_NL]: ... + def replace(self, new: Union[_NL, List[_NL]]) -> None: ... + def get_lineno(self) -> int: ... + def changed(self) -> None: ... + def remove(self) -> Optional[int]: ... + @property + def next_sibling(self) -> Optional[_NL]: ... + @property + def prev_sibling(self) -> Optional[_NL]: ... + def leaves(self) -> Iterator[Leaf]: ... + def depth(self) -> int: ... + def get_suffix(self) -> Text: ... + if sys.version_info < (3,): + def get_prefix(self) -> Text: ... + def set_prefix(self, prefix: Text) -> None: ... + +class Node(Base): + fixers_applied: List[Any] + def __init__(self, type: int, children: List[_NL], context: Optional[Any] = ..., prefix: Optional[Text] = ..., fixers_applied: Optional[List[Any]] = ...) -> None: ... + def set_child(self, i: int, child: _NL) -> None: ... + def insert_child(self, i: int, child: _NL) -> None: ... + def append_child(self, child: _NL) -> None: ... + +class Leaf(Base): + lineno: int + column: int + value: Text + fixers_applied: List[Any] + def __init__(self, type: int, value: Text, context: Optional[_Context] = ..., prefix: Optional[Text] = ..., fixers_applied: List[Any] = ...) -> None: ... + +def convert(gr: Grammar, raw_node: _RawNode) -> _NL: ... + +class BasePattern: + type: int + content: Optional[Text] + name: Optional[Text] + def optimize(self) -> BasePattern: ... # sic, subclasses are free to optimize themselves into different patterns + def match(self, node: _NL, results: Optional[_Results] = ...) -> bool: ... + def match_seq(self, nodes: List[_NL], results: Optional[_Results] = ...) -> bool: ... + def generate_matches(self, nodes: List[_NL]) -> Iterator[Tuple[int, _Results]]: ... + +class LeafPattern(BasePattern): + def __init__(self, type: Optional[int] = ..., content: Optional[Text] = ..., name: Optional[Text] = ...) -> None: ... + +class NodePattern(BasePattern): + wildcards: bool + def __init__(self, type: Optional[int] = ..., content: Optional[Text] = ..., name: Optional[Text] = ...) -> None: ... + +class WildcardPattern(BasePattern): + min: int + max: int + def __init__(self, content: Optional[Text] = ..., min: int = ..., max: int = ..., name: Optional[Text] = ...) -> None: ... + +class NegatedPattern(BasePattern): + def __init__(self, content: Optional[Text] = ...) -> None: ... + +def generate_matches(patterns: List[BasePattern], nodes: List[_NL]) -> Iterator[Tuple[int, _Results]]: ...