Re-organize directory structure (#4971)

See discussion in #2491

Co-authored-by: Ivan Levkivskyi <ilevkivskyi@dropbox.com>

stdlib/lib2to3/pgen2/driver.pyi

@@ -0,0 +1,20 @@
+from _typeshed import StrPath
+from lib2to3.pgen2.grammar import Grammar
+from lib2to3.pytree import _NL, _Convert
+from logging import Logger
+from typing import IO, Any, Iterable, Optional, Text
+
+class Driver:
+    grammar: Grammar
+    logger: Logger
+    convert: _Convert
+    def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ..., logger: Optional[Logger] = ...) -> None: ...
+    def parse_tokens(self, tokens: Iterable[Any], debug: bool = ...) -> _NL: ...
+    def parse_stream_raw(self, stream: IO[Text], debug: bool = ...) -> _NL: ...
+    def parse_stream(self, stream: IO[Text], debug: bool = ...) -> _NL: ...
+    def parse_file(self, filename: StrPath, encoding: Optional[Text] = ..., debug: bool = ...) -> _NL: ...
+    def parse_string(self, text: Text, debug: bool = ...) -> _NL: ...
+
+def load_grammar(
+    gt: Text = ..., gp: Optional[Text] = ..., save: bool = ..., force: bool = ..., logger: Optional[Logger] = ...
+) -> Grammar: ...

stdlib/lib2to3/pgen2/grammar.pyi

@@ -0,0 +1,26 @@
+from _typeshed import StrPath
+from typing import Dict, List, Optional, Text, Tuple, TypeVar
+
+_P = TypeVar("_P")
+_Label = Tuple[int, Optional[Text]]
+_DFA = List[List[Tuple[int, int]]]
+_DFAS = Tuple[_DFA, Dict[int, int]]
+
+class Grammar:
+    symbol2number: Dict[Text, int]
+    number2symbol: Dict[int, Text]
+    states: List[_DFA]
+    dfas: Dict[int, _DFAS]
+    labels: List[_Label]
+    keywords: Dict[Text, int]
+    tokens: Dict[int, int]
+    symbol2label: Dict[Text, int]
+    start: int
+    def __init__(self) -> None: ...
+    def dump(self, filename: StrPath) -> None: ...
+    def load(self, filename: StrPath) -> None: ...
+    def copy(self: _P) -> _P: ...
+    def report(self) -> None: ...
+
+opmap_raw: Text
+opmap: Dict[Text, Text]

stdlib/lib2to3/pgen2/literals.pyi

@@ -0,0 +1,7 @@
+from typing import Dict, Match, Text
+
+simple_escapes: Dict[Text, Text]
+
+def escape(m: Match[str]) -> Text: ...
+def evalString(s: Text) -> Text: ...
+def test() -> None: ...

stdlib/lib2to3/pgen2/parse.pyi

@@ -0,0 +1,26 @@
+from lib2to3.pgen2.grammar import _DFAS, Grammar
+from lib2to3.pytree import _NL, _Convert, _RawNode
+from typing import Any, List, Optional, Sequence, Set, Text, Tuple
+
+_Context = Sequence[Any]
+
+class ParseError(Exception):
+    msg: Text
+    type: int
+    value: Optional[Text]
+    context: _Context
+    def __init__(self, msg: Text, type: int, value: Optional[Text], context: _Context) -> None: ...
+
+class Parser:
+    grammar: Grammar
+    convert: _Convert
+    stack: List[Tuple[_DFAS, int, _RawNode]]
+    rootnode: Optional[_NL]
+    used_names: Set[Text]
+    def __init__(self, grammar: Grammar, convert: Optional[_Convert] = ...) -> None: ...
+    def setup(self, start: Optional[int] = ...) -> None: ...
+    def addtoken(self, type: int, value: Optional[Text], context: _Context) -> bool: ...
+    def classify(self, type: int, value: Optional[Text], context: _Context) -> int: ...
+    def shift(self, type: int, value: Optional[Text], newstate: int, context: _Context) -> None: ...
+    def push(self, type: int, newdfa: _DFAS, newstate: int, context: _Context) -> None: ...
+    def pop(self) -> None: ...

stdlib/lib2to3/pgen2/pgen.pyi

@@ -0,0 +1,46 @@
+from _typeshed import StrPath
+from lib2to3.pgen2 import grammar
+from lib2to3.pgen2.tokenize import _TokenInfo
+from typing import IO, Any, Dict, Iterable, Iterator, List, NoReturn, Optional, Text, Tuple
+
+class PgenGrammar(grammar.Grammar): ...
+
+class ParserGenerator:
+    filename: StrPath
+    stream: IO[Text]
+    generator: Iterator[_TokenInfo]
+    first: Dict[Text, Dict[Text, int]]
+    def __init__(self, filename: StrPath, stream: Optional[IO[Text]] = ...) -> None: ...
+    def make_grammar(self) -> PgenGrammar: ...
+    def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: ...
+    def make_label(self, c: PgenGrammar, label: Text) -> int: ...
+    def addfirstsets(self) -> None: ...
+    def calcfirst(self, name: Text) -> None: ...
+    def parse(self) -> Tuple[Dict[Text, List[DFAState]], Text]: ...
+    def make_dfa(self, start: NFAState, finish: NFAState) -> List[DFAState]: ...
+    def dump_nfa(self, name: Text, start: NFAState, finish: NFAState) -> List[DFAState]: ...
+    def dump_dfa(self, name: Text, dfa: Iterable[DFAState]) -> None: ...
+    def simplify_dfa(self, dfa: List[DFAState]) -> None: ...
+    def parse_rhs(self) -> Tuple[NFAState, NFAState]: ...
+    def parse_alt(self) -> Tuple[NFAState, NFAState]: ...
+    def parse_item(self) -> Tuple[NFAState, NFAState]: ...
+    def parse_atom(self) -> Tuple[NFAState, NFAState]: ...
+    def expect(self, type: int, value: Optional[Any] = ...) -> Text: ...
+    def gettoken(self) -> None: ...
+    def raise_error(self, msg: str, *args: Any) -> NoReturn: ...
+
+class NFAState:
+    arcs: List[Tuple[Optional[Text], NFAState]]
+    def __init__(self) -> None: ...
+    def addarc(self, next: NFAState, label: Optional[Text] = ...) -> None: ...
+
+class DFAState:
+    nfaset: Dict[NFAState, Any]
+    isfinal: bool
+    arcs: Dict[Text, DFAState]
+    def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: ...
+    def addarc(self, next: DFAState, label: Text) -> None: ...
+    def unifystate(self, old: DFAState, new: DFAState) -> None: ...
+    def __eq__(self, other: Any) -> bool: ...
+
+def generate_grammar(filename: StrPath = ...) -> PgenGrammar: ...

stdlib/lib2to3/pgen2/token.pyi

@@ -0,0 +1,71 @@
+import sys
+from typing import Dict, Text
+
+ENDMARKER: int
+NAME: int
+NUMBER: int
+STRING: int
+NEWLINE: int
+INDENT: int
+DEDENT: int
+LPAR: int
+RPAR: int
+LSQB: int
+RSQB: int
+COLON: int
+COMMA: int
+SEMI: int
+PLUS: int
+MINUS: int
+STAR: int
+SLASH: int
+VBAR: int
+AMPER: int
+LESS: int
+GREATER: int
+EQUAL: int
+DOT: int
+PERCENT: int
+BACKQUOTE: int
+LBRACE: int
+RBRACE: int
+EQEQUAL: int
+NOTEQUAL: int
+LESSEQUAL: int
+GREATEREQUAL: int
+TILDE: int
+CIRCUMFLEX: int
+LEFTSHIFT: int
+RIGHTSHIFT: int
+DOUBLESTAR: int
+PLUSEQUAL: int
+MINEQUAL: int
+STAREQUAL: int
+SLASHEQUAL: int
+PERCENTEQUAL: int
+AMPEREQUAL: int
+VBAREQUAL: int
+CIRCUMFLEXEQUAL: int
+LEFTSHIFTEQUAL: int
+RIGHTSHIFTEQUAL: int
+DOUBLESTAREQUAL: int
+DOUBLESLASH: int
+DOUBLESLASHEQUAL: int
+OP: int
+COMMENT: int
+NL: int
+if sys.version_info >= (3,):
+    RARROW: int
+if sys.version_info >= (3, 5):
+    AT: int
+    ATEQUAL: int
+    AWAIT: int
+    ASYNC: int
+ERRORTOKEN: int
+N_TOKENS: int
+NT_OFFSET: int
+tok_name: Dict[int, Text]
+
+def ISTERMINAL(x: int) -> bool: ...
+def ISNONTERMINAL(x: int) -> bool: ...
+def ISEOF(x: int) -> bool: ...

stdlib/lib2to3/pgen2/tokenize.pyi

@@ -0,0 +1,23 @@
+from lib2to3.pgen2.token import *  # noqa
+from typing import Callable, Iterable, Iterator, List, Text, Tuple
+
+_Coord = Tuple[int, int]
+_TokenEater = Callable[[int, Text, _Coord, _Coord, Text], None]
+_TokenInfo = Tuple[int, Text, _Coord, _Coord, Text]
+
+class TokenError(Exception): ...
+class StopTokenizing(Exception): ...
+
+def tokenize(readline: Callable[[], Text], tokeneater: _TokenEater = ...) -> None: ...
+
+class Untokenizer:
+    tokens: List[Text]
+    prev_row: int
+    prev_col: int
+    def __init__(self) -> None: ...
+    def add_whitespace(self, start: _Coord) -> None: ...
+    def untokenize(self, iterable: Iterable[_TokenInfo]) -> Text: ...
+    def compat(self, token: Tuple[int, Text], iterable: Iterable[_TokenInfo]) -> None: ...
+
+def untokenize(iterable: Iterable[_TokenInfo]) -> Text: ...
+def generate_tokens(readline: Callable[[], Text]) -> Iterator[_TokenInfo]: ...