Removed stubs from generator and grammar_parser and put the annotations into the corresponding files

parso/pgen2/generator.py

@@ -27,11 +27,14 @@ because we made some optimizations.
 """
 
 from ast import literal_eval
+from typing import TypeVar, Generic, Mapping, Sequence, Set, Union
 
 from parso.pgen2.grammar_parser import GrammarParser, NFAState
 
+_TokenTypeT = TypeVar("_TokenTypeT")
 
-class Grammar(object):
+
+class Grammar(Generic[_TokenTypeT]):
     """
     Once initialized, this class supplies the grammar tables for the
     parsing engine implemented by parse.py.  The parsing engine
@@ -41,18 +44,21 @@ class Grammar(object):
     dfas.
     """
 
-    def __init__(self, start_nonterminal, rule_to_dfas, reserved_syntax_strings):
+    def __init__(self,
-        self.nonterminal_to_dfas = rule_to_dfas  # Dict[str, List[DFAState]]
+                 start_nonterminal: str,
+                 rule_to_dfas: Mapping[str, Sequence['DFAState[_TokenTypeT]']],
+                 reserved_syntax_strings: Mapping[str, 'ReservedString']):
+        self.nonterminal_to_dfas = rule_to_dfas
         self.reserved_syntax_strings = reserved_syntax_strings
         self.start_nonterminal = start_nonterminal
 
 
-class DFAPlan(object):
+class DFAPlan:
     """
     Plans are used for the parser to create stack nodes and do the proper
     DFA state transitions.
     """
-    def __init__(self, next_dfa, dfa_pushes=[]):
+    def __init__(self, next_dfa: 'DFAState', dfa_pushes: Sequence['DFAState'] = []):
         self.next_dfa = next_dfa
         self.dfa_pushes = dfa_pushes
 
@@ -60,7 +66,7 @@ class DFAPlan(object):
         return '%s(%s, %s)' % (self.__class__.__name__, self.next_dfa, self.dfa_pushes)
 
 
-class DFAState(object):
+class DFAState(Generic[_TokenTypeT]):
     """
     The DFAState object is the core class for pretty much anything. DFAState
     are the vertices of an ordered graph while arcs and transitions are the
@@ -70,20 +76,21 @@ class DFAState(object):
     transitions are then calculated to connect the DFA state machines that have
     different nonterminals.
     """
-    def __init__(self, from_rule, nfa_set, final):
+    def __init__(self, from_rule: str, nfa_set: Set[NFAState], final: NFAState):
         assert isinstance(nfa_set, set)
         assert isinstance(next(iter(nfa_set)), NFAState)
         assert isinstance(final, NFAState)
         self.from_rule = from_rule
         self.nfa_set = nfa_set
-        self.arcs = {}  # map from terminals/nonterminals to DFAState
+        # map from terminals/nonterminals to DFAState
+        self.arcs: Mapping[str, DFAState] = {}
         # In an intermediary step we set these nonterminal arcs (which has the
         # same structure as arcs). These don't contain terminals anymore.
-        self.nonterminal_arcs = {}
+        self.nonterminal_arcs: Mapping[str, DFAState] = {}
 
         # Transitions are basically the only thing that  the parser is using
         # with is_final. Everyting else is purely here to create a parser.
-        self.transitions = {}  #: Dict[Union[TokenType, ReservedString], DFAPlan]
+        self.transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan] = {}
         self.is_final = final in nfa_set
 
     def add_arc(self, next_, label):
@@ -119,14 +126,14 @@ class DFAState(object):
         )
 
 
-class ReservedString(object):
+class ReservedString:
     """
     Most grammars will have certain keywords and operators that are mentioned
     in the grammar as strings (e.g. "if") and not token types (e.g. NUMBER).
     This class basically is the former.
     """
 
-    def __init__(self, value):
+    def __init__(self, value: str):
         self.value = value
 
     def __repr__(self):
@@ -232,7 +239,7 @@ def _dump_dfas(dfas):
             print("    %s -> %d" % (nonterminal, dfas.index(next_)))
 
 
-def generate_grammar(bnf_grammar, token_namespace):
+def generate_grammar(bnf_grammar: str, token_namespace) -> Grammar:
     """
     ``bnf_text`` is a grammar in extended BNF (using * for repetition, + for
     at-least-once repetition, [] for optional parts, | for alternatives and ()

parso/pgen2/generator.pyi

@@ -1,38 +0,0 @@
-from typing import Any, Generic, Mapping, Sequence, Set, TypeVar, Union
-
-from parso.pgen2.grammar_parser import NFAState
-
-_TokenTypeT = TypeVar("_TokenTypeT")
-
-class Grammar(Generic[_TokenTypeT]):
-    nonterminal_to_dfas: Mapping[str, Sequence[DFAState[_TokenTypeT]]]
-    reserved_syntax_strings: Mapping[str, ReservedString]
-    start_nonterminal: str
-    def __init__(
-        self,
-        start_nonterminal: str,
-        rule_to_dfas: Mapping[str, Sequence[DFAState]],
-        reserved_syntax_strings: Mapping[str, ReservedString],
-    ) -> None: ...
-
-class DFAPlan:
-    next_dfa: DFAState
-    dfa_pushes: Sequence[DFAState]
-
-class DFAState(Generic[_TokenTypeT]):
-    from_rule: str
-    nfa_set: Set[NFAState]
-    is_final: bool
-    arcs: Mapping[str, DFAState]  # map from all terminals/nonterminals to DFAState
-    nonterminal_arcs: Mapping[str, DFAState]
-    transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan]
-    def __init__(
-        self, from_rule: str, nfa_set: Set[NFAState], final: NFAState
-    ) -> None: ...
-
-class ReservedString:
-    value: str
-    def __init__(self, value: str) -> None: ...
-    def __repr__(self) -> str: ...
-
-def generate_grammar(bnf_grammar: str, token_namespace: Any) -> Grammar[Any]: ...

parso/pgen2/grammar_parser.py

@@ -4,17 +4,41 @@
 # Modifications:
 # Copyright David Halter and Contributors
 # Modifications are dual-licensed: MIT and PSF.
+from typing import Optional, Iterator, Tuple, List
 
 from parso.python.tokenize import tokenize
 from parso.utils import parse_version_string
 from parso.python.token import PythonTokenTypes
 
 
-class GrammarParser():
+class NFAArc:
+    def __init__(self, next_: 'NFAState', nonterminal_or_string: Optional[str]):
+        self.next: NFAState = next_
+        self.nonterminal_or_string: Optional[str] = nonterminal_or_string
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.nonterminal_or_string)
+
+
+class NFAState:
+    def __init__(self, from_rule: str):
+        self.from_rule: str = from_rule
+        self.arcs: List[NFAArc] = []
+
+    def add_arc(self, next_, nonterminal_or_string=None):
+        assert nonterminal_or_string is None or isinstance(nonterminal_or_string, str)
+        assert isinstance(next_, NFAState)
+        self.arcs.append(NFAArc(next_, nonterminal_or_string))
+
+    def __repr__(self):
+        return '<%s: from %s>' % (self.__class__.__name__, self.from_rule)
+
+
+class GrammarParser:
     """
     The parser for Python grammar files.
     """
-    def __init__(self, bnf_grammar):
+    def __init__(self, bnf_grammar: str):
         self._bnf_grammar = bnf_grammar
         self.generator = tokenize(
             bnf_grammar,
@@ -22,7 +46,7 @@ class GrammarParser():
         )
         self._gettoken()  # Initialize lookahead
 
-    def parse(self):
+    def parse(self) -> Iterator[Tuple[NFAState, NFAState]]:
         # grammar: (NEWLINE | rule)* ENDMARKER
         while self.type != PythonTokenTypes.ENDMARKER:
             while self.type == PythonTokenTypes.NEWLINE:
@@ -134,26 +158,3 @@ class GrammarParser():
         line = self._bnf_grammar.splitlines()[self.begin[0] - 1]
         raise SyntaxError(msg, ('<grammar>', self.begin[0],
                                 self.begin[1], line))
-
-
-class NFAArc(object):
-    def __init__(self, next_, nonterminal_or_string):
-        self.next = next_
-        self.nonterminal_or_string = nonterminal_or_string
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.nonterminal_or_string)
-
-
-class NFAState(object):
-    def __init__(self, from_rule):
-        self.from_rule = from_rule
-        self.arcs = []  # List[nonterminal (str), NFAState]
-
-    def add_arc(self, next_, nonterminal_or_string=None):
-        assert nonterminal_or_string is None or isinstance(nonterminal_or_string, str)
-        assert isinstance(next_, NFAState)
-        self.arcs.append(NFAArc(next_, nonterminal_or_string))
-
-    def __repr__(self):
-        return '<%s: from %s>' % (self.__class__.__name__, self.from_rule)

parso/pgen2/grammar_parser.pyi

@@ -1,20 +0,0 @@
-from typing import Generator, List, Optional, Tuple
-
-from parso.python.token import TokenType
-
-class GrammarParser:
-    generator: Generator[TokenType, None, None]
-    def __init__(self, bnf_grammar: str) -> None: ...
-    def parse(self) -> Generator[Tuple[NFAState, NFAState], None, None]: ...
-
-class NFAArc:
-    next: NFAState
-    nonterminal_or_string: Optional[str]
-    def __init__(
-        self, next_: NFAState, nonterminal_or_string: Optional[str]
-    ) -> None: ...
-
-class NFAState:
-    from_rule: str
-    arcs: List[NFAArc]
-    def __init__(self, from_rule: str) -> None: ...