diff --git a/parso/pgen2/generator.py b/parso/pgen2/generator.py index 9f36502..3ddf27b 100644 --- a/parso/pgen2/generator.py +++ b/parso/pgen2/generator.py @@ -27,11 +27,14 @@ because we made some optimizations. """ from ast import literal_eval +from typing import TypeVar, Generic, Mapping, Sequence, Set, Union from parso.pgen2.grammar_parser import GrammarParser, NFAState +_TokenTypeT = TypeVar("_TokenTypeT") -class Grammar(object): + +class Grammar(Generic[_TokenTypeT]): """ Once initialized, this class supplies the grammar tables for the parsing engine implemented by parse.py. The parsing engine @@ -41,18 +44,21 @@ class Grammar(object): dfas. """ - def __init__(self, start_nonterminal, rule_to_dfas, reserved_syntax_strings): - self.nonterminal_to_dfas = rule_to_dfas # Dict[str, List[DFAState]] + def __init__(self, + start_nonterminal: str, + rule_to_dfas: Mapping[str, Sequence['DFAState[_TokenTypeT]']], + reserved_syntax_strings: Mapping[str, 'ReservedString']): + self.nonterminal_to_dfas = rule_to_dfas self.reserved_syntax_strings = reserved_syntax_strings self.start_nonterminal = start_nonterminal -class DFAPlan(object): +class DFAPlan: """ Plans are used for the parser to create stack nodes and do the proper DFA state transitions. """ - def __init__(self, next_dfa, dfa_pushes=[]): + def __init__(self, next_dfa: 'DFAState', dfa_pushes: Sequence['DFAState'] = []): self.next_dfa = next_dfa self.dfa_pushes = dfa_pushes @@ -60,7 +66,7 @@ class DFAPlan(object): return '%s(%s, %s)' % (self.__class__.__name__, self.next_dfa, self.dfa_pushes) -class DFAState(object): +class DFAState(Generic[_TokenTypeT]): """ The DFAState object is the core class for pretty much anything. DFAState are the vertices of an ordered graph while arcs and transitions are the @@ -70,20 +76,21 @@ class DFAState(object): transitions are then calculated to connect the DFA state machines that have different nonterminals. """ - def __init__(self, from_rule, nfa_set, final): + def __init__(self, from_rule: str, nfa_set: Set[NFAState], final: NFAState): assert isinstance(nfa_set, set) assert isinstance(next(iter(nfa_set)), NFAState) assert isinstance(final, NFAState) self.from_rule = from_rule self.nfa_set = nfa_set - self.arcs = {} # map from terminals/nonterminals to DFAState + # map from terminals/nonterminals to DFAState + self.arcs: Mapping[str, DFAState] = {} # In an intermediary step we set these nonterminal arcs (which has the # same structure as arcs). These don't contain terminals anymore. - self.nonterminal_arcs = {} + self.nonterminal_arcs: Mapping[str, DFAState] = {} # Transitions are basically the only thing that the parser is using # with is_final. Everyting else is purely here to create a parser. - self.transitions = {} #: Dict[Union[TokenType, ReservedString], DFAPlan] + self.transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan] = {} self.is_final = final in nfa_set def add_arc(self, next_, label): @@ -119,14 +126,14 @@ class DFAState(object): ) -class ReservedString(object): +class ReservedString: """ Most grammars will have certain keywords and operators that are mentioned in the grammar as strings (e.g. "if") and not token types (e.g. NUMBER). This class basically is the former. """ - def __init__(self, value): + def __init__(self, value: str): self.value = value def __repr__(self): @@ -232,7 +239,7 @@ def _dump_dfas(dfas): print(" %s -> %d" % (nonterminal, dfas.index(next_))) -def generate_grammar(bnf_grammar, token_namespace): +def generate_grammar(bnf_grammar: str, token_namespace) -> Grammar: """ ``bnf_text`` is a grammar in extended BNF (using * for repetition, + for at-least-once repetition, [] for optional parts, | for alternatives and () diff --git a/parso/pgen2/generator.pyi b/parso/pgen2/generator.pyi deleted file mode 100644 index 0d67a18..0000000 --- a/parso/pgen2/generator.pyi +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, Generic, Mapping, Sequence, Set, TypeVar, Union - -from parso.pgen2.grammar_parser import NFAState - -_TokenTypeT = TypeVar("_TokenTypeT") - -class Grammar(Generic[_TokenTypeT]): - nonterminal_to_dfas: Mapping[str, Sequence[DFAState[_TokenTypeT]]] - reserved_syntax_strings: Mapping[str, ReservedString] - start_nonterminal: str - def __init__( - self, - start_nonterminal: str, - rule_to_dfas: Mapping[str, Sequence[DFAState]], - reserved_syntax_strings: Mapping[str, ReservedString], - ) -> None: ... - -class DFAPlan: - next_dfa: DFAState - dfa_pushes: Sequence[DFAState] - -class DFAState(Generic[_TokenTypeT]): - from_rule: str - nfa_set: Set[NFAState] - is_final: bool - arcs: Mapping[str, DFAState] # map from all terminals/nonterminals to DFAState - nonterminal_arcs: Mapping[str, DFAState] - transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan] - def __init__( - self, from_rule: str, nfa_set: Set[NFAState], final: NFAState - ) -> None: ... - -class ReservedString: - value: str - def __init__(self, value: str) -> None: ... - def __repr__(self) -> str: ... - -def generate_grammar(bnf_grammar: str, token_namespace: Any) -> Grammar[Any]: ... diff --git a/parso/pgen2/grammar_parser.py b/parso/pgen2/grammar_parser.py index c1bd90a..582efb4 100644 --- a/parso/pgen2/grammar_parser.py +++ b/parso/pgen2/grammar_parser.py @@ -4,17 +4,41 @@ # Modifications: # Copyright David Halter and Contributors # Modifications are dual-licensed: MIT and PSF. +from typing import Optional, Iterator, Tuple, List from parso.python.tokenize import tokenize from parso.utils import parse_version_string from parso.python.token import PythonTokenTypes -class GrammarParser(): +class NFAArc: + def __init__(self, next_: 'NFAState', nonterminal_or_string: Optional[str]): + self.next: NFAState = next_ + self.nonterminal_or_string: Optional[str] = nonterminal_or_string + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.nonterminal_or_string) + + +class NFAState: + def __init__(self, from_rule: str): + self.from_rule: str = from_rule + self.arcs: List[NFAArc] = [] + + def add_arc(self, next_, nonterminal_or_string=None): + assert nonterminal_or_string is None or isinstance(nonterminal_or_string, str) + assert isinstance(next_, NFAState) + self.arcs.append(NFAArc(next_, nonterminal_or_string)) + + def __repr__(self): + return '<%s: from %s>' % (self.__class__.__name__, self.from_rule) + + +class GrammarParser: """ The parser for Python grammar files. """ - def __init__(self, bnf_grammar): + def __init__(self, bnf_grammar: str): self._bnf_grammar = bnf_grammar self.generator = tokenize( bnf_grammar, @@ -22,7 +46,7 @@ class GrammarParser(): ) self._gettoken() # Initialize lookahead - def parse(self): + def parse(self) -> Iterator[Tuple[NFAState, NFAState]]: # grammar: (NEWLINE | rule)* ENDMARKER while self.type != PythonTokenTypes.ENDMARKER: while self.type == PythonTokenTypes.NEWLINE: @@ -134,26 +158,3 @@ class GrammarParser(): line = self._bnf_grammar.splitlines()[self.begin[0] - 1] raise SyntaxError(msg, ('', self.begin[0], self.begin[1], line)) - - -class NFAArc(object): - def __init__(self, next_, nonterminal_or_string): - self.next = next_ - self.nonterminal_or_string = nonterminal_or_string - - def __repr__(self): - return '<%s: %s>' % (self.__class__.__name__, self.nonterminal_or_string) - - -class NFAState(object): - def __init__(self, from_rule): - self.from_rule = from_rule - self.arcs = [] # List[nonterminal (str), NFAState] - - def add_arc(self, next_, nonterminal_or_string=None): - assert nonterminal_or_string is None or isinstance(nonterminal_or_string, str) - assert isinstance(next_, NFAState) - self.arcs.append(NFAArc(next_, nonterminal_or_string)) - - def __repr__(self): - return '<%s: from %s>' % (self.__class__.__name__, self.from_rule) diff --git a/parso/pgen2/grammar_parser.pyi b/parso/pgen2/grammar_parser.pyi deleted file mode 100644 index b73a5a6..0000000 --- a/parso/pgen2/grammar_parser.pyi +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Generator, List, Optional, Tuple - -from parso.python.token import TokenType - -class GrammarParser: - generator: Generator[TokenType, None, None] - def __init__(self, bnf_grammar: str) -> None: ... - def parse(self) -> Generator[Tuple[NFAState, NFAState], None, None]: ... - -class NFAArc: - next: NFAState - nonterminal_or_string: Optional[str] - def __init__( - self, next_: NFAState, nonterminal_or_string: Optional[str] - ) -> None: ... - -class NFAState: - from_rule: str - arcs: List[NFAArc] - def __init__(self, from_rule: str) -> None: ...