VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-12 23:51:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use more nonterminal/terminal terminology

Browse Source
This commit is contained in:
Dave Halter
2018-06-17 17:37:15 +02:00
parent 23362ec2d3
commit 1f27fa9320
1 changed files with 23 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
parso/pgen2/pgen.py
View File

@@ -26,12 +26,12 @@ from parso.pgen2.grammar_parser import GrammarParser, NFAState
class ParserGenerator(object): class ParserGenerator(object):
def __init__(self, rule_to_dfas, token_namespace): def __init__(self, rule_to_dfas, token_namespace):
self._token_namespace = token_namespace self._token_namespace = token_namespace
self._rule_to_dfas = rule_to_dfas self._nonterminal_to_dfas = rule_to_dfas
def make_grammar(self, grammar): def make_grammar(self, grammar):
self._first = {} # map from symbol name to set of tokens self._first = {} # map from symbol name to set of tokens
names = list(self._rule_to_dfas.keys()) names = list(self._nonterminal_to_dfas.keys())
names.sort() names.sort()
for name in names: for name in names:
if name not in self._first: if name not in self._first:
@@ -43,7 +43,7 @@ class ParserGenerator(object):
grammar.number2symbol[i] = name grammar.number2symbol[i] = name
for name in names: for name in names:
dfas = self._rule_to_dfas[name] dfas = self._nonterminal_to_dfas[name]
states = [] states = []
for state in dfas: for state in dfas:
arcs = [] arcs = []
@@ -112,33 +112,37 @@ class ParserGenerator(object):
return ilabel return ilabel
def _calcfirst(self, name): def _calcfirst(self, name):
dfa = self._rule_to_dfas[name] dfas = self._nonterminal_to_dfas[name]
self._first[name] = None # dummy to detect left recursion self._first[name] = None # dummy to detect left recursion
state = dfa[0] state = dfas[0]
totalset = {} totalset = {}
overlapcheck = {} overlapcheck = {}
for label, next in state.arcs.items(): for nonterminal_or_string, next in state.arcs.items():
if label in self._rule_to_dfas: if nonterminal_or_string in self._nonterminal_to_dfas:
if label in self._first: # It's a nonterminal and we have either a left recursion issue
fset = self._first[label] # in the grammare or we have to recurse.
try:
fset = self._first[nonterminal_or_string]
except KeyError:
self._calcfirst(nonterminal_or_string)
fset = self._first[nonterminal_or_string]
else:
if fset is None: if fset is None:
raise ValueError("left recursion for rule %r" % name) raise ValueError("left recursion for rule %r" % name)
else:
self._calcfirst(label)
fset = self._first[label]
totalset.update(fset) totalset.update(fset)
overlapcheck[label] = fset overlapcheck[nonterminal_or_string] = fset
else: else:
totalset[label] = 1 # It's a string. We have finally found a possible first token.
overlapcheck[label] = {label: 1} totalset[nonterminal_or_string] = 1
overlapcheck[nonterminal_or_string] = {nonterminal_or_string: 1}
inverse = {} inverse = {}
for label, itsfirst in overlapcheck.items(): for nonterminal_or_string, itsfirst in overlapcheck.items():
for symbol in itsfirst: for symbol in itsfirst:
if symbol in inverse: if symbol in inverse:
raise ValueError("rule %s is ambiguous; %s is in the" raise ValueError("rule %s is ambiguous; %s is in the"
" first sets of %s as well as %s" % " first sets of %s as well as %s" %
(name, symbol, label, inverse[symbol])) (name, symbol, nonterminal_or_string, inverse[symbol]))
inverse[symbol] = label inverse[symbol] = nonterminal_or_string
self._first[name] = totalset self._first[name] = totalset
@@ -150,7 +154,7 @@ class DFAState(object):
self.from_rule = from_rule self.from_rule = from_rule
self.nfa_set = nfa_set self.nfa_set = nfa_set
self.isfinal = final in nfa_set self.isfinal = final in nfa_set
self.arcs = {} # map from label to DFAState self.arcs = {} # map from nonterminals or strings to DFAState
def add_arc(self, next_, label): def add_arc(self, next_, label):
assert isinstance(label, str) assert isinstance(label, str)