VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-08-12 10:54:59 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make some methods private when creating grammars.

Browse Source
This commit is contained in:
Dave Halter
2017-05-19 16:39:14 -04:00
parent 88f8e172d5
commit 3357613ac2
1 changed files with 52 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parso/pgen2/pgen.py
+52 -52
View File
@@ -14,10 +14,10 @@ class ParserGenerator(object):
def __init__(self, bnf_text): def __init__(self, bnf_text):
self._bnf_text = bnf_text self._bnf_text = bnf_text
self.generator = tokenize.source_tokens(bnf_text) self.generator = tokenize.source_tokens(bnf_text)
self.gettoken() # Initialize lookahead self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self.parse() self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens self.first = {} # map from symbol name to set of tokens
self.addfirstsets() self._addfirstsets()
def make_grammar(self): def make_grammar(self):
c = grammar.Grammar(self._bnf_text) c = grammar.Grammar(self._bnf_text)
@@ -35,25 +35,25 @@ class ParserGenerator(object):
for state in dfa: for state in dfa:
arcs = [] arcs = []
for label, next in state.arcs.items(): for label, next in state.arcs.items():
arcs.append((self.make_label(c, label), dfa.index(next))) arcs.append((self._make_label(c, label), dfa.index(next)))
if state.isfinal: if state.isfinal:
arcs.append((0, dfa.index(state))) arcs.append((0, dfa.index(state)))
states.append(arcs) states.append(arcs)
c.states.append(states) c.states.append(states)
c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) c.dfas[c.symbol2number[name]] = (states, self._make_first(c, name))
c.start = c.symbol2number[self.startsymbol] c.start = c.symbol2number[self.startsymbol]
return c return c
def make_first(self, c, name): def _make_first(self, c, name):
rawfirst = self.first[name] rawfirst = self.first[name]
first = {} first = {}
for label in rawfirst: for label in rawfirst:
ilabel = self.make_label(c, label) ilabel = self._make_label(c, label)
##assert ilabel not in first # XXX failed on <> ... != ##assert ilabel not in first # XXX failed on <> ... !=
first[ilabel] = 1 first[ilabel] = 1
return first return first
def make_label(self, c, label): def _make_label(self, c, label):
# XXX Maybe this should be a method on a subclass of converter? # XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels) ilabel = len(c.labels)
if label[0].isalpha(): if label[0].isalpha():
@@ -99,15 +99,15 @@ class ParserGenerator(object):
c.tokens[itoken] = ilabel c.tokens[itoken] = ilabel
return ilabel return ilabel
def addfirstsets(self): def _addfirstsets(self):
names = list(self.dfas.keys()) names = list(self.dfas.keys())
names.sort() names.sort()
for name in names: for name in names:
if name not in self.first: if name not in self.first:
self.calcfirst(name) self._calcfirst(name)
#print name, self.first[name].keys() #print name, self.first[name].keys()
def calcfirst(self, name): def _calcfirst(self, name):
dfa = self.dfas[name] dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion self.first[name] = None # dummy to detect left recursion
state = dfa[0] state = dfa[0]
@@ -120,7 +120,7 @@ class ParserGenerator(object):
if fset is None: if fset is None:
raise ValueError("recursion for rule %r" % name) raise ValueError("recursion for rule %r" % name)
else: else:
self.calcfirst(label) self._calcfirst(label)
fset = self.first[label] fset = self.first[label]
totalset.update(fset) totalset.update(fset)
overlapcheck[label] = fset overlapcheck[label] = fset
@@ -137,23 +137,23 @@ class ParserGenerator(object):
inverse[symbol] = label inverse[symbol] = label
self.first[name] = totalset self.first[name] = totalset
def parse(self): def _parse(self):
dfas = {} dfas = {}
startsymbol = None startsymbol = None
# MSTART: (NEWLINE | RULE)* ENDMARKER # MSTART: (NEWLINE | RULE)* ENDMARKER
while self.type != token.ENDMARKER: while self.type != token.ENDMARKER:
while self.type == token.NEWLINE: while self.type == token.NEWLINE:
self.gettoken() self._gettoken()
# RULE: NAME ':' RHS NEWLINE # RULE: NAME ':' RHS NEWLINE
name = self.expect(token.NAME) name = self._expect(token.NAME)
self.expect(token.OP, ":") self._expect(token.OP, ":")
a, z = self.parse_rhs() a, z = self._parse_rhs()
self.expect(token.NEWLINE) self._expect(token.NEWLINE)
#self.dump_nfa(name, a, z) #self._dump_nfa(name, a, z)
dfa = self.make_dfa(a, z) dfa = self._make_dfa(a, z)
#self.dump_dfa(name, dfa) #self._dump_dfa(name, dfa)
# oldlen = len(dfa) # oldlen = len(dfa)
self.simplify_dfa(dfa) self._simplify_dfa(dfa)
# newlen = len(dfa) # newlen = len(dfa)
dfas[name] = dfa dfas[name] = dfa
#print name, oldlen, newlen #print name, oldlen, newlen
@@ -161,7 +161,7 @@ class ParserGenerator(object):
startsymbol = name startsymbol = name
return dfas, startsymbol return dfas, startsymbol
def make_dfa(self, start, finish): def _make_dfa(self, start, finish):
# To turn an NFA into a DFA, we define the states of the DFA # To turn an NFA into a DFA, we define the states of the DFA
# to correspond to *sets* of states of the NFA. Then do some # to correspond to *sets* of states of the NFA. Then do some
# state reduction. Let's represent sets as dicts with 1 for # state reduction. Let's represent sets as dicts with 1 for
@@ -200,7 +200,7 @@ class ParserGenerator(object):
state.addarc(st, label) state.addarc(st, label)
return states # List of DFAState instances; first one is start return states # List of DFAState instances; first one is start
def dump_nfa(self, name, start, finish): def _dump_nfa(self, name, start, finish):
print("Dump of NFA for", name) print("Dump of NFA for", name)
todo = [start] todo = [start]
for i, state in enumerate(todo): for i, state in enumerate(todo):
@@ -216,14 +216,14 @@ class ParserGenerator(object):
else: else:
print(" %s -> %d" % (label, j)) print(" %s -> %d" % (label, j))
def dump_dfa(self, name, dfa): def _dump_dfa(self, name, dfa):
print("Dump of DFA for", name) print("Dump of DFA for", name)
for i, state in enumerate(dfa): for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "") print(" State", i, state.isfinal and "(final)" or "")
for label, next in state.arcs.items(): for label, next in state.arcs.items():
print(" %s -> %d" % (label, dfa.index(next))) print(" %s -> %d" % (label, dfa.index(next)))
def simplify_dfa(self, dfa): def _simplify_dfa(self, dfa):
# This is not theoretically optimal, but works well enough. # This is not theoretically optimal, but works well enough.
# Algorithm: repeatedly look for two states that have the same # Algorithm: repeatedly look for two states that have the same
# set of arcs (same labels pointing to the same nodes) and # set of arcs (same labels pointing to the same nodes) and
@@ -244,9 +244,9 @@ class ParserGenerator(object):
changes = True changes = True
break break
def parse_rhs(self): def _parse_rhs(self):
# RHS: ALT ('|' ALT)* # RHS: ALT ('|' ALT)*
a, z = self.parse_alt() a, z = self._parse_alt()
if self.value != "|": if self.value != "|":
return a, z return a, z
else: else:
@@ -255,75 +255,75 @@ class ParserGenerator(object):
aa.addarc(a) aa.addarc(a)
z.addarc(zz) z.addarc(zz)
while self.value == "|": while self.value == "|":
self.gettoken() self._gettoken()
a, z = self.parse_alt() a, z = self._parse_alt()
aa.addarc(a) aa.addarc(a)
z.addarc(zz) z.addarc(zz)
return aa, zz return aa, zz
def parse_alt(self): def _parse_alt(self):
# ALT: ITEM+ # ALT: ITEM+
a, b = self.parse_item() a, b = self._parse_item()
while (self.value in ("(", "[") or while (self.value in ("(", "[") or
self.type in (token.NAME, token.STRING)): self.type in (token.NAME, token.STRING)):
c, d = self.parse_item() c, d = self._parse_item()
b.addarc(c) b.addarc(c)
b = d b = d
return a, b return a, b
def parse_item(self): def _parse_item(self):
# ITEM: '[' RHS ']' | ATOM ['+' | '*'] # ITEM: '[' RHS ']' | ATOM ['+' | '*']
if self.value == "[": if self.value == "[":
self.gettoken() self._gettoken()
a, z = self.parse_rhs() a, z = self._parse_rhs()
self.expect(token.OP, "]") self._expect(token.OP, "]")
a.addarc(z) a.addarc(z)
return a, z return a, z
else: else:
a, z = self.parse_atom() a, z = self._parse_atom()
value = self.value value = self.value
if value not in ("+", "*"): if value not in ("+", "*"):
return a, z return a, z
self.gettoken() self._gettoken()
z.addarc(a) z.addarc(a)
if value == "+": if value == "+":
return a, z return a, z
else: else:
return a, a return a, a
def parse_atom(self): def _parse_atom(self):
# ATOM: '(' RHS ')' | NAME | STRING # ATOM: '(' RHS ')' | NAME | STRING
if self.value == "(": if self.value == "(":
self.gettoken() self._gettoken()
a, z = self.parse_rhs() a, z = self._parse_rhs()
self.expect(token.OP, ")") self._expect(token.OP, ")")
return a, z return a, z
elif self.type in (token.NAME, token.STRING): elif self.type in (token.NAME, token.STRING):
a = NFAState() a = NFAState()
z = NFAState() z = NFAState()
a.addarc(z, self.value) a.addarc(z, self.value)
self.gettoken() self._gettoken()
return a, z return a, z
else: else:
self.raise_error("expected (...) or NAME or STRING, got %s/%s", self._raise_error("expected (...) or NAME or STRING, got %s/%s",
self.type, self.value) self.type, self.value)
def expect(self, type, value=None): def _expect(self, type, value=None):
if self.type != type or (value is not None and self.value != value): if self.type != type or (value is not None and self.value != value):
self.raise_error("expected %s/%s, got %s/%s", self._raise_error("expected %s/%s, got %s/%s",
type, value, self.type, self.value) type, value, self.type, self.value)
value = self.value value = self.value
self.gettoken() self._gettoken()
return value return value
def gettoken(self): def _gettoken(self):
tup = next(self.generator) tup = next(self.generator)
while tup[0] in (token.COMMENT, token.NL): while tup[0] in (token.COMMENT, token.NL):
tup = next(self.generator) tup = next(self.generator)
self.type, self.value, self.begin, prefix = tup self.type, self.value, self.begin, prefix = tup
#print tokenize.tok_name[self.type], repr(self.value) #print tokenize.tok_name[self.type], repr(self.value)
def raise_error(self, msg, *args): def _raise_error(self, msg, *args):
if args: if args:
try: try:
msg = msg % args msg = msg % args