forked from VimPlug/jedi
pgen2: Don't overwrite type
This commit is contained in:
+18
-18
@@ -104,29 +104,29 @@ class PgenParser(object):
|
|||||||
self.error_recovery = error_recovery
|
self.error_recovery = error_recovery
|
||||||
|
|
||||||
def parse(self, tokenizer):
|
def parse(self, tokenizer):
|
||||||
for type, value, prefix, start_pos in tokenizer:
|
for type_, value, prefix, start_pos in tokenizer:
|
||||||
if self.addtoken(type, value, prefix, start_pos):
|
if self.addtoken(type_, value, prefix, start_pos):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# We never broke out -- EOF is too soon -- Unfinished statement.
|
# We never broke out -- EOF is too soon -- Unfinished statement.
|
||||||
self.error_recovery(self.grammar, self.stack, type, value,
|
self.error_recovery(self.grammar, self.stack, type_, value,
|
||||||
start_pos, prefix, self.addtoken)
|
start_pos, prefix, self.addtoken)
|
||||||
# Add the ENDMARKER again.
|
# Add the ENDMARKER again.
|
||||||
if not self.addtoken(type, value, prefix, start_pos):
|
if not self.addtoken(type_, value, prefix, start_pos):
|
||||||
raise InternalParseError("incomplete input", type, value, start_pos)
|
raise InternalParseError("incomplete input", type_, value, start_pos)
|
||||||
return self.rootnode
|
return self.rootnode
|
||||||
|
|
||||||
def addtoken(self, type, value, prefix, start_pos):
|
def addtoken(self, type_, value, prefix, start_pos):
|
||||||
"""Add a token; return True if this is the end of the program."""
|
"""Add a token; return True if this is the end of the program."""
|
||||||
# Map from token to label
|
# Map from token to label
|
||||||
if type == tokenize.NAME:
|
if type_ == tokenize.NAME:
|
||||||
# Check for reserved words (keywords)
|
# Check for reserved words (keywords)
|
||||||
try:
|
try:
|
||||||
ilabel = self.grammar.keywords[value]
|
ilabel = self.grammar.keywords[value]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
ilabel = self.grammar.tokens[type]
|
ilabel = self.grammar.tokens[type_]
|
||||||
else:
|
else:
|
||||||
ilabel = self.grammar.tokens[type]
|
ilabel = self.grammar.tokens[type_]
|
||||||
|
|
||||||
# Loop until the token is shifted; may raise exceptions
|
# Loop until the token is shifted; may raise exceptions
|
||||||
while True:
|
while True:
|
||||||
@@ -140,7 +140,7 @@ class PgenParser(object):
|
|||||||
# Look it up in the list of labels
|
# Look it up in the list of labels
|
||||||
assert t < 256
|
assert t < 256
|
||||||
# Shift a token; we're done with it
|
# Shift a token; we're done with it
|
||||||
self.shift(type, value, newstate, prefix, start_pos)
|
self.shift(type_, value, newstate, prefix, start_pos)
|
||||||
# Pop while we are in an accept-only state
|
# Pop while we are in an accept-only state
|
||||||
state = newstate
|
state = newstate
|
||||||
while states[state] == [(0, state)]:
|
while states[state] == [(0, state)]:
|
||||||
@@ -166,36 +166,36 @@ class PgenParser(object):
|
|||||||
self.pop()
|
self.pop()
|
||||||
if not self.stack:
|
if not self.stack:
|
||||||
# Done parsing, but another token is input
|
# Done parsing, but another token is input
|
||||||
raise InternalParseError("too much input", type, value, start_pos)
|
raise InternalParseError("too much input", type_, value, start_pos)
|
||||||
else:
|
else:
|
||||||
self.error_recovery(self.grammar, self.stack, type,
|
self.error_recovery(self.grammar, self.stack, type_,
|
||||||
value, start_pos, prefix, self.addtoken)
|
value, start_pos, prefix, self.addtoken)
|
||||||
break
|
break
|
||||||
|
|
||||||
def shift(self, type, value, newstate, prefix, start_pos):
|
def shift(self, type_, value, newstate, prefix, start_pos):
|
||||||
"""Shift a token. (Internal)"""
|
"""Shift a token. (Internal)"""
|
||||||
dfa, state, node = self.stack[-1]
|
dfa, state, node = self.stack[-1]
|
||||||
newnode = self.convert_leaf(self.grammar, type, value, prefix, start_pos)
|
newnode = self.convert_leaf(self.grammar, type_, value, prefix, start_pos)
|
||||||
node[-1].append(newnode)
|
node[-1].append(newnode)
|
||||||
self.stack[-1] = (dfa, newstate, node)
|
self.stack[-1] = (dfa, newstate, node)
|
||||||
|
|
||||||
def push(self, type, newdfa, newstate):
|
def push(self, type_, newdfa, newstate):
|
||||||
"""Push a nonterminal. (Internal)"""
|
"""Push a nonterminal. (Internal)"""
|
||||||
dfa, state, node = self.stack[-1]
|
dfa, state, node = self.stack[-1]
|
||||||
newnode = (type, [])
|
newnode = (type_, [])
|
||||||
self.stack[-1] = (dfa, newstate, node)
|
self.stack[-1] = (dfa, newstate, node)
|
||||||
self.stack.append((newdfa, 0, newnode))
|
self.stack.append((newdfa, 0, newnode))
|
||||||
|
|
||||||
def pop(self):
|
def pop(self):
|
||||||
"""Pop a nonterminal. (Internal)"""
|
"""Pop a nonterminal. (Internal)"""
|
||||||
popdfa, popstate, (type, children) = self.stack.pop()
|
popdfa, popstate, (type_, children) = self.stack.pop()
|
||||||
# If there's exactly one child, return that child instead of creating a
|
# If there's exactly one child, return that child instead of creating a
|
||||||
# new node. We still create expr_stmt and file_input though, because a
|
# new node. We still create expr_stmt and file_input though, because a
|
||||||
# lot of Jedi depends on its logic.
|
# lot of Jedi depends on its logic.
|
||||||
if len(children) == 1:
|
if len(children) == 1:
|
||||||
newnode = children[0]
|
newnode = children[0]
|
||||||
else:
|
else:
|
||||||
newnode = self.convert_node(self.grammar, type, children)
|
newnode = self.convert_node(self.grammar, type_, children)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Equal to:
|
# Equal to:
|
||||||
|
|||||||
Reference in New Issue
Block a user