VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-24 21:21:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Actually parse some first things with the new approach

Browse Source
This commit is contained in:
Dave Halter
2018-06-21 23:56:34 +02:00
parent 2a082d69df
commit f03a87b876
2 changed files with 35 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
parso/pgen2/grammar.py
View File

@@ -132,8 +132,8 @@ class Grammar(object):
dfa_state.ilabel_to_plan = plans = {}
for terminal_or_nonterminal, next_dfa in dfa_state.arcs.items():
if terminal_or_nonterminal in self.nonterminal2number:
for t, plan in self._first_plans[terminal_or_nonterminal].items():
plans[self._make_label(t)] = plan
for t, pushes in self._first_plans[terminal_or_nonterminal].items():
plans[self._make_label(t)] = DFAPlan(next_dfa, pushes)
else:
ilabel = self._make_label(terminal_or_nonterminal)
plans[ilabel] = DFAPlan(next_dfa)
@@ -230,17 +230,14 @@ class Grammar(object):
totalset.update(fset)
overlapcheck[nonterminal_or_string] = fset
for t, plan in self._first_plans[nonterminal_or_string].items():
for t, pushes in self._first_plans[nonterminal_or_string].items():
assert not self._first_plans[nonterminal].get(t)
self._first_plans[nonterminal][t] = DFAPlan(
plan.next_dfa,
[next_] + plan.dfa_pushes
)
self._first_plans[nonterminal][t] = [next_] + pushes
else:
# It's a string. We have finally found a possible first token.
totalset.add(nonterminal_or_string)
overlapcheck[nonterminal_or_string] = set([nonterminal_or_string])
self._first_plans[nonterminal][nonterminal_or_string] = DFAPlan(next_)
self._first_plans[nonterminal][nonterminal_or_string] = [next_]
inverse = {}
for nonterminal_or_string, first_set in overlapcheck.items():

53
parso/pgen2/parse.py
View File

@@ -173,41 +173,48 @@ class PgenParser(object):
stack = self._stack
grammar = self.grammar
try:
plan = stack[-1].dfa.ilabel_to_plan[ilabel]
except KeyError:
self.error_recovery(grammar, stack, type_,
value, start_pos, prefix, self.add_token)
return False
while True:
try:
plan = stack[-1].dfa.ilabel_to_plan[ilabel]
break
except KeyError:
if stack[-1].dfa.is_final:
tos = stack.pop()
# If there's exactly one child, return that child instead of
# creating a new node. We still create expr_stmt and
# file_input though, because a lot of Jedi depends on its
# logic.
if len(tos.nodes) == 1:
new_node = tos.nodes[0]
else:
# XXX don't use that type
xxx_type = grammar.nonterminal2number[tos.dfa.from_rule]
new_node = self.convert_node(grammar, xxx_type, tos.nodes)
try:
stack[-1].nodes.append(new_node)
except IndexError:
# Stack is empty, set the rootnode.
self.rootnode = new_node
return True
else:
self.error_recovery(grammar, stack, type_,
value, start_pos, prefix, self.add_token)
return False
stack[-1].dfa = plan.next_dfa
for push in plan.dfa_pushes:
stack.append(StackNode(push))
leaf = self.convert_leaf(grammar, type_, value, prefix, start_pos)
stack[-1].nodes.append(leaf)
while stack[-1].dfa.is_final:
tos = stack.pop()
# If there's exactly one child, return that child instead of
# creating a new node. We still create expr_stmt and
# file_input though, because a lot of Jedi depends on its
# logic.
if len(tos.nodes) == 1:
new_node = tos.nodes[0]
else:
new_node = self.convert_node(grammar, type_, tos.nodes)
try:
stack[-1].nodes.append(new_node)
except IndexError:
# Stack is empty, set the rootnode.
self.rootnode = new_node
return True
return False
def add_token(self, type_, value, start_pos, prefix):
"""Add a token; return True if this is the end of the program."""
self._new_add_token(type_, value, start_pos, prefix)
ilabel = token_to_ilabel(self.grammar, type_, value)
# Loop until the token is shifted; may raise exceptions