From f03a87b876937e2027b6554d36ce84ee2944a672 Mon Sep 17 00:00:00 2001
From: Dave Halter <davidhalter88@gmail.com>
Date: Thu, 21 Jun 2018 23:56:34 +0200
Subject: [PATCH] Actually parse some first things with the new approach

---
 parso/pgen2/grammar.py | 13 ++++-------
 parso/pgen2/parse.py   | 53 ++++++++++++++++++++++++------------------
 2 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/parso/pgen2/grammar.py b/parso/pgen2/grammar.py
index 208912d..988cdd8 100644
--- a/parso/pgen2/grammar.py
+++ b/parso/pgen2/grammar.py
@@ -132,8 +132,8 @@ class Grammar(object):
                 dfa_state.ilabel_to_plan = plans = {}
                 for terminal_or_nonterminal, next_dfa in dfa_state.arcs.items():
                     if terminal_or_nonterminal in self.nonterminal2number:
-                        for t, plan in self._first_plans[terminal_or_nonterminal].items():
-                            plans[self._make_label(t)] = plan
+                        for t, pushes in self._first_plans[terminal_or_nonterminal].items():
+                            plans[self._make_label(t)] = DFAPlan(next_dfa, pushes)
                     else:
                         ilabel = self._make_label(terminal_or_nonterminal)
                         plans[ilabel] = DFAPlan(next_dfa)
@@ -230,17 +230,14 @@ class Grammar(object):
                 totalset.update(fset)
                 overlapcheck[nonterminal_or_string] = fset
 
-                for t, plan in self._first_plans[nonterminal_or_string].items():
+                for t, pushes in self._first_plans[nonterminal_or_string].items():
                     assert not self._first_plans[nonterminal].get(t)
-                    self._first_plans[nonterminal][t] = DFAPlan(
-                        plan.next_dfa,
-                        [next_] + plan.dfa_pushes
-                    )
+                    self._first_plans[nonterminal][t] = [next_] + pushes
             else:
                 # It's a string. We have finally found a possible first token.
                 totalset.add(nonterminal_or_string)
                 overlapcheck[nonterminal_or_string] = set([nonterminal_or_string])
-                self._first_plans[nonterminal][nonterminal_or_string] = DFAPlan(next_)
+                self._first_plans[nonterminal][nonterminal_or_string] = [next_]
 
         inverse = {}
         for nonterminal_or_string, first_set in overlapcheck.items():
diff --git a/parso/pgen2/parse.py b/parso/pgen2/parse.py
index 473bc6b..2c5e753 100644
--- a/parso/pgen2/parse.py
+++ b/parso/pgen2/parse.py
@@ -173,41 +173,48 @@ class PgenParser(object):
         stack = self._stack
         grammar = self.grammar
 
-        try:
-            plan = stack[-1].dfa.ilabel_to_plan[ilabel]
-        except KeyError:
-            self.error_recovery(grammar, stack, type_,
-                                value, start_pos, prefix, self.add_token)
-            return False
+        while True:
+            try:
+                plan = stack[-1].dfa.ilabel_to_plan[ilabel]
+                break
+            except KeyError:
+                if stack[-1].dfa.is_final:
+                    tos = stack.pop()
+                    # If there's exactly one child, return that child instead of
+                    # creating a new node.  We still create expr_stmt and
+                    # file_input though, because a lot of Jedi depends on its
+                    # logic.
+                    if len(tos.nodes) == 1:
+                        new_node = tos.nodes[0]
+                    else:
+                        # XXX don't use that type
+                        xxx_type = grammar.nonterminal2number[tos.dfa.from_rule]
+                        new_node = self.convert_node(grammar, xxx_type, tos.nodes)
+
+                    try:
+                        stack[-1].nodes.append(new_node)
+                    except IndexError:
+                        # Stack is empty, set the rootnode.
+                        self.rootnode = new_node
+                        return True
+                else:
+                    self.error_recovery(grammar, stack, type_,
+                                        value, start_pos, prefix, self.add_token)
+                    return False
 
         stack[-1].dfa = plan.next_dfa
+
         for push in plan.dfa_pushes:
             stack.append(StackNode(push))
 
         leaf = self.convert_leaf(grammar, type_, value, prefix, start_pos)
         stack[-1].nodes.append(leaf)
 
-        while stack[-1].dfa.is_final:
-            tos = stack.pop()
-            # If there's exactly one child, return that child instead of
-            # creating a new node.  We still create expr_stmt and
-            # file_input though, because a lot of Jedi depends on its
-            # logic.
-            if len(tos.nodes) == 1:
-                new_node = tos.nodes[0]
-            else:
-                new_node = self.convert_node(grammar, type_, tos.nodes)
-
-            try:
-                stack[-1].nodes.append(new_node)
-            except IndexError:
-                # Stack is empty, set the rootnode.
-                self.rootnode = new_node
-                return True
         return False
 
     def add_token(self, type_, value, start_pos, prefix):
         """Add a token; return True if this is the end of the program."""
+        self._new_add_token(type_, value, start_pos, prefix)
         ilabel = token_to_ilabel(self.grammar, type_, value)
 
         # Loop until the token is shifted; may raise exceptions