Diff parser: Add error dedents if necessary, see also davidhalter/jedi#1499

parso/python/diff.py

@@ -12,7 +12,7 @@ import logging
 
 from parso.utils import split_lines
 from parso.python.parser import Parser
-from parso.python.tree import EndMarker
+from parso.python.tree import EndMarker, PythonErrorLeaf
 from parso.python.tokenize import PythonToken
 from parso.python.token import PythonTokenTypes
 
@@ -427,7 +427,9 @@ class DiffParser(object):
 
 
 class _NodesTreeNode(object):
-    _ChildrenGroup = namedtuple('_ChildrenGroup', 'prefix children line_offset last_line_offset_leaf')
+    _ChildrenGroup = namedtuple(
+        '_ChildrenGroup',
+        'prefix children line_offset last_line_offset_leaf add_error_dedent')
 
     def __init__(self, tree_node, parent=None):
         self.tree_node = tree_node
@@ -437,7 +439,8 @@ class _NodesTreeNode(object):
 
     def finish(self):
         children = []
-        for prefix, children_part, line_offset, last_line_offset_leaf in self._children_groups:
+        for prefix, children_part, line_offset, last_line_offset_leaf, add_error_dedent \
+                in self._children_groups:
             first_leaf = _get_next_leaf_if_indentation(
                 children_part[0].get_first_leaf()
             )
@@ -449,6 +452,8 @@ class _NodesTreeNode(object):
                         children_part, line_offset, last_line_offset_leaf)
                 except _PositionUpdatingFinished:
                     pass
+            if add_error_dedent:
+                children.append(PythonErrorLeaf('ERROR_DEDENT', '', children[-1].end_pos))
             children += children_part
         self.tree_node.children = children
         # Reset the parents
@@ -461,10 +466,13 @@ class _NodesTreeNode(object):
     def add_child_node(self, child_node):
         self._node_children.append(child_node)
 
-    def add_tree_nodes(self, prefix, children, line_offset=0, last_line_offset_leaf=None):
+    def add_tree_nodes(self, prefix, children, line_offset=0,
+                       last_line_offset_leaf=None, add_error_dedent=False):
         if last_line_offset_leaf is None:
             last_line_offset_leaf = children[-1].get_last_leaf()
-        group = self._ChildrenGroup(prefix, children, line_offset, last_line_offset_leaf)
+        group = self._ChildrenGroup(
+            prefix, children, line_offset, last_line_offset_leaf, add_error_dedent
+        )
         self._children_groups.append(group)
 
     def get_last_line(self, suffix):
@@ -506,6 +514,7 @@ class _NodesTree(object):
 
     def _get_insertion_node(self, indentation_node):
         indentation = indentation_node.start_pos[1]
+        need_error_dedent = False
 
         previous_node = None
         # find insertion node
@@ -518,6 +527,7 @@ class _NodesTree(object):
                 if indentation > node_indentation and previous_node is not None:
                     # This means that it was not dedented enough.
                     node = previous_node
+                    need_error_dedent = True
                     break
                 if indentation >= node_indentation:  # Not a Dedent
                     # We might be at the most outer layer: modules. We
@@ -527,12 +537,13 @@ class _NodesTree(object):
             elif tree_node.type == 'file_input':
                 if indentation > 0 and previous_node is not None:
                     node = previous_node
+                    need_error_dedent = True
                 break
             previous_node = node
 
         while True:
             if node == self._working_stack[-1]:
-                return node
+                return need_error_dedent, node
             self._working_stack.pop()
 
     def add_parsed_nodes(self, tree_nodes):
@@ -544,9 +555,9 @@ class _NodesTree(object):
 
         assert tree_nodes[0].type != 'newline'
 
-        node = self._get_insertion_node(tree_nodes[0])
+        need_error_dedent, node = self._get_insertion_node(tree_nodes[0])
         assert node.tree_node.type in ('suite', 'file_input')
-        node.add_tree_nodes(old_prefix, tree_nodes)
+        node.add_tree_nodes(old_prefix, tree_nodes, add_error_dedent=need_error_dedent)
         # tos = Top of stack
         self._update_tos(tree_nodes[-1])
 

test/test_diff_parser.py

@@ -87,7 +87,7 @@ class Differ(object):
         _assert_valid_graph(new_module)
 
         without_diff_parser_module = parse(code)
-        #_assert_modules_are_equal(new_module, without_diff_parser_module)
+        _assert_modules_are_equal(new_module, without_diff_parser_module)
 
         error_node = _check_error_leaves_nodes(new_module)
         assert expect_error_leaves == (error_node is not None), error_node