Fix an issue with parser endings and therefore adapt a few tests.

jedi/parser/fast.py

@@ -130,7 +130,7 @@ class DiffParser(object):
 
         line_length = len(lines_new)
         lines_old = splitlines(self._parser.source, keepends=True)
-        sm = difflib.SequenceMatcher(None, lines_old, lines_new)
+        sm = difflib.SequenceMatcher(None, lines_old, self._parser_lines_new)
         print(len(lines_old), line_length, lines_old, lines_new)
         for operation, i1, i2, j1, j2 in sm.get_opcodes():
             debug.dbg('diff %s old[%s:%s] new[%s:%s]',
@@ -212,13 +212,13 @@ class DiffParser(object):
                     print('COPY', until_line_new, nodes)
                     self._copy_count += 1
                     debug.dbg(
-                        'diff actuall copy %s to %s',
+                        'diff actually copy %s to %s',
                         nodes[0].start_pos[0],
-                        nodes[-1].end_pos[0]
+                        nodes[-1].end_pos
                     )
+                    self._update_positions(nodes, line_offset)
                     parent = self._insert_nodes(nodes)
                     self._update_names_dict(parent, nodes)
-                    self._update_positions(nodes, line_offset)
                 # We have copied as much as possible (but definitely not too
                 # much). Therefore we just parse the rest.
                 # We might not reach the end, because there's a statement
@@ -267,15 +267,13 @@ class DiffParser(object):
 
         last_leaf = nodes[-1].last_leaf()
         is_endmarker = last_leaf.type == self.endmarker_type
-        last_non_endmarker = last_leaf
         if is_endmarker:
             self._parsed_until_line = last_leaf.start_pos[0]
             if last_leaf.prefix.endswith('\n') or \
                     not last_leaf.prefix and last_leaf.get_previous_leaf().type == 'newline':
                 self._parsed_until_line -= 1
         else:
-
-            if last_non_endmarker.type == 'newline':
+            if last_leaf.type == 'newline':
                 # Newlines end on the next line, which means that they would cover
                 # the next line. That line is not fully parsed at this point.
                 self._parsed_until_line = last_leaf.start_pos[0]
@@ -425,9 +423,6 @@ class DiffParser(object):
             nodes = self._get_children_nodes(node)
             parent = self._insert_nodes(nodes)
             self._merge_parsed_node(parent, node)
-            #if until_line - 1 == len(self._parser_lines_new):
-                # We are done in any case. This is special case, because the l
-                #return
 
     def _get_children_nodes(self, node):
         nodes = node.children

test/test_parser/test_diff_parser.py

@@ -39,15 +39,17 @@ def test_add_to_end():
     assert jedi.Script(a + b, path='example.py').completions()
 
 
-def _check_error_leafs(node):
+def _check_error_leaves_nodes(node):
+    if node.type in ('error_leaf', 'error_node'):
+        return True
+
     try:
         children = node.children
     except AttributeError:
-        if node.type == 'error_leaf':
-            return True
+        pass
     else:
         for child in children:
-            if _check_error_leafs(child):
+            if _check_error_leaves_nodes(child):
                 return True
     return False
 
@@ -73,7 +75,7 @@ class Differ(object):
         self.parser.module = new_module
         self.parser._parsed = new_module
         if not allow_error_leafs:
-            assert not _check_error_leafs(new_module)
+            assert not _check_error_leaves_nodes(new_module)
         return new_module
 
 
@@ -92,15 +94,21 @@ def test_change_and_undo(differ):
     differ.parse(func_before + 'b', copies=1, parsers=1)
     # b has changed to a again, so parse that.
     differ.parse(func_before + 'a', copies=1, parsers=1)
-    # Same as before no parsers should be used.
-    differ.parse(func_before + 'a', copies=1)
+    # Same as before parsers should be used at the end, because it doesn't end
+    # with newlines and that leads to complications.
+    differ.parse(func_before + 'a', copies=1, parsers=1)
+
+    # Now that we have a newline at the end, everything is easier in Python
+    # syntax, we can parse once and then get a copy.
+    differ.parse(func_before + 'a\n', copies=1, parsers=1)
+    differ.parse(func_before + 'a\n', copies=1)
 
     # Getting rid of an old parser: Still no parsers used.
-    differ.parse('a', copies=1)
+    differ.parse('a\n', copies=1)
     # Now the file has completely changed and we need to parse.
-    differ.parse('b', parsers=1)
+    differ.parse('b\n', parsers=1)
     # And again.
-    differ.parse('a', parsers=1)
+    differ.parse('a\n', parsers=1)
 
 
 def test_positions(differ):
@@ -112,7 +120,7 @@ def test_positions(differ):
     assert m.start_pos == (1, 0)
     assert m.end_pos == (3, 1)
 
-    m = differ.parse('a', copies=1)
+    m = differ.parse('a', parsers=1)
     assert m.start_pos == (1, 0)
     assert m.end_pos == (1, 1)
 
@@ -151,7 +159,7 @@ def test_func_with_for_and_comment(differ):
         # COMMENT
         a""")
     differ.initialize(src)
-    differ.parse('a\n' + src, copies=1, parsers=1)
+    differ.parse('a\n' + src, copies=1, parsers=2)
 
 
 def test_one_statement_func(differ):