Fix an issue in the diff parser.

jedi/parser/__init__.py

@@ -21,7 +21,7 @@ import re
 from jedi.parser import tree as pt
 from jedi.parser import tokenize
 from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER,
-                               STRING)
+                               STRING, tok_name)
 from jedi.parser.pgen2.pgen import generate_grammar
 from jedi.parser.pgen2.parse import PgenParser
 
@@ -321,7 +321,7 @@ class ParserWithRecovery(Parser):
                 # Otherwise the parser will get into trouble and DEDENT too early.
                 self._omit_dedent_list.append(self._indent_counter)
             else:
-                error_leaf = pt.ErrorLeaf(typ, value, start_pos, prefix)
+                error_leaf = pt.ErrorLeaf(tok_name[typ].lower(), value, start_pos, prefix)
                 stack[-1][2][1].append(error_leaf)
 
     def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):

jedi/parser/fast.py

@@ -44,7 +44,7 @@ def _merge_used_names(base_dict, other_dict):
 def _get_last_line(node_or_leaf):
     last_leaf = node_or_leaf.last_leaf()
     if last_leaf.type == 'error_leaf':
-        typ = tok_name[last_leaf.original_type].lower()
+        typ = last_leaf.original_type
     else:
         typ = last_leaf.type
     if typ == 'newline':
@@ -86,6 +86,17 @@ def _is_flow_node(node):
     return value in ('if', 'for', 'while', 'try')
 
 
+def _last_leaf_is_newline(last_leaf):
+    if last_leaf.prefix.endswith('\n'):
+        return True
+    if last_leaf.prefix:
+        return False
+    previous_leaf = last_leaf.get_previous_leaf()
+    return (previous_leaf.type == 'newline' or
+            previous_leaf.type == 'error_leaf' and
+            previous_leaf.original_type == 'newline')
+
+
 class DiffParser(object):
     endmarker_type = 'endmarker'
 
@@ -269,8 +280,7 @@ class DiffParser(object):
         is_endmarker = last_leaf.type == self.endmarker_type
         if is_endmarker:
             self._parsed_until_line = last_leaf.start_pos[0]
-            if last_leaf.prefix.endswith('\n') or \
+            if _last_leaf_is_newline(last_leaf):
-                    not last_leaf.prefix and last_leaf.get_previous_leaf().type == 'newline':
                 self._parsed_until_line -= 1
         else:
             if last_leaf.type == 'newline':

jedi/parser/tree.py

@@ -692,9 +692,8 @@ class ErrorLeaf(LeafWithNewLines):
         self.original_type = original_type
 
     def __repr__(self):
-        token_type = token.tok_name[self.original_type]
         return "<%s: %s:%s, %s)>" % \
-            (type(self).__name__, token_type, repr(self.value), self.start_pos)
+            (type(self).__name__, self.original_type, repr(self.value), self.start_pos)
 
 
 class IsScopeMeta(type):

test/test_parser/test_diff_parser.py

@@ -8,9 +8,8 @@ from jedi._compatibility import u
 from jedi.common import splitlines
 from jedi import cache
 from jedi.parser import load_grammar
-from jedi.parser.fast import FastParser, DiffParser
+from jedi.parser.fast import DiffParser
 from jedi.parser import ParserWithRecovery
-from jedi.parser.utils import save_parser
 
 
 def test_add_to_end():
@@ -284,3 +283,18 @@ def test_wrong_whitespace(differ):
 
     code += """abc(\npass\n    """
     differ.parse(code, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_issues_with_error_leaves(differ):
+    code = dedent('''
+    def ints():
+        str..
+        str
+    ''')
+    code2 = dedent('''
+    def ints():
+        str.
+        str
+    ''')
+    differ.initialize(code)
+    differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True)