diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 9df7b7ad..492ca222 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -21,7 +21,7 @@ import re from jedi.parser import tree as pt from jedi.parser import tokenize from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, - STRING) + STRING, tok_name) from jedi.parser.pgen2.pgen import generate_grammar from jedi.parser.pgen2.parse import PgenParser @@ -321,7 +321,7 @@ class ParserWithRecovery(Parser): # Otherwise the parser will get into trouble and DEDENT too early. self._omit_dedent_list.append(self._indent_counter) else: - error_leaf = pt.ErrorLeaf(typ, value, start_pos, prefix) + error_leaf = pt.ErrorLeaf(tok_name[typ].lower(), value, start_pos, prefix) stack[-1][2][1].append(error_leaf) def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos): diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index a6f41bb2..8ee49491 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -44,7 +44,7 @@ def _merge_used_names(base_dict, other_dict): def _get_last_line(node_or_leaf): last_leaf = node_or_leaf.last_leaf() if last_leaf.type == 'error_leaf': - typ = tok_name[last_leaf.original_type].lower() + typ = last_leaf.original_type else: typ = last_leaf.type if typ == 'newline': @@ -86,6 +86,17 @@ def _is_flow_node(node): return value in ('if', 'for', 'while', 'try') +def _last_leaf_is_newline(last_leaf): + if last_leaf.prefix.endswith('\n'): + return True + if last_leaf.prefix: + return False + previous_leaf = last_leaf.get_previous_leaf() + return (previous_leaf.type == 'newline' or + previous_leaf.type == 'error_leaf' and + previous_leaf.original_type == 'newline') + + class DiffParser(object): endmarker_type = 'endmarker' @@ -269,8 +280,7 @@ class DiffParser(object): is_endmarker = last_leaf.type == self.endmarker_type if is_endmarker: self._parsed_until_line = last_leaf.start_pos[0] - if last_leaf.prefix.endswith('\n') or \ - not last_leaf.prefix and last_leaf.get_previous_leaf().type == 'newline': + if _last_leaf_is_newline(last_leaf): self._parsed_until_line -= 1 else: if last_leaf.type == 'newline': diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index bcdc49c9..275bf780 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -692,9 +692,8 @@ class ErrorLeaf(LeafWithNewLines): self.original_type = original_type def __repr__(self): - token_type = token.tok_name[self.original_type] return "<%s: %s:%s, %s)>" % \ - (type(self).__name__, token_type, repr(self.value), self.start_pos) + (type(self).__name__, self.original_type, repr(self.value), self.start_pos) class IsScopeMeta(type): diff --git a/test/test_parser/test_diff_parser.py b/test/test_parser/test_diff_parser.py index 68d73383..84c44a02 100644 --- a/test/test_parser/test_diff_parser.py +++ b/test/test_parser/test_diff_parser.py @@ -8,9 +8,8 @@ from jedi._compatibility import u from jedi.common import splitlines from jedi import cache from jedi.parser import load_grammar -from jedi.parser.fast import FastParser, DiffParser +from jedi.parser.fast import DiffParser from jedi.parser import ParserWithRecovery -from jedi.parser.utils import save_parser def test_add_to_end(): @@ -284,3 +283,18 @@ def test_wrong_whitespace(differ): code += """abc(\npass\n """ differ.parse(code, parsers=1, copies=1, expect_error_leaves=True) + + +def test_issues_with_error_leaves(differ): + code = dedent(''' + def ints(): + str.. + str + ''') + code2 = dedent(''' + def ints(): + str. + str + ''') + differ.initialize(code) + differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True)