diff --git a/parso/python/diff.py b/parso/python/diff.py index cda422a..719f3ed 100644 --- a/parso/python/diff.py +++ b/parso/python/diff.py @@ -19,6 +19,8 @@ from parso.python.token import PythonTokenTypes LOG = logging.getLogger(__name__) DEBUG_DIFF_PARSER = False +_INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT' + def _assert_valid_graph(node): """ @@ -33,11 +35,12 @@ def _assert_valid_graph(node): # Ignore INDENT is necessary, because indent/dedent tokens don't # contain value/prefix and are just around, because of the tokenizer. - error_tokens = 'INDENT', 'ERROR_DEDENT', 'DEDENT' - if node.type == 'error_leaf' and node.token_type in error_tokens: + if node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS: + assert not node.value + assert not node.prefix return while previous_leaf and previous_leaf.type == 'error_leaf' \ - and previous_leaf.token_type in error_tokens: + and previous_leaf.token_type in _INDENTATION_TOKENS: assert previous_leaf.end_pos <= node.start_pos, \ (previous_leaf, node) previous_leaf = previous_leaf.get_previous_leaf() @@ -428,6 +431,11 @@ class _NodesTreeNode(object): children = [] for prefix, children_part, line_offset, last_line_offset_leaf in self._children_groups: first_leaf = children_part[0].get_first_leaf() + + while first_leaf.type == 'error_leaf' \ + and first_leaf.token_type in _INDENTATION_TOKENS: + first_leaf = first_leaf.get_next_leaf() + first_leaf.prefix = prefix + first_leaf.prefix if line_offset != 0: try: diff --git a/parso/python/tree.py b/parso/python/tree.py index 3e26b00..a3d8338 100644 --- a/parso/python/tree.py +++ b/parso/python/tree.py @@ -169,7 +169,9 @@ class EndMarker(_LeafWithoutNewlines): @utf8_repr def __repr__(self): - return "<%s: prefix=%s>" % (type(self).__name__, repr(self.prefix)) + return "<%s: prefix=%s end_pos=%s>" % ( + type(self).__name__, repr(self.prefix), self.end_pos + ) class Newline(PythonLeaf): diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 53fa79f..7121e70 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -1069,3 +1069,31 @@ def test_opening_bracket_at_end(differ): differ.initialize(code1) differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True) differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True) + + +def test_all_sorts_of_indentation(differ): + code1 = dedent('''\ + class C: + 1 + def f(): + 'same' + + if foo: + a = b + end + ''') + code2 = dedent('''\ + class C: + 1 + def f(yield await %|( + 'same' + + \x02\x06\x0f\x1c\x11 + if foo: + a = b + + end + ''') + differ.initialize(code1) + differ.parse(code2, copies=2, parsers=3, expect_error_leaves=True) + differ.parse(code1, copies=1, parsers=3)