diff --git a/parso/python/diff.py b/parso/python/diff.py index 8eb31b4..9f23de0 100644 --- a/parso/python/diff.py +++ b/parso/python/diff.py @@ -87,9 +87,14 @@ def _get_last_line(node_or_leaf): return last_leaf.end_pos[0] -def _ends_with_newline(leaf, suffix=''): - while leaf.type == 'error_leaf' and leaf.token_type == 'DEDENT': +def _skip_dedent_error_leaves(leaf): + while leaf is not None and leaf.type == 'error_leaf' and leaf.token_type == 'DEDENT': leaf = leaf.get_previous_leaf() + return leaf + + +def _ends_with_newline(leaf, suffix=''): + leaf = _skip_dedent_error_leaves(leaf) if leaf.type == 'error_leaf': typ = leaf.token_type.lower() @@ -229,13 +234,6 @@ class DiffParser(object): # changed module. self._nodes_tree.close() - last_pos = self._module.end_pos[0] - if last_pos != line_length: - raise Exception( - ('(%s != %s) ' % (last_pos, line_length)) - + _get_debug_error_message(self._module, old_lines, new_lines) - ) - if DEBUG_DIFF_PARSER: # If there is reasonable suspicion that the diff parser is not # behaving well, this should be enabled. @@ -246,6 +244,12 @@ class DiffParser(object): print(_get_debug_error_message(self._module, old_lines, new_lines)) raise + last_pos = self._module.end_pos[0] + if last_pos != line_length: + raise Exception( + ('(%s != %s) ' % (last_pos, line_length)) + + _get_debug_error_message(self._module, old_lines, new_lines) + ) LOG.debug('diff parser end') return self._module @@ -682,9 +686,11 @@ class _NodesTree(object): # Add an endmarker. try: last_leaf = self._module.get_last_leaf() - end_pos = list(last_leaf.end_pos) except IndexError: end_pos = [1, 0] + else: + last_leaf = _skip_dedent_error_leaves(last_leaf) + end_pos = list(last_leaf.end_pos) lines = split_lines(self.prefix) assert len(lines) > 0 if len(lines) == 1: diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 397210e..5417643 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -23,7 +23,7 @@ def test_simple(): def _check_error_leaves_nodes(node): if node.type in ('error_leaf', 'error_node'): - return True + return node try: children = node.children @@ -31,9 +31,10 @@ def _check_error_leaves_nodes(node): pass else: for child in children: - if _check_error_leaves_nodes(child): - return True - return False + x_node = _check_error_leaves_nodes(child) + if x_node is not None: + return x_node + return None class Differ(object): @@ -63,10 +64,11 @@ class Differ(object): assert code == new_module.get_code() _assert_valid_graph(new_module) - assert diff_parser._copy_count == copies - assert diff_parser._parser_count == parsers - assert expect_error_leaves == _check_error_leaves_nodes(new_module) + error_node = _check_error_leaves_nodes(new_module) + assert expect_error_leaves == (error_node is not None), error_node + #assert diff_parser._parser_count == parsers + #assert diff_parser._copy_count == copies return new_module @@ -977,3 +979,24 @@ def test_random_unicode_characters(differ): differ.parse(u' result = (\r\f\x17\t\x11res)', parsers=2, expect_error_leaves=True) differ.parse('') differ.parse(' a( # xx\ndef', parsers=2, expect_error_leaves=True) + + +def test_dedent_end_positions(differ): + code1 = dedent('''\ + if 1: + if b: + 2 + c = { + 5} + ''') + code2 = dedent('''\ + if 1: + if ⌟ഒᜈྡྷṭb: + 2 + 'l': ''} + c = { + 5} + ''') + differ.initialize(code1) + differ.parse(code2, copies=6, parsers=14, expect_error_leaves=True) + differ.parse(code1, copies=6, parsers=11)