From d81e393c0c18eee41358c8e4b26cbd68627e16bc Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Apr 2020 21:48:14 +0200 Subject: [PATCH] Fix indentation issues with backslashes and def error recovery --- parso/python/diff.py | 41 ++++++++++++++++++++-------------------- test/test_diff_parser.py | 9 ++------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/parso/python/diff.py b/parso/python/diff.py index 0973afc..79f78ab 100644 --- a/parso/python/diff.py +++ b/parso/python/diff.py @@ -23,6 +23,7 @@ _INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT' NEWLINE = PythonTokenTypes.NEWLINE DEDENT = PythonTokenTypes.DEDENT +NAME = PythonTokenTypes.NAME ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT ENDMARKER = PythonTokenTypes.ENDMARKER @@ -173,12 +174,6 @@ def _func_or_class_has_suite(node): return node.type in ('classdef', 'funcdef') and node.children[-1].type == 'suite' -def _get_suite_func_or_class_parent(node): - while node.parent.type in ('funcdef', 'classdef', 'async_stmt', 'async_funcdef', 'decorated'): - node = node.parent - return node - - def _suite_or_file_input_is_valid(pgen_grammar, stack): if not _flows_finished(pgen_grammar, stack): return False @@ -390,7 +385,7 @@ class DiffParser(object): node = self._try_parse_part(until_line) nodes = node.children - self._nodes_tree.add_parsed_nodes(nodes) + self._nodes_tree.add_parsed_nodes(nodes, self._keyword_token_indents) if self._replace_tos_indent is not None: self._nodes_tree.indents[-1] = self._replace_tos_indent @@ -441,6 +436,7 @@ class DiffParser(object): ) stack = self._active_parser.stack self._replace_tos_indent = None + self._keyword_token_indents = {} # print('start', line_offset + 1, indents) for token in tokens: # print(token, indents) @@ -483,6 +479,9 @@ class DiffParser(object): yield PythonToken(ENDMARKER, '', token.start_pos, '') break + if typ == NAME and token.string in ('class', 'def'): + self._keyword_token_indents[token.start_pos] = list(indents) + yield token @@ -491,17 +490,12 @@ class _NodesTreeNode(object): '_ChildrenGroup', 'prefix children line_offset last_line_offset_leaf') - def __init__(self, tree_node, parent=None): + def __init__(self, tree_node, parent=None, indentation=0): self.tree_node = tree_node self._children_groups = [] self.parent = parent self._node_children = [] - - if self.tree_node.type == 'file_input': - self.indentation = 0 - else: - n = _get_suite_func_or_class_parent(self.tree_node) - self.indentation = _get_indentation(n) + self.indentation = indentation def finish(self): children = [] @@ -585,7 +579,7 @@ class _NodesTree(object): return node self._working_stack.pop() - def add_parsed_nodes(self, tree_nodes): + def add_parsed_nodes(self, tree_nodes, keyword_token_indents): old_prefix = self.prefix tree_nodes = self._remove_endmarker(tree_nodes) if not tree_nodes: @@ -598,19 +592,23 @@ class _NodesTree(object): assert node.tree_node.type in ('suite', 'file_input') node.add_tree_nodes(old_prefix, tree_nodes) # tos = Top of stack - self._update_parsed_node_tos(tree_nodes[-1]) + self._update_parsed_node_tos(tree_nodes[-1], keyword_token_indents) - def _update_parsed_node_tos(self, tree_node): + def _update_parsed_node_tos(self, tree_node, keyword_token_indents): if tree_node.type == 'suite': - new_tos = _NodesTreeNode(tree_node) + def_leaf = tree_node.parent.children[0] + new_tos = _NodesTreeNode( + tree_node, + indentation=keyword_token_indents[def_leaf.start_pos][-1], + ) new_tos.add_tree_nodes('', list(tree_node.children)) self._working_stack[-1].add_child_node(new_tos) self._working_stack.append(new_tos) - self._update_parsed_node_tos(tree_node.children[-1]) + self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents) elif _func_or_class_has_suite(tree_node): - self._update_parsed_node_tos(tree_node.children[-1]) + self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents) def _remove_endmarker(self, tree_nodes): """ @@ -766,7 +764,8 @@ class _NodesTree(object): indent = _get_suite_indentation(suite) added_indents.append(indent) - suite_tos = _NodesTreeNode(suite) + + suite_tos = _NodesTreeNode(suite, indentation=_get_indentation(last_node)) # Don't need to pass line_offset here, it's already done by the # parent. suite_nodes, new_working_stack, new_prefix, ai = self._copy_nodes( diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 47969d5..b962532 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -1671,12 +1671,7 @@ def test_yet_another_backslash(differ): differ.parse(code1, parsers=ANY, copies=ANY) -@pytest.mark.xfail -def test_very_weird_indentation(differ): - """ - This test for now fails, because it's just hard to implement this, but it's - also not really important. - """ +def test_backslash_before_def(differ): code1 = dedent('''\ def f(): x @@ -1696,4 +1691,4 @@ def test_very_weird_indentation(differ): ''') differ.initialize(code1) - differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True) + differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)