From 12269629225683c113a1da5723ac8b98181165f8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 11 Sep 2016 13:03:29 +0200 Subject: [PATCH 1/2] Remove dedents from the parser tree. No need for them. --- jedi/api/helpers.py | 9 +++++++++ jedi/evaluate/compiled/fake.py | 4 ++-- jedi/evaluate/docstrings.py | 2 +- jedi/parser/__init__.py | 6 ++++++ jedi/parser/tree.py | 2 +- jedi/settings.py | 2 +- test/test_parser/test_parser.py | 4 +--- 7 files changed, 21 insertions(+), 8 deletions(-) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 8102cc7f..14c8bac7 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -101,6 +101,10 @@ def _get_code_for_stack(code_lines, module, position): if leaf.type in ('indent', 'dedent'): return u('') elif leaf.type == 'error_leaf' or leaf.type == 'string': + if leaf.start_pos[0] < position[0]: + # On a different line, we just begin anew. + return u('') + # Error leafs cannot be parsed, completion in strings is also # impossible. raise OnErrorLeaf(leaf) @@ -266,9 +270,14 @@ def _get_call_signature_details_from_error_node(node, position): def get_call_signature_details(module, position): leaf = module.get_leaf_for_position(position, include_prefixes=True) + if leaf.start_pos >= position: + # Whitespace / comments after the leaf count towards the previous leaf. + leaf = leaf.get_previous_leaf() + if leaf == ')': if leaf.end_pos == position: leaf = leaf.get_next_leaf() + # Now that we know where we are in the syntax tree, we start to look at # parents for possible function definitions. node = leaf.parent diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 0338b962..80da3a91 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -181,9 +181,9 @@ def _get_faked(module, obj, name=None): doc = '"""%s"""' % obj.__doc__ # TODO need escapes. suite = result.children[-1] string = pt.String(pt.zero_position_modifier, doc, (0, 0), '') - new_line = pt.Newline('\n', (0, 0), '') + new_line = pt.Newline(pt.zero_position_modifier, '\n', (0, 0)) docstr_node = pt.Node('simple_stmt', [string, new_line]) - suite.children.insert(2, docstr_node) + suite.children.insert(1, docstr_node) return result diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index d2ab34ed..929122c3 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -136,7 +136,7 @@ def _evaluate_for_statement_string(evaluator, string, module): pseudo_cls = p.module.subscopes[0] # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, # which is also not the last item, because there's a newline. - stmt = pseudo_cls.children[-1].children[-2].children[-2] + stmt = pseudo_cls.children[-1].children[-1].children[-2] except (AttributeError, IndexError): return [] diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 22f38605..8ca39adc 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -169,6 +169,12 @@ class Parser(object): try: new_node = Parser.AST_MAPPING[symbol](children) except KeyError: + if symbol == 'suite': + # We don't want the INDENT/DEDENT in our parser tree. Those + # leaves are just cancer. They are virtual leaves and not real + # ones and therefore have pseudo start/end positions and no + # prefixes. Just ignore them. + children = [children[0]] + children[2:-1] new_node = pt.Node(symbol, children) # We need to check raw_node always, because the same node can be diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index c5a2d554..a35fe437 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -74,7 +74,7 @@ class DocstringMixin(object): elif isinstance(self, ClassOrFunc): node = self.children[self.children.index(':') + 1] if is_node(node, 'suite'): # Normally a suite - node = node.children[2] # -> NEWLINE INDENT stmt + node = node.children[1] # -> NEWLINE stmt else: # ExprStmt simple_stmt = self.parent c = simple_stmt.parent.children diff --git a/jedi/settings.py b/jedi/settings.py index 13dcfd45..b5e15759 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -130,7 +130,7 @@ On Linux, if environment variable ``$XDG_CACHE_HOME`` is set, # parser # ---------------- -fast_parser = True +fast_parser = False """ Use the fast parser. This means that reparsing is only being done if something has been changed e.g. to a function. If this happens, only the diff --git a/test/test_parser/test_parser.py b/test/test_parser/test_parser.py index 0ec75d0a..14ec763a 100644 --- a/test/test_parser/test_parser.py +++ b/test/test_parser/test_parser.py @@ -177,9 +177,7 @@ def test_end_pos_error_correction(): m = ParserWithRecovery(load_grammar(), s).module func = m.children[0] assert func.type == 'funcdef' - # This is not exactly correct, but ok, because it doesn't make a difference - # at all. We just want to make sure that the module end_pos is correct! - assert func.end_pos == (3, 0) + assert func.end_pos == (2, 2) assert m.end_pos == (2, 2) From 7667cba17e6609c4c8fc58693bad6b00a58ad79e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 11 Sep 2016 13:20:24 +0200 Subject: [PATCH 2/2] Remove old indent/dedent usages. Now they are not needed anymore. --- jedi/api/helpers.py | 18 +----------------- jedi/evaluate/docstrings.py | 2 +- jedi/evaluate/helpers.py | 2 +- jedi/parser/__init__.py | 4 ---- jedi/parser/tree.py | 22 ++-------------------- 5 files changed, 5 insertions(+), 43 deletions(-) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 14c8bac7..a0044269 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -51,19 +51,6 @@ class OnErrorLeaf(Exception): def _is_on_comment(leaf, position): - # We might be on a comment. - if leaf.type == 'endmarker': - try: - dedent = leaf.get_previous_leaf() - if dedent.type == 'dedent' and dedent.prefix: - # TODO This is needed because the fast parser uses multiple - # endmarker tokens within a file which is obviously ugly. - # This is so ugly that I'm not even commenting how it exactly - # happens, but let me tell you that I want to get rid of it. - leaf = dedent - except IndexError: - pass - comment_lines = common.splitlines(leaf.prefix) difference = leaf.start_pos[0] - position[0] prefix_start_pos = leaf.get_start_pos_of_prefix() @@ -98,9 +85,7 @@ def _get_code_for_stack(code_lines, module, position): except IndexError: return u('') - if leaf.type in ('indent', 'dedent'): - return u('') - elif leaf.type == 'error_leaf' or leaf.type == 'string': + if leaf.type == 'error_leaf' or leaf.type == 'string': if leaf.start_pos[0] < position[0]: # On a different line, we just begin anew. return u('') @@ -146,7 +131,6 @@ def get_stack_at_position(grammar, code_lines, module, pos): # completion. # Use Z as a prefix because it's not part of a number suffix. safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI' - # Remove as many indents from **all** code lines as possible. code = code + safeword p = parser.ParserWithRecovery(grammar, code, start_parsing=False) diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 929122c3..7c33b28b 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -134,7 +134,7 @@ def _evaluate_for_statement_string(evaluator, string, module): p = ParserWithRecovery(load_grammar(), code % indent_block(string)) try: pseudo_cls = p.module.subscopes[0] - # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, + # First pick suite, then simple_stmt and then the node, # which is also not the last item, because there's a newline. stmt = pseudo_cls.children[-1].children[-1].children[-2] except (AttributeError, IndexError): diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 27293eda..d3bef265 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -27,7 +27,7 @@ def deep_ast_copy(obj, parent=None, new_elements=None): for child in obj.children: typ = child.type if typ in ('newline', 'operator', 'keyword', 'number', 'string', - 'indent', 'dedent', 'endmarker', 'error_leaf'): + 'endmarker', 'error_leaf'): # At the moment we're not actually copying those primitive # elements, because there's really no need to. The parents are # obviously wrong, but that's not an issue. diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 8ca39adc..64f70903 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -223,10 +223,6 @@ class Parser(object): return pt.Number(self.position_modifier, value, start_pos, prefix) elif type == NEWLINE: return pt.Newline(self.position_modifier, value, start_pos, prefix) - elif type == INDENT: - return pt.Indent(self.position_modifier, value, start_pos, prefix) - elif type == DEDENT: - return pt.Dedent(self.position_modifier, value, start_pos, prefix) elif type == ENDMARKER: return pt.EndMarker(self.position_modifier, value, start_pos, prefix) else: diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index a35fe437..601a6152 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -147,7 +147,7 @@ class Base(object): return scope def get_definition(self): - if self.type in ('newline', 'dedent', 'indent', 'endmarker'): + if self.type in ('newline', 'endmarker'): raise ValueError('Cannot get the indentation of whitespace or indentation.') scope = self while scope.parent is not None: @@ -294,11 +294,7 @@ class Leaf(Base): def get_start_pos_of_prefix(self): try: - previous_leaf = self - while True: - previous_leaf = previous_leaf.get_previous_leaf() - if previous_leaf.type not in ('indent', 'dedent'): - return previous_leaf.end_pos + return self.get_previous_leaf().end_pos except IndexError: return 1, 0 # It's the first leaf. @@ -426,16 +422,6 @@ class String(Literal): __slots__ = () -class Indent(Leaf): - type = 'indent' - __slots__ = () - - -class Dedent(Leaf): - type = 'dedent' - __slots__ = () - - class Operator(Leaf): type = 'operator' __slots__ = () @@ -550,10 +536,6 @@ class BaseNode(Base): try: return c.get_leaf_for_position(position, include_prefixes) except AttributeError: - while c.type in ('indent', 'dedent'): - # We'd rather not have indents and dedents as a leaf, - # because they don't contain indentation information. - c = c.get_next_leaf() return c return None