diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 07eba752..ed0a8c5d 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -77,13 +77,6 @@ class Parser(object): :param top_module: Use this module as a parent instead of `self.module`. """ def __init__(self, grammar, source, module_path=None, tokenizer=None): - """ - This is the way I imagine a parser describing the init function - """ - - if not source.endswith('\n'): - source += '\n' - self._ast_mapping = { 'expr_stmt': pt.ExprStmt, 'classdef': pt.Class, @@ -128,13 +121,22 @@ class Parser(object): self.used_names = {} self.scope_names_stack = [{}] self.error_statement_stacks = [] - # For fast parser + + added_newline = False + # The Python grammar needs a newline at the end of each statement. + if not source.endswith('\n'): + source += '\n' + added_newline = True + + # For the fast parser. self.position_modifier = pt.PositionModifier() p = PgenParser(grammar, self.convert_node, self.convert_leaf, self.error_recovery) tokenizer = tokenizer or tokenize.source_tokens(source) self.module = p.parse(self._tokenize(tokenizer)) + if added_newline: + self._remove_last_newline(source) self.module.used_names = self.used_names self.module.path = module_path self.module.set_global_names(self.global_names) @@ -306,3 +308,14 @@ class Parser(object): def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.module) + + def _remove_last_newline(self, source): + endmarker = self.module.children[-1] + print('ENDNL', self.module.children, repr(endmarker.prefix)) + # The newline is either in the endmarker as a prefix or the previous + # leaf as a newline token. + if endmarker.prefix.endswith('\n'): + endmarker.prefix = endmarker.prefix[:-1] + else: + newline = endmarker.get_previous() + newline.value = '' diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 5ac57d51..e48646a8 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -65,7 +65,7 @@ def check_fp(src, number_parsers_used): # TODO Don't change get_code, the whole thing should be the same. # -> Need to refactor the parser first, though. - assert src == p.module.get_code()[:-1] + assert src == p.module.get_code() assert p.number_parsers_used == number_parsers_used return p.module @@ -105,6 +105,23 @@ def test_positions(): assert m.end_pos == (1, 1) +def test_if(): + # Empty the parser cache for the path None. + cache.parser_cache.pop(None, None) + src = dedent('''\ + def func(): + x = 3 + if x: + def y(): + x + x = 3 + + pass + ''') + + # Two parsers needed, one for pass and one for the function. + m = check_fp(src, 2) + def test_incomplete_function(): source = '''return ImportErr''' diff --git a/test/test_parser/test_get_code.py b/test/test_parser/test_get_code.py index 8f552938..6d3b9c34 100644 --- a/test/test_parser/test_get_code.py +++ b/test/test_parser/test_get_code.py @@ -3,7 +3,7 @@ import difflib import pytest from jedi._compatibility import u -from jedi.parser import Parser +from jedi.parser import Parser, load_grammar code_basic_features = u(''' """A mod docstring""" @@ -85,3 +85,19 @@ def method_with_docstring(): pass ''') assert Parser(s).module.get_code() == s + + +def test_end_newlines(): + """ + The Python grammar explicitly needs a newline at the end. Jedi though still + wants to be able, to return the exact same code without the additional new + line the parser needs. + """ + def test(source): + assert Parser(load_grammar(), source).module.get_code() == source + + test('a') + test('a\nb') + test('a\n') + test('a\n\n') + test('a\n#comment')