Tests and implementation to remove the last newline again in the parser tree, to be able to exactly reproduce the parser input.

jedi/parser/__init__.py

@@ -77,13 +77,6 @@ class Parser(object):
     :param top_module: Use this module as a parent instead of `self.module`.
     """
     def __init__(self, grammar, source, module_path=None, tokenizer=None):
-        """
-        This is the way I imagine a parser describing the init function
-        """
-
-        if not source.endswith('\n'):
-            source += '\n'
-
         self._ast_mapping = {
             'expr_stmt': pt.ExprStmt,
             'classdef': pt.Class,
@@ -128,13 +121,22 @@ class Parser(object):
         self.used_names = {}
         self.scope_names_stack = [{}]
         self.error_statement_stacks = []
-        # For fast parser
+
+        added_newline = False
+        # The Python grammar needs a newline at the end of each statement.
+        if not source.endswith('\n'):
+            source += '\n'
+            added_newline = True
+
+        # For the fast parser.
         self.position_modifier = pt.PositionModifier()
         p = PgenParser(grammar, self.convert_node, self.convert_leaf,
                        self.error_recovery)
         tokenizer = tokenizer or tokenize.source_tokens(source)
         self.module = p.parse(self._tokenize(tokenizer))
 
+        if added_newline:
+            self._remove_last_newline(source)
         self.module.used_names = self.used_names
         self.module.path = module_path
         self.module.set_global_names(self.global_names)
@@ -306,3 +308,14 @@ class Parser(object):
 
     def __repr__(self):
         return "<%s: %s>" % (type(self).__name__, self.module)
+
+    def _remove_last_newline(self, source):
+        endmarker = self.module.children[-1]
+        print('ENDNL', self.module.children, repr(endmarker.prefix))
+        # The newline is either in the endmarker as a prefix or the previous
+        # leaf as a newline token.
+        if endmarker.prefix.endswith('\n'):
+            endmarker.prefix = endmarker.prefix[:-1]
+        else:
+            newline = endmarker.get_previous()
+            newline.value = ''

test/test_parser/test_fast_parser.py

@@ -65,7 +65,7 @@ def check_fp(src, number_parsers_used):
 
     # TODO Don't change get_code, the whole thing should be the same.
     # -> Need to refactor the parser first, though.
-    assert src == p.module.get_code()[:-1]
+    assert src == p.module.get_code()
     assert p.number_parsers_used == number_parsers_used
     return p.module
 
@@ -105,6 +105,23 @@ def test_positions():
     assert m.end_pos == (1, 1)
 
 
+def test_if():
+    # Empty the parser cache for the path None.
+    cache.parser_cache.pop(None, None)
+    src = dedent('''\
+    def func():
+        x = 3
+        if x:
+            def y():
+                x
+        x = 3
+
+    pass
+    ''')
+
+    # Two parsers needed, one for pass and one for the function.
+    m = check_fp(src, 2)
+
 
 def test_incomplete_function():
     source = '''return ImportErr'''

test/test_parser/test_get_code.py

@@ -3,7 +3,7 @@ import difflib
 import pytest
 
 from jedi._compatibility import u
-from jedi.parser import Parser
+from jedi.parser import Parser, load_grammar
 
 code_basic_features = u('''
 """A mod docstring"""
@@ -85,3 +85,19 @@ def method_with_docstring():
     pass
 ''')
     assert Parser(s).module.get_code() == s
+
+
+def test_end_newlines():
+    """
+    The Python grammar explicitly needs a newline at the end. Jedi though still
+    wants to be able, to return the exact same code without the additional new
+    line the parser needs.
+    """
+    def test(source):
+        assert Parser(load_grammar(), source).module.get_code() == source
+
+    test('a')
+    test('a\nb')
+    test('a\n')
+    test('a\n\n')
+    test('a\n#comment')