Get the position modifiers right.

jedi/parser/fast.py

@@ -118,10 +118,9 @@ class ParserNode(object):
 
         try:
             # With fast_parser we have either 1 subscope or only statements.
-            self._content_scope = self._names_dict_scope = parser.module.subscopes[0]
+            self._content_scope = parser.module.subscopes[0]
         except IndexError:
             self._content_scope = parser.module
-            self._names_dict_scope = parser.module
 
         # We need to be able to reset the original children of a parser.
         self._old_children = list(self._content_scope.children)
@@ -183,14 +182,13 @@ class ParserNode(object):
             if self.parser is not None:
                 # The first Parser node contains all the others and is
                 # typically empty.
-                dcts.insert(0, self._names_dict_scope.names_dict)
+                dcts.insert(0, self._content_scope.names_dict)
             print('DCTS', self.parser, dcts, self._node_children)
             self._content_scope.names_dict = MergedNamesDict(dcts)
 
     def parent_until_indent(self, indent=None):
         if indent is None or self._indent >= indent and self.parent:
             if self.parent is not None:
-                print('until_indent')
                 self.close()
                 return self.parent.parent_until_indent(indent)
         return self
@@ -231,7 +229,7 @@ class ParserNode(object):
         # Changing the line offsets is very important, because if they don't
         # fit, all the start_pos values will be wrong.
         m = node.parser.module
-        m.line_offset += line_offset + 1 - m.start_pos[0]
+        node.parser.position_modifier.line = line_offset + 1 - m.start_pos[0]
         self._fast_module.modules.append(m)
         node.parent = self
 
@@ -264,7 +262,6 @@ class ParserNode(object):
     def add_parser(self, parser, code):
         # TODO REMOVE
         raise NotImplementedError
-        print('add parser')
         return self.add_node(ParserNode(self._fast_module, parser, code, self), True)
 
     def all_sub_nodes(self):
@@ -294,12 +291,14 @@ class FastParser(use_metaclass(CachedFastParser)):
         self.current_node.set_parser(self, '')
 
     def update(self, code):
+        # For testing purposes: It is important that the number of parsers used
+        # can be minimized. With this variable we can test it.
         self.number_parsers_used = 0
         self.module.reset_caches()
         try:
             self._parse(code)
         except:
-            # FastParser is cached, be careful with exceptions
+            # FastParser is cached, be careful with exceptions.
             self._reset_caches()
             raise
 
@@ -446,7 +445,7 @@ class FastParser(use_metaclass(CachedFastParser)):
         """
         h = hash(code)
         for index, node in enumerate(nodes):
-            print('EQ', node, repr(node.code), repr(code), id(node))
+            print('EQ', node, repr(node.code), repr(code))
             if node.hash == h and node.code == code:
                 node.reset_node()
                 nodes.remove(node)
@@ -471,7 +470,6 @@ class FastParser(use_metaclass(CachedFastParser)):
             code_part_actually_used = '\n'.join(used_lines)
             node.set_parser(p, code_part_actually_used)
 
-        print('add', id(node))
         self.current_node.add_node(node, line_offset)
         return node
 
@@ -527,7 +525,6 @@ class FastTokenizer(object):
             return current
 
         if self.previous[0] == DEDENT and not self._in_flow:
-            print('w', self.current, self.previous, self._first_stmt)
             self._first_stmt = False
             return self._close()
         elif self.previous[0] in (None, NEWLINE, INDENT):
@@ -594,7 +591,6 @@ class FastTokenizer(object):
             return tokenize.DEDENT, '', start_pos, ''
         elif not self._returned_endmarker:
             self._returned_endmarker = True
-            print('end')
             return ENDMARKER, '', start_pos, ''
         else:
             raise StopIteration

jedi/parser/tree.py

@@ -158,6 +158,8 @@ class Leaf(Base):
 
     @start_pos.setter
     def start_pos(self, value):
+        # TODO I think this is wrong, because the position_modifier.line needs
+        # to be looked at as well. Probably it needs to be substracted.
         self._start_pos = value
 
     @property
@@ -593,7 +595,7 @@ class SubModule(Scope, Module):
     of a module.
     """
     __slots__ = ('path', 'global_names', 'used_names', '_name',
-                 'line_offset', 'use_as_parent', 'error_statement_stacks')
+                 'use_as_parent', 'error_statement_stacks')
     type = 'file_input'
 
     def __init__(self, children):
@@ -608,7 +610,6 @@ class SubModule(Scope, Module):
         super(SubModule, self).__init__(children)
         self.path = None  # Set later.
         # this may be changed depending on fast_parser
-        self.line_offset = 0
 
     def set_global_names(self, names):
         """

test/test_parser/test_fast_parser.py

@@ -59,34 +59,51 @@ def test_carriage_return_splitting():
     assert [n.value for lst in p.module.names_dict.values() for n in lst] == ['Foo']
 
 
+def check_fp(src, number_parsers_used):
+    p = FastParser(load_grammar(), u(src))
+    cache.save_parser(None, None, p, pickling=False)
+
+    # TODO Don't change get_code, the whole thing should be the same.
+    # -> Need to refactor the parser first, though.
+    assert src == p.module.get_code()[:-1]
+    assert p.number_parsers_used == number_parsers_used
+    return p.module
+
+
 def test_change_and_undo():
-
+    # Empty the parser cache for the path None.
-    def fp(src, number_parsers_used):
-        p = FastParser(load_grammar(), u(src))
-        cache.save_parser(None, None, p, pickling=False)
-
-        # TODO Don't change get_code, the whole thing should be the same.
-        # -> Need to refactor the parser first, though.
-        assert src == p.module.get_code()[:-1]
-        assert p.number_parsers_used == number_parsers_used
-
     cache.parser_cache.pop(None, None)
     func_before = 'def func():\n    pass\n'
     # Parse the function and a.
-    fp(func_before + 'a', 2)
+    check_fp(func_before + 'a', 2)
     # Parse just b.
-    fp(func_before + 'b', 1)
+    check_fp(func_before + 'b', 1)
     # b has changed to a again, so parse that.
-    fp(func_before + 'a', 1)
+    check_fp(func_before + 'a', 1)
     # Same as before no parsers should be used.
-    fp(func_before + 'a', 0)
+    check_fp(func_before + 'a', 0)
 
     # Getting rid of an old parser: Still no parsers used.
-    fp('a', 0)
+    check_fp('a', 0)
     # Now the file has completely change and we need to parse.
-    fp('b', 1)
+    check_fp('b', 1)
     # And again.
-    fp('a', 1)
+    check_fp('a', 1)
+
+
+def test_positions():
+    # Empty the parser cache for the path None.
+    cache.parser_cache.pop(None, None)
+
+    func_before = 'class A:\n pass\n'
+    m = check_fp(func_before + 'a', 2)
+    assert m.start_pos == (1, 0)
+    assert m.end_pos == (3, 1)
+
+    m = check_fp('a', 0)
+    assert m.start_pos == (1, 0)
+    assert m.end_pos == (1, 1)
+
 
 
 def test_incomplete_function():