Remove the position_modifier from the parser.

jedi/evaluate/compiled/__init__.py

@@ -10,7 +10,7 @@ from functools import partial
 from jedi._compatibility import builtins as _builtins, unicode
 from jedi import debug
 from jedi.cache import underscore_memoization, memoize_method
-from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
+from jedi.parser.tree import Param, Base, Operator
 from jedi.evaluate.helpers import FakeName
 from . import fake
 
@@ -89,7 +89,7 @@ class CompiledObject(Base):
         for p in tokens:
             parts = [FakeName(part) for part in p.strip().split('=')]
             if len(parts) > 1:
-                parts.insert(1, Operator(zero_position_modifier, '=', (0, 0)))
+                parts.insert(1, Operator('=', (0, 0)))
             params.append(Param(parts, self))
         return params
 

jedi/evaluate/compiled/fake.py

@@ -180,8 +180,8 @@ def _get_faked(module, obj, name=None):
         # contain it).
         doc = '"""%s"""' % obj.__doc__  # TODO need escapes.
         suite = result.children[-1]
-        string = pt.String(pt.zero_position_modifier, doc, (0, 0), '')
+        string = pt.String(doc, (0, 0), '')
-        new_line = pt.Newline(pt.zero_position_modifier, '\n', (0, 0))
+        new_line = pt.Newline('\n', (0, 0))
         docstr_node = pt.Node('simple_stmt', [string, new_line])
         suite.children.insert(1, docstr_node)
         return result

jedi/evaluate/compiled/mixed.py

@@ -80,6 +80,11 @@ class MixedName(compiled.CompiledName):
         # This means a start_pos that doesn't exist (compiled objects).
         return (0, 0)
 
+    @start_pos.setter
+    def start_pos(self, value):
+        # Ignore the __init__'s start_pos setter call.
+        pass
+
 
 class LazyMixedNamesDict(compiled.LazyNamesDict):
     name_class = MixedName

jedi/evaluate/helpers.py

@@ -172,7 +172,7 @@ class FakeName(tree.Name):
         In case is_definition is defined (not None), that bool value will be
         returned.
         """
-        super(FakeName, self).__init__(tree.zero_position_modifier, name_str, start_pos)
+        super(FakeName, self).__init__(name_str, start_pos)
         self.parent = parent
         self._is_definition = is_definition
 

jedi/evaluate/pep0484.py

@@ -64,15 +64,15 @@ def _fix_forward_reference(evaluator, node):
         try:
             p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
                        start_symbol='eval_input')
-            newnode = p.get_parsed_node()
+            new_node = p.get_parsed_node()
         except ParseError:
             debug.warning('Annotation not parsed: %s' % evaled_node.obj)
             return node
         else:
             module = node.get_parent_until()
-            p.position_modifier.line = module.end_pos[0]
+            new_node.move(module.end_pos[0])
-            newnode.parent = module
+            new_node.parent = module
-            return newnode
+            return new_node
     else:
         return node
 
@@ -188,7 +188,6 @@ def _find_type_from_comment_hint(evaluator, node, varlist, name):
     if not match:
         return []
     annotation = tree.String(
-        tree.zero_position_modifier,
         repr(str(match.group(1).strip())),
         node.start_pos)
     annotation.parent = node.parent

jedi/evaluate/representation.py

@@ -273,8 +273,7 @@ class LazyInstanceDict(object):
 
 class InstanceName(tree.Name):
     def __init__(self, origin_name, parent):
-        super(InstanceName, self).__init__(tree.zero_position_modifier,
+        super(InstanceName, self).__init__(origin_name.value,
-                                           origin_name.value,
                                            origin_name.start_pos)
         self._origin_name = origin_name
         self.parent = parent

jedi/parser/__init__.py

@@ -104,9 +104,6 @@ class Parser(object):
         self._last_failed_start_pos = (0, 0)
         self._global_names = []
 
-        # For the fast parser.
-        self.position_modifier = pt.PositionModifier()
-
         self.source = source
         self._added_newline = False
         # The Python grammar needs a newline at the end of each statement.
@@ -211,9 +208,9 @@ class Parser(object):
                 if value in ('def', 'class', 'lambda'):
                     self._scope_names_stack.append({})
 
-                return pt.Keyword(self.position_modifier, value, start_pos, prefix)
+                return pt.Keyword(value, start_pos, prefix)
             else:
-                name = pt.Name(self.position_modifier, value, start_pos, prefix)
+                name = pt.Name(value, start_pos, prefix)
                 # Keep a listing of all used names
                 arr = self._used_names.setdefault(name.value, [])
                 arr.append(name)
@@ -221,22 +218,17 @@ class Parser(object):
                 arr.append(name)
                 return name
         elif type == STRING:
-            return pt.String(self.position_modifier, value, start_pos, prefix)
+            return pt.String(value, start_pos, prefix)
         elif type == NUMBER:
-            return pt.Number(self.position_modifier, value, start_pos, prefix)
+            return pt.Number(value, start_pos, prefix)
         elif type == NEWLINE:
-            return pt.Newline(self.position_modifier, value, start_pos, prefix)
+            return pt.Newline(value, start_pos, prefix)
         elif type == ENDMARKER:
-            return pt.EndMarker(self.position_modifier, value, start_pos, prefix)
+            return pt.EndMarker(value, start_pos, prefix)
         else:
-            return pt.Operator(self.position_modifier, value, start_pos, prefix)
+            return pt.Operator(value, start_pos, prefix)
 
     def remove_last_newline(self):
-        """
-        In all of this we need to work with _start_pos, because if we worked
-        with start_pos, we would need to check the position_modifier as well
-        (which is accounted for in the start_pos property).
-        """
         endmarker = self._parsed.children[-1]
         # The newline is either in the endmarker as a prefix or the previous
         # leaf as a newline token.
@@ -252,7 +244,7 @@ class Parser(object):
                 except IndexError:
                     pass
             last_line = re.sub('.*\n', '', prefix)
-            endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line)
+            endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
         else:
             try:
                 newline = endmarker.get_previous_leaf()
@@ -274,14 +266,15 @@ class Parser(object):
                     break
                 else:
                     newline.value = ''
-                    if self._last_failed_start_pos > newline._start_pos:
+                    if self._last_failed_start_pos > newline.start_pos:
                         # It may be the case that there was a syntax error in a
                         # function. In that case error correction removes the
                         # right newline. So we use the previously assigned
                         # _last_failed_start_pos variable to account for that.
-                        endmarker._start_pos = self._last_failed_start_pos
+                        endmarker.start_pos = self._last_failed_start_pos
+                        raise NotImplementedError
                     else:
-                        endmarker._start_pos = newline._start_pos
+                        endmarker.start_pos = newline.start_pos
                     break
 
 
@@ -370,7 +363,7 @@ class ParserWithRecovery(Parser):
                 # Otherwise the parser will get into trouble and DEDENT too early.
                 self._omit_dedent_list.append(self._indent_counter)
             else:
-                error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix)
+                error_leaf = pt.ErrorLeaf(typ, value, start_pos, prefix)
                 stack[-1][2][1].append(error_leaf)
 
     def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):

jedi/parser/fast.py

@@ -169,8 +169,8 @@ class DiffParser(object):
             line_stmt = self._get_old_line_stmt(parsed_until_line_old + 1)
             if line_stmt is None:
                 # Parse 1 line at least. We don't need more, because we just
-                # want to get into a state where the old parser has starting
+                # want to get into a state where the old parser has statements
-                # statements again (not e.g. lines within parentheses).
+                # again that can be copied (e.g. not lines within parentheses).
                 self._parse(self._parsed_until_line + 1)
             else:
                 print('copy', line_stmt.end_pos, parsed_until_line_old, until_line_old, line_stmt)
@@ -476,7 +476,7 @@ class DiffParser(object):
             end_pos[0] += len(lines) - 1
             end_pos[1] = len(lines[-1])
 
-        endmarker = EndMarker(self._parser.position_modifier, '', tuple(end_pos), self._prefix)
+        endmarker = EndMarker('', tuple(end_pos), self._prefix)
         endmarker.parent = self._new_module
         self._new_children.append(endmarker)
 

jedi/parser/tree.py

@@ -73,9 +73,6 @@ class PositionModifier(object):
         self.line = 0
 
 
-zero_position_modifier = PositionModifier()
-
-
 class DocstringMixin(object):
     __slots__ = ()
 
@@ -288,22 +285,22 @@ class Base(object):
 
 
 class Leaf(Base):
-    __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix')
+    __slots__ = ('value', 'parent', 'line', 'indent', 'prefix')
 
-    def __init__(self, position_modifier, value, start_pos, prefix=''):
+    def __init__(self, value, start_pos, prefix=''):
-        self.position_modifier = position_modifier
         self.value = value
-        self._start_pos = start_pos
+        self.start_pos = start_pos
         self.prefix = prefix
         self.parent = None
 
     @property
     def start_pos(self):
-        return self._start_pos[0] + self.position_modifier.line, self._start_pos[1]
+        return self.line, self.indent
 
     @start_pos.setter
     def start_pos(self, value):
-        self._start_pos = value[0] - self.position_modifier.line, value[1]
+        self.line = value[0]
+        self.indent = value[1]
 
     def get_start_pos_of_prefix(self):
         try:
@@ -313,12 +310,10 @@ class Leaf(Base):
 
     @property
     def end_pos(self):
-        return (self._start_pos[0] + self.position_modifier.line,
+        return self.line, self.indent + len(self.value)
-                self._start_pos[1] + len(self.value))
 
-    def move(self, line_offset, column_offset):
+    def move(self, line_offset):
-        self._start_pos = (self._start_pos[0] + line_offset,
+        self.line += line_offset
-                           self._start_pos[1] + column_offset)
 
     def first_leaf(self):
         return self
@@ -353,15 +348,14 @@ class LeafWithNewLines(Leaf):
         Literals and whitespace end_pos are more complicated than normal
         end_pos, because the containing newlines may change the indexes.
         """
-        end_pos_line, end_pos_col = self.start_pos
         lines = self.value.split('\n')
-        end_pos_line += len(lines) - 1
+        end_pos_line = self.line + len(lines) - 1
         # Check for multiline token
-        if self.start_pos[0] == end_pos_line:
+        if self.line == end_pos_line:
-            end_pos_col += len(lines[-1])
+            end_pos_indent = self.indent + len(lines[-1])
         else:
-            end_pos_col = len(lines[-1])
+            end_pos_indent = len(lines[-1])
-        return end_pos_line, end_pos_col
+        return end_pos_line, end_pos_indent
 
     @utf8_repr
     def __repr__(self):
@@ -399,7 +393,7 @@ class Name(Leaf):
 
     def __repr__(self):
         return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,
-                                   self.start_pos[0], self.start_pos[1])
+                                   self.line, self.indent)
 
     def is_definition(self):
         if self.parent.type in ('power', 'atom_expr'):
@@ -505,12 +499,12 @@ class BaseNode(Base):
         self.children = children
         self.parent = None
 
-    def move(self, line_offset, column_offset):
+    def move(self, line_offset):
         """
         Move the Node's start_pos.
         """
         for c in self.children:
-            c.move(line_offset, column_offset)
+            c.move(line_offset)
 
     @property
     def start_pos(self):
@@ -687,8 +681,8 @@ class ErrorLeaf(LeafWithNewLines):
     __slots__ = ('original_type')
     type = 'error_leaf'
 
-    def __init__(self, position_modifier, original_type, value, start_pos, prefix=''):
+    def __init__(self, original_type, value, start_pos, prefix=''):
-        super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix)
+        super(ErrorLeaf, self).__init__(value, start_pos, prefix)
         self.original_type = original_type
 
     def __repr__(self):
@@ -820,7 +814,7 @@ class Module(Scope):
             string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
         # Positions are not real, but a module starts at (1, 0)
         p = (1, 0)
-        name = Name(zero_position_modifier, string, p)
+        name = Name(string, p)
         name.parent = self
         return name
 
@@ -1106,7 +1100,7 @@ class Lambda(Function):
     @property
     def name(self):
         # Borrow the position of the <Keyword: lambda> AST node.
-        return Name(self.children[0].position_modifier, '<lambda>', self.children[0].start_pos)
+        return Name('<lambda>', self.children[0].start_pos)
 
     def _get_paramlist_code(self):
         return '(' + ''.join(param.get_code() for param in self.params).strip() + ')'

test/run.py

@@ -118,6 +118,7 @@ from io import StringIO
 from functools import reduce
 
 import jedi
+from jedi import debug
 from jedi._compatibility import unicode, is_py3
 from jedi.parser import Parser, load_grammar
 from jedi.api.classes import Definition
@@ -186,7 +187,7 @@ class IntegrationTestCase(object):
             for match in re.finditer('(?:[^ ]+)', correct):
                 string = match.group(0)
                 parser = Parser(load_grammar(), string, start_symbol='eval_input')
-                parser.position_modifier.line = self.line_nr
+                parser.get_root_node().move(self.line_nr)
                 element = parser.get_parsed_node()
                 element.parent = jedi.api.completion.get_user_scope(
                     script._get_module(),
@@ -198,6 +199,7 @@ class IntegrationTestCase(object):
                                     % (match.string, self.line_nr - 1))
 
                 should_be |= set(Definition(evaluator, r) for r in results)
+            debug.dbg('Finished getting types', color='YELLOW')
 
             # Because the objects have different ids, `repr`, then compare.
             should = set(comparison(r) for r in should_be)

test/test_parser/test_diff_parser.py

@@ -200,7 +200,7 @@ def test_for_on_one_line(differ):
 
 
 def test_open_parentheses(differ):
-    func = 'def func():\n a'
+    func = 'def func():\n a\n'
     code = 'isinstance(\n\n' + func
     new_code = 'isinstance(\n' + func
     differ.initialize(code)
@@ -208,10 +208,13 @@ def test_open_parentheses(differ):
     differ.parse(new_code, parsers=1, expect_error_leafs=True)
 
     new_code = 'a = 1\n' + new_code
-    differ.parse(new_code, parsers=2, expect_error_leafs=True)
+    differ.parse(new_code, copies=1, parsers=1, expect_error_leafs=True)
 
-    differ.initialize(new_code)
+    func += 'def other_func():\n pass\n'
-    differ.parse('isinstance()\n' + func, parsers=2, copies=0)
+    differ.initialize('isinstance(\n' + func)
+    # Cannot copy all, because the prefix of the function is once a newline and
+    # once not.
+    differ.parse('isinstance()\n' + func, parsers=2, copies=1)
 
 
 def test_backslash(differ):