diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index f004a142..731b6a71 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -10,7 +10,7 @@ from functools import partial from jedi._compatibility import builtins as _builtins, unicode from jedi import debug from jedi.cache import underscore_memoization, memoize_method -from jedi.parser.tree import Param, Base, Operator, zero_position_modifier +from jedi.parser.tree import Param, Base, Operator from jedi.evaluate.helpers import FakeName from . import fake @@ -89,7 +89,7 @@ class CompiledObject(Base): for p in tokens: parts = [FakeName(part) for part in p.strip().split('=')] if len(parts) > 1: - parts.insert(1, Operator(zero_position_modifier, '=', (0, 0))) + parts.insert(1, Operator('=', (0, 0))) params.append(Param(parts, self)) return params diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 80da3a91..e94543ed 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -180,8 +180,8 @@ def _get_faked(module, obj, name=None): # contain it). doc = '"""%s"""' % obj.__doc__ # TODO need escapes. suite = result.children[-1] - string = pt.String(pt.zero_position_modifier, doc, (0, 0), '') - new_line = pt.Newline(pt.zero_position_modifier, '\n', (0, 0)) + string = pt.String(doc, (0, 0), '') + new_line = pt.Newline('\n', (0, 0)) docstr_node = pt.Node('simple_stmt', [string, new_line]) suite.children.insert(1, docstr_node) return result diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 69119f18..d5e1569e 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -80,6 +80,11 @@ class MixedName(compiled.CompiledName): # This means a start_pos that doesn't exist (compiled objects). return (0, 0) + @start_pos.setter + def start_pos(self, value): + # Ignore the __init__'s start_pos setter call. + pass + class LazyMixedNamesDict(compiled.LazyNamesDict): name_class = MixedName diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index d3bef265..7c61ac29 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -172,7 +172,7 @@ class FakeName(tree.Name): In case is_definition is defined (not None), that bool value will be returned. """ - super(FakeName, self).__init__(tree.zero_position_modifier, name_str, start_pos) + super(FakeName, self).__init__(name_str, start_pos) self.parent = parent self._is_definition = is_definition diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 2387fe64..749936d7 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -64,15 +64,15 @@ def _fix_forward_reference(evaluator, node): try: p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj), start_symbol='eval_input') - newnode = p.get_parsed_node() + new_node = p.get_parsed_node() except ParseError: debug.warning('Annotation not parsed: %s' % evaled_node.obj) return node else: module = node.get_parent_until() - p.position_modifier.line = module.end_pos[0] - newnode.parent = module - return newnode + new_node.move(module.end_pos[0]) + new_node.parent = module + return new_node else: return node @@ -188,7 +188,6 @@ def _find_type_from_comment_hint(evaluator, node, varlist, name): if not match: return [] annotation = tree.String( - tree.zero_position_modifier, repr(str(match.group(1).strip())), node.start_pos) annotation.parent = node.parent diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 8f3d642c..251d9cbb 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -273,8 +273,7 @@ class LazyInstanceDict(object): class InstanceName(tree.Name): def __init__(self, origin_name, parent): - super(InstanceName, self).__init__(tree.zero_position_modifier, - origin_name.value, + super(InstanceName, self).__init__(origin_name.value, origin_name.start_pos) self._origin_name = origin_name self.parent = parent diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 406a3a2e..9a8ae09a 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -104,9 +104,6 @@ class Parser(object): self._last_failed_start_pos = (0, 0) self._global_names = [] - # For the fast parser. - self.position_modifier = pt.PositionModifier() - self.source = source self._added_newline = False # The Python grammar needs a newline at the end of each statement. @@ -211,9 +208,9 @@ class Parser(object): if value in ('def', 'class', 'lambda'): self._scope_names_stack.append({}) - return pt.Keyword(self.position_modifier, value, start_pos, prefix) + return pt.Keyword(value, start_pos, prefix) else: - name = pt.Name(self.position_modifier, value, start_pos, prefix) + name = pt.Name(value, start_pos, prefix) # Keep a listing of all used names arr = self._used_names.setdefault(name.value, []) arr.append(name) @@ -221,22 +218,17 @@ class Parser(object): arr.append(name) return name elif type == STRING: - return pt.String(self.position_modifier, value, start_pos, prefix) + return pt.String(value, start_pos, prefix) elif type == NUMBER: - return pt.Number(self.position_modifier, value, start_pos, prefix) + return pt.Number(value, start_pos, prefix) elif type == NEWLINE: - return pt.Newline(self.position_modifier, value, start_pos, prefix) + return pt.Newline(value, start_pos, prefix) elif type == ENDMARKER: - return pt.EndMarker(self.position_modifier, value, start_pos, prefix) + return pt.EndMarker(value, start_pos, prefix) else: - return pt.Operator(self.position_modifier, value, start_pos, prefix) + return pt.Operator(value, start_pos, prefix) def remove_last_newline(self): - """ - In all of this we need to work with _start_pos, because if we worked - with start_pos, we would need to check the position_modifier as well - (which is accounted for in the start_pos property). - """ endmarker = self._parsed.children[-1] # The newline is either in the endmarker as a prefix or the previous # leaf as a newline token. @@ -252,7 +244,7 @@ class Parser(object): except IndexError: pass last_line = re.sub('.*\n', '', prefix) - endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line) + endmarker.start_pos = endmarker.line - 1, last_end + len(last_line) else: try: newline = endmarker.get_previous_leaf() @@ -274,14 +266,15 @@ class Parser(object): break else: newline.value = '' - if self._last_failed_start_pos > newline._start_pos: + if self._last_failed_start_pos > newline.start_pos: # It may be the case that there was a syntax error in a # function. In that case error correction removes the # right newline. So we use the previously assigned # _last_failed_start_pos variable to account for that. - endmarker._start_pos = self._last_failed_start_pos + endmarker.start_pos = self._last_failed_start_pos + raise NotImplementedError else: - endmarker._start_pos = newline._start_pos + endmarker.start_pos = newline.start_pos break @@ -370,7 +363,7 @@ class ParserWithRecovery(Parser): # Otherwise the parser will get into trouble and DEDENT too early. self._omit_dedent_list.append(self._indent_counter) else: - error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix) + error_leaf = pt.ErrorLeaf(typ, value, start_pos, prefix) stack[-1][2][1].append(error_leaf) def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos): diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index e251562c..589a77d5 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -169,8 +169,8 @@ class DiffParser(object): line_stmt = self._get_old_line_stmt(parsed_until_line_old + 1) if line_stmt is None: # Parse 1 line at least. We don't need more, because we just - # want to get into a state where the old parser has starting - # statements again (not e.g. lines within parentheses). + # want to get into a state where the old parser has statements + # again that can be copied (e.g. not lines within parentheses). self._parse(self._parsed_until_line + 1) else: print('copy', line_stmt.end_pos, parsed_until_line_old, until_line_old, line_stmt) @@ -476,7 +476,7 @@ class DiffParser(object): end_pos[0] += len(lines) - 1 end_pos[1] = len(lines[-1]) - endmarker = EndMarker(self._parser.position_modifier, '', tuple(end_pos), self._prefix) + endmarker = EndMarker('', tuple(end_pos), self._prefix) endmarker.parent = self._new_module self._new_children.append(endmarker) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 7c3380b9..5a888ee8 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -73,9 +73,6 @@ class PositionModifier(object): self.line = 0 -zero_position_modifier = PositionModifier() - - class DocstringMixin(object): __slots__ = () @@ -288,22 +285,22 @@ class Base(object): class Leaf(Base): - __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix') + __slots__ = ('value', 'parent', 'line', 'indent', 'prefix') - def __init__(self, position_modifier, value, start_pos, prefix=''): - self.position_modifier = position_modifier + def __init__(self, value, start_pos, prefix=''): self.value = value - self._start_pos = start_pos + self.start_pos = start_pos self.prefix = prefix self.parent = None @property def start_pos(self): - return self._start_pos[0] + self.position_modifier.line, self._start_pos[1] + return self.line, self.indent @start_pos.setter def start_pos(self, value): - self._start_pos = value[0] - self.position_modifier.line, value[1] + self.line = value[0] + self.indent = value[1] def get_start_pos_of_prefix(self): try: @@ -313,12 +310,10 @@ class Leaf(Base): @property def end_pos(self): - return (self._start_pos[0] + self.position_modifier.line, - self._start_pos[1] + len(self.value)) + return self.line, self.indent + len(self.value) - def move(self, line_offset, column_offset): - self._start_pos = (self._start_pos[0] + line_offset, - self._start_pos[1] + column_offset) + def move(self, line_offset): + self.line += line_offset def first_leaf(self): return self @@ -353,15 +348,14 @@ class LeafWithNewLines(Leaf): Literals and whitespace end_pos are more complicated than normal end_pos, because the containing newlines may change the indexes. """ - end_pos_line, end_pos_col = self.start_pos lines = self.value.split('\n') - end_pos_line += len(lines) - 1 + end_pos_line = self.line + len(lines) - 1 # Check for multiline token - if self.start_pos[0] == end_pos_line: - end_pos_col += len(lines[-1]) + if self.line == end_pos_line: + end_pos_indent = self.indent + len(lines[-1]) else: - end_pos_col = len(lines[-1]) - return end_pos_line, end_pos_col + end_pos_indent = len(lines[-1]) + return end_pos_line, end_pos_indent @utf8_repr def __repr__(self): @@ -399,7 +393,7 @@ class Name(Leaf): def __repr__(self): return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, - self.start_pos[0], self.start_pos[1]) + self.line, self.indent) def is_definition(self): if self.parent.type in ('power', 'atom_expr'): @@ -505,12 +499,12 @@ class BaseNode(Base): self.children = children self.parent = None - def move(self, line_offset, column_offset): + def move(self, line_offset): """ Move the Node's start_pos. """ for c in self.children: - c.move(line_offset, column_offset) + c.move(line_offset) @property def start_pos(self): @@ -687,8 +681,8 @@ class ErrorLeaf(LeafWithNewLines): __slots__ = ('original_type') type = 'error_leaf' - def __init__(self, position_modifier, original_type, value, start_pos, prefix=''): - super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix) + def __init__(self, original_type, value, start_pos, prefix=''): + super(ErrorLeaf, self).__init__(value, start_pos, prefix) self.original_type = original_type def __repr__(self): @@ -820,7 +814,7 @@ class Module(Scope): string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) # Positions are not real, but a module starts at (1, 0) p = (1, 0) - name = Name(zero_position_modifier, string, p) + name = Name(string, p) name.parent = self return name @@ -1106,7 +1100,7 @@ class Lambda(Function): @property def name(self): # Borrow the position of the AST node. - return Name(self.children[0].position_modifier, '', self.children[0].start_pos) + return Name('', self.children[0].start_pos) def _get_paramlist_code(self): return '(' + ''.join(param.get_code() for param in self.params).strip() + ')' diff --git a/test/run.py b/test/run.py index 3eef7343..f27b1371 100755 --- a/test/run.py +++ b/test/run.py @@ -118,6 +118,7 @@ from io import StringIO from functools import reduce import jedi +from jedi import debug from jedi._compatibility import unicode, is_py3 from jedi.parser import Parser, load_grammar from jedi.api.classes import Definition @@ -186,7 +187,7 @@ class IntegrationTestCase(object): for match in re.finditer('(?:[^ ]+)', correct): string = match.group(0) parser = Parser(load_grammar(), string, start_symbol='eval_input') - parser.position_modifier.line = self.line_nr + parser.get_root_node().move(self.line_nr) element = parser.get_parsed_node() element.parent = jedi.api.completion.get_user_scope( script._get_module(), @@ -198,6 +199,7 @@ class IntegrationTestCase(object): % (match.string, self.line_nr - 1)) should_be |= set(Definition(evaluator, r) for r in results) + debug.dbg('Finished getting types', color='YELLOW') # Because the objects have different ids, `repr`, then compare. should = set(comparison(r) for r in should_be) diff --git a/test/test_parser/test_diff_parser.py b/test/test_parser/test_diff_parser.py index 090ba1c5..e76e15d6 100644 --- a/test/test_parser/test_diff_parser.py +++ b/test/test_parser/test_diff_parser.py @@ -200,7 +200,7 @@ def test_for_on_one_line(differ): def test_open_parentheses(differ): - func = 'def func():\n a' + func = 'def func():\n a\n' code = 'isinstance(\n\n' + func new_code = 'isinstance(\n' + func differ.initialize(code) @@ -208,10 +208,13 @@ def test_open_parentheses(differ): differ.parse(new_code, parsers=1, expect_error_leafs=True) new_code = 'a = 1\n' + new_code - differ.parse(new_code, parsers=2, expect_error_leafs=True) + differ.parse(new_code, copies=1, parsers=1, expect_error_leafs=True) - differ.initialize(new_code) - differ.parse('isinstance()\n' + func, parsers=2, copies=0) + func += 'def other_func():\n pass\n' + differ.initialize('isinstance(\n' + func) + # Cannot copy all, because the prefix of the function is once a newline and + # once not. + differ.parse('isinstance()\n' + func, parsers=2, copies=1) def test_backslash(differ):