VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-06 14:04:26 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove the position_modifier from the parser.

Browse Source
This commit is contained in:
Dave Halter
2016-09-18 00:50:31 +02:00
parent ed71d05ed7
commit 885cf62a12
11 changed files with 62 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
jedi/evaluate/compiled/__init__.py
View File

@@ -10,7 +10,7 @@ from functools import partial
from jedi._compatibility import builtins as _builtins, unicode
from jedi import debug
from jedi.cache import underscore_memoization, memoize_method
from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
from jedi.parser.tree import Param, Base, Operator
from jedi.evaluate.helpers import FakeName
from . import fake
@@ -89,7 +89,7 @@ class CompiledObject(Base):
for p in tokens:
parts = [FakeName(part) for part in p.strip().split('=')]
if len(parts) > 1:
parts.insert(1, Operator(zero_position_modifier, '=', (0, 0)))
parts.insert(1, Operator('=', (0, 0)))
params.append(Param(parts, self))
return params

4
jedi/evaluate/compiled/fake.py
View File

@@ -180,8 +180,8 @@ def _get_faked(module, obj, name=None):
# contain it).
doc = '"""%s"""' % obj.__doc__ # TODO need escapes.
suite = result.children[-1]
string = pt.String(pt.zero_position_modifier, doc, (0, 0), '')
new_line = pt.Newline(pt.zero_position_modifier, '\n', (0, 0))
string = pt.String(doc, (0, 0), '')
new_line = pt.Newline('\n', (0, 0))
docstr_node = pt.Node('simple_stmt', [string, new_line])
suite.children.insert(1, docstr_node)
return result

5
jedi/evaluate/compiled/mixed.py
View File

@@ -80,6 +80,11 @@ class MixedName(compiled.CompiledName):
# This means a start_pos that doesn't exist (compiled objects).
return (0, 0)
@start_pos.setter
def start_pos(self, value):
# Ignore the __init__'s start_pos setter call.
pass
class LazyMixedNamesDict(compiled.LazyNamesDict):
name_class = MixedName

2
jedi/evaluate/helpers.py
View File

@@ -172,7 +172,7 @@ class FakeName(tree.Name):
In case is_definition is defined (not None), that bool value will be
returned.
"""
super(FakeName, self).__init__(tree.zero_position_modifier, name_str, start_pos)
super(FakeName, self).__init__(name_str, start_pos)
self.parent = parent
self._is_definition = is_definition

9
jedi/evaluate/pep0484.py
View File

@@ -64,15 +64,15 @@ def _fix_forward_reference(evaluator, node):
try:
p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
start_symbol='eval_input')
newnode = p.get_parsed_node()
new_node = p.get_parsed_node()
except ParseError:
debug.warning('Annotation not parsed: %s' % evaled_node.obj)
return node
else:
module = node.get_parent_until()
p.position_modifier.line = module.end_pos[0]
newnode.parent = module
return newnode
new_node.move(module.end_pos[0])
new_node.parent = module
return new_node
else:
return node
@@ -188,7 +188,6 @@ def _find_type_from_comment_hint(evaluator, node, varlist, name):
if not match:
return []
annotation = tree.String(
tree.zero_position_modifier,
repr(str(match.group(1).strip())),
node.start_pos)
annotation.parent = node.parent

3
jedi/evaluate/representation.py
View File

@@ -273,8 +273,7 @@ class LazyInstanceDict(object):
class InstanceName(tree.Name):
def __init__(self, origin_name, parent):
super(InstanceName, self).__init__(tree.zero_position_modifier,
origin_name.value,
super(InstanceName, self).__init__(origin_name.value,
origin_name.start_pos)
self._origin_name = origin_name
self.parent = parent

33
jedi/parser/__init__.py
View File

@@ -104,9 +104,6 @@ class Parser(object):
self._last_failed_start_pos = (0, 0)
self._global_names = []
# For the fast parser.
self.position_modifier = pt.PositionModifier()
self.source = source
self._added_newline = False
# The Python grammar needs a newline at the end of each statement.
@@ -211,9 +208,9 @@ class Parser(object):
if value in ('def', 'class', 'lambda'):
self._scope_names_stack.append({})
return pt.Keyword(self.position_modifier, value, start_pos, prefix)
return pt.Keyword(value, start_pos, prefix)
else:
name = pt.Name(self.position_modifier, value, start_pos, prefix)
name = pt.Name(value, start_pos, prefix)
# Keep a listing of all used names
arr = self._used_names.setdefault(name.value, [])
arr.append(name)
@@ -221,22 +218,17 @@ class Parser(object):
arr.append(name)
return name
elif type == STRING:
return pt.String(self.position_modifier, value, start_pos, prefix)
return pt.String(value, start_pos, prefix)
elif type == NUMBER:
return pt.Number(self.position_modifier, value, start_pos, prefix)
return pt.Number(value, start_pos, prefix)
elif type == NEWLINE:
return pt.Newline(self.position_modifier, value, start_pos, prefix)
return pt.Newline(value, start_pos, prefix)
elif type == ENDMARKER:
return pt.EndMarker(self.position_modifier, value, start_pos, prefix)
return pt.EndMarker(value, start_pos, prefix)
else:
return pt.Operator(self.position_modifier, value, start_pos, prefix)
return pt.Operator(value, start_pos, prefix)
def remove_last_newline(self):
"""
In all of this we need to work with _start_pos, because if we worked
with start_pos, we would need to check the position_modifier as well
(which is accounted for in the start_pos property).
"""
endmarker = self._parsed.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
@@ -252,7 +244,7 @@ class Parser(object):
except IndexError:
pass
last_line = re.sub('.*\n', '', prefix)
endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line)
endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
else:
try:
newline = endmarker.get_previous_leaf()
@@ -274,14 +266,15 @@ class Parser(object):
break
else:
newline.value = ''
if self._last_failed_start_pos > newline._start_pos:
if self._last_failed_start_pos > newline.start_pos:
# It may be the case that there was a syntax error in a
# function. In that case error correction removes the
# right newline. So we use the previously assigned
# _last_failed_start_pos variable to account for that.
endmarker._start_pos = self._last_failed_start_pos
endmarker.start_pos = self._last_failed_start_pos
raise NotImplementedError
else:
endmarker._start_pos = newline._start_pos
endmarker.start_pos = newline.start_pos
break
@@ -370,7 +363,7 @@ class ParserWithRecovery(Parser):
# Otherwise the parser will get into trouble and DEDENT too early.
self._omit_dedent_list.append(self._indent_counter)
else:
error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix)
error_leaf = pt.ErrorLeaf(typ, value, start_pos, prefix)
stack[-1][2][1].append(error_leaf)
def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):

6
jedi/parser/fast.py
View File

@@ -169,8 +169,8 @@ class DiffParser(object):
line_stmt = self._get_old_line_stmt(parsed_until_line_old + 1)
if line_stmt is None:
# Parse 1 line at least. We don't need more, because we just
# want to get into a state where the old parser has starting
# statements again (not e.g. lines within parentheses).
# want to get into a state where the old parser has statements
# again that can be copied (e.g. not lines within parentheses).
self._parse(self._parsed_until_line + 1)
else:
print('copy', line_stmt.end_pos, parsed_until_line_old, until_line_old, line_stmt)
@@ -476,7 +476,7 @@ class DiffParser(object):
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker(self._parser.position_modifier, '', tuple(end_pos), self._prefix)
endmarker = EndMarker('', tuple(end_pos), self._prefix)
endmarker.parent = self._new_module
self._new_children.append(endmarker)

48
jedi/parser/tree.py
View File

@@ -73,9 +73,6 @@ class PositionModifier(object):
self.line = 0
zero_position_modifier = PositionModifier()
class DocstringMixin(object):
__slots__ = ()
@@ -288,22 +285,22 @@ class Base(object):
class Leaf(Base):
__slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix')
__slots__ = ('value', 'parent', 'line', 'indent', 'prefix')
def __init__(self, position_modifier, value, start_pos, prefix=''):
self.position_modifier = position_modifier
def __init__(self, value, start_pos, prefix=''):
self.value = value
self._start_pos = start_pos
self.start_pos = start_pos
self.prefix = prefix
self.parent = None
@property
def start_pos(self):
return self._start_pos[0] + self.position_modifier.line, self._start_pos[1]
return self.line, self.indent
@start_pos.setter
def start_pos(self, value):
self._start_pos = value[0] - self.position_modifier.line, value[1]
self.line = value[0]
self.indent = value[1]
def get_start_pos_of_prefix(self):
try:
@@ -313,12 +310,10 @@ class Leaf(Base):
@property
def end_pos(self):
return (self._start_pos[0] + self.position_modifier.line,
self._start_pos[1] + len(self.value))
return self.line, self.indent + len(self.value)
def move(self, line_offset, column_offset):
self._start_pos = (self._start_pos[0] + line_offset,
self._start_pos[1] + column_offset)
def move(self, line_offset):
self.line += line_offset
def first_leaf(self):
return self
@@ -353,15 +348,14 @@ class LeafWithNewLines(Leaf):
Literals and whitespace end_pos are more complicated than normal
end_pos, because the containing newlines may change the indexes.
"""
end_pos_line, end_pos_col = self.start_pos
lines = self.value.split('\n')
end_pos_line += len(lines) - 1
end_pos_line = self.line + len(lines) - 1
# Check for multiline token
if self.start_pos[0] == end_pos_line:
end_pos_col += len(lines[-1])
if self.line == end_pos_line:
end_pos_indent = self.indent + len(lines[-1])
else:
end_pos_col = len(lines[-1])
return end_pos_line, end_pos_col
end_pos_indent = len(lines[-1])
return end_pos_line, end_pos_indent
@utf8_repr
def __repr__(self):
@@ -399,7 +393,7 @@ class Name(Leaf):
def __repr__(self):
return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,
self.start_pos[0], self.start_pos[1])
self.line, self.indent)
def is_definition(self):
if self.parent.type in ('power', 'atom_expr'):
@@ -505,12 +499,12 @@ class BaseNode(Base):
self.children = children
self.parent = None
def move(self, line_offset, column_offset):
def move(self, line_offset):
"""
Move the Node's start_pos.
"""
for c in self.children:
c.move(line_offset, column_offset)
c.move(line_offset)
@property
def start_pos(self):
@@ -687,8 +681,8 @@ class ErrorLeaf(LeafWithNewLines):
__slots__ = ('original_type')
type = 'error_leaf'
def __init__(self, position_modifier, original_type, value, start_pos, prefix=''):
super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix)
def __init__(self, original_type, value, start_pos, prefix=''):
super(ErrorLeaf, self).__init__(value, start_pos, prefix)
self.original_type = original_type
def __repr__(self):
@@ -820,7 +814,7 @@ class Module(Scope):
string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
# Positions are not real, but a module starts at (1, 0)
p = (1, 0)
name = Name(zero_position_modifier, string, p)
name = Name(string, p)
name.parent = self
return name
@@ -1106,7 +1100,7 @@ class Lambda(Function):
@property
def name(self):
# Borrow the position of the <Keyword: lambda> AST node.
return Name(self.children[0].position_modifier, '<lambda>', self.children[0].start_pos)
return Name('<lambda>', self.children[0].start_pos)
def _get_paramlist_code(self):
return '(' + ''.join(param.get_code() for param in self.params).strip() + ')'

4
test/run.py
View File

@@ -118,6 +118,7 @@ from io import StringIO
from functools import reduce
import jedi
from jedi import debug
from jedi._compatibility import unicode, is_py3
from jedi.parser import Parser, load_grammar
from jedi.api.classes import Definition
@@ -186,7 +187,7 @@ class IntegrationTestCase(object):
for match in re.finditer('(?:[^ ]+)', correct):
string = match.group(0)
parser = Parser(load_grammar(), string, start_symbol='eval_input')
parser.position_modifier.line = self.line_nr
parser.get_root_node().move(self.line_nr)
element = parser.get_parsed_node()
element.parent = jedi.api.completion.get_user_scope(
script._get_module(),
@@ -198,6 +199,7 @@ class IntegrationTestCase(object):
% (match.string, self.line_nr - 1))
should_be |= set(Definition(evaluator, r) for r in results)
debug.dbg('Finished getting types', color='YELLOW')
# Because the objects have different ids, `repr`, then compare.
should = set(comparison(r) for r in should_be)

11
test/test_parser/test_diff_parser.py
View File

@@ -200,7 +200,7 @@ def test_for_on_one_line(differ):
def test_open_parentheses(differ):
func = 'def func():\n a'
func = 'def func():\n a\n'
code = 'isinstance(\n\n' + func
new_code = 'isinstance(\n' + func
differ.initialize(code)
@@ -208,10 +208,13 @@ def test_open_parentheses(differ):
differ.parse(new_code, parsers=1, expect_error_leafs=True)
new_code = 'a = 1\n' + new_code
differ.parse(new_code, parsers=2, expect_error_leafs=True)
differ.parse(new_code, copies=1, parsers=1, expect_error_leafs=True)
differ.initialize(new_code)
differ.parse('isinstance()\n' + func, parsers=2, copies=0)
func += 'def other_func():\n pass\n'
differ.initialize('isinstance(\n' + func)
# Cannot copy all, because the prefix of the function is once a newline and
# once not.
differ.parse('isinstance()\n' + func, parsers=2, copies=1)
def test_backslash(differ):