VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

added a line_offset attribute - enables fast_parser to change positions

Browse Source
This commit is contained in:
David Halter
2012-12-16 17:23:15 +01:00
parent 099e2410ad
commit c1d442fa17
5 changed files with 138 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
jedi/evaluate.py
View File

@@ -714,7 +714,8 @@ class Generator(use_metaclass(cache.CachedMetaClass, parsing.Base)):
none_pos = (0, 0) none_pos = (0, 0)
executes_generator = ('__next__', 'send') executes_generator = ('__next__', 'send')
for n in ('close', 'throw') + executes_generator: for n in ('close', 'throw') + executes_generator:
name = parsing.Name([(n, none_pos)], none_pos, none_pos) name = parsing.Name(builtin.Builtin.scope, [(n, none_pos)],
none_pos, none_pos)
if n in executes_generator: if n in executes_generator:
name.parent = self name.parent = self
names.append(name) names.append(name)
@@ -1332,7 +1333,8 @@ def follow_call_list(call_list):
if isinstance(nested_lc, parsing.ListComprehension): if isinstance(nested_lc, parsing.ListComprehension):
# is nested LC # is nested LC
input = nested_lc.stmt input = nested_lc.stmt
loop = parsing.ForFlow([input], lc.stmt.start_pos, module = input.get_parent_until()
loop = parsing.ForFlow(module, [input], lc.stmt.start_pos,
lc.middle, True) lc.middle, True)
if parent is None: if parent is None:
loop.parent = lc.stmt.parent loop.parent = lc.stmt.parent

28
jedi/fast_parser.py
View File

@@ -10,9 +10,10 @@ parser_cache = {}
class Module(parsing.Simple, parsing.Module): class Module(parsing.Simple, parsing.Module):
def __init__(self, parsers): def __init__(self, parsers):
self._end_pos = None, None self._end_pos = None, None
super(Module, self).__init__((1,0)) super(Module, self).__init__(self, (1,0))
self.parsers = parsers self.parsers = parsers
self.reset_caches() self.reset_caches()
self.line_offset = 0
def reset_caches(self): def reset_caches(self):
""" This module does a whole lot of caching, because it uses different """ This module does a whole lot of caching, because it uses different
@@ -105,6 +106,16 @@ class Module(parsing.Simple, parsing.Module):
raise NotImplementedError("Parser doesn't exist.") raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.is_builtin return self.parsers[0].module.is_builtin
@property
def start_pos(self):
""" overwrite start_pos of Simple """
return 1, 0
@start_pos.setter
def start_pos(self):
""" ignore """
pass
@property @property
def end_pos(self): def end_pos(self):
return self._end_pos return self._end_pos
@@ -172,7 +183,7 @@ class FastParser(use_metaclass(CachedFastParser)):
self.user_position = user_position self.user_position = user_position
self.reset_caches() self.reset_caches()
self.parsers = [] self.parsers = []
self.module.parsers = self.parsers self.module = Module(self.parsers)
self._parse(code) self._parse(code)
@@ -189,12 +200,19 @@ class FastParser(use_metaclass(CachedFastParser)):
parts[0] += parts[1] parts[0] += parts[1]
parts.pop(1) parts.pop(1)
self.parsers = []
self.module.parsers = self.parsers
hashes = {p.hash:p for p in self.parsers}
line_offset = 0 line_offset = 0
start = 0 start = 0
p = None p = None
for s in parts: for code_part in parts:
lines = s.count('\n') lines = code_part.count('\n')
if p is None or line_offset >= p.end_pos[0] - 2: if p is None or line_offset >= p.end_pos[0] - 2:
h = hash(code_part)
if h in hashes:
print(h)
p = parsing.PyFuzzyParser(code[start:], p = parsing.PyFuzzyParser(code[start:],
self.module_path, self.user_position, self.module_path, self.user_position,
line_offset=line_offset, stop_on_scope=True, line_offset=line_offset, stop_on_scope=True,
@@ -203,7 +221,7 @@ class FastParser(use_metaclass(CachedFastParser)):
p.module.parent = self.module p.module.parent = self.module
self.parsers.append(p) self.parsers.append(p)
line_offset += lines line_offset += lines
start += len(s) start += len(code_part)
def reset_caches(self): def reset_caches(self):
self._user_scope = None self._user_scope = None

5
jedi/helpers.py
View File

@@ -166,7 +166,8 @@ def fast_parent_copy(obj):
setattr(new_obj, key, new_elements[value]) setattr(new_obj, key, new_elements[value])
except KeyError: except KeyError:
pass pass
elif key in ['parent_stmt', 'parent_function', 'set_parent']: elif key in ['parent_stmt', 'parent_function', 'set_parent',
'module']:
continue continue
elif isinstance(value, list): elif isinstance(value, list):
setattr(new_obj, key, list_rec(value)) setattr(new_obj, key, list_rec(value))
@@ -230,7 +231,7 @@ def scan_array_for_pos(arr, pos):
elif s.execution is not None: elif s.execution is not None:
end = s.execution.end_pos end = s.execution.end_pos
if s.execution.start_pos < pos and \ if s.execution.start_pos < pos and \
(end is None or pos < end): (None in end or pos < end):
c, index, stop = scan_array_for_pos( c, index, stop = scan_array_for_pos(
s.execution, pos) s.execution, pos)
if stop: if stop:

17
jedi/imports.py
View File

@@ -29,16 +29,16 @@ class ImportPath(parsing.Base):
An ImportPath is the path of a `parsing.Import` object. An ImportPath is the path of a `parsing.Import` object.
""" """
class _GlobalNamespace(object): class _GlobalNamespace(object):
def __init__(self):
self.start_pos = 0, 0
self.line_offset = 0
def get_defined_names(self): def get_defined_names(self):
return [] return []
def get_imports(self): def get_imports(self):
return [] return []
@property
def start_pos(self):
return (0, 0)
def get_parent_until(self): def get_parent_until(self):
return None return None
@@ -88,8 +88,9 @@ class ImportPath(parsing.Base):
# This is not an existing Import statement. Therefore, set position to # This is not an existing Import statement. Therefore, set position to
# 0 (0 is not a valid line number). # 0 (0 is not a valid line number).
zero = (0, 0) zero = (0, 0)
n = parsing.Name(i.namespace.names[1:], zero, zero, self.import_stmt) names = i.namespace.names[1:]
new = parsing.Import(zero, zero, n) n = parsing.Name(i.module, names, zero, zero, self.import_stmt)
new = parsing.Import(i.module, zero, zero, n)
new.parent = parent new.parent = parent
debug.dbg('Generated a nested import: %s' % new) debug.dbg('Generated a nested import: %s' % new)
return new return new
@@ -135,8 +136,8 @@ class ImportPath(parsing.Base):
names = [] names = []
for module_loader, name, is_pkg in pkgutil.iter_modules(search_path): for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
inf_pos = (float('inf'), float('inf')) inf_pos = (float('inf'), float('inf'))
names.append(parsing.Name([(name, inf_pos)], inf_pos, inf_pos, names.append(parsing.Name(self.GlobalNamespace, [(name, inf_pos)],
self.import_stmt)) inf_pos, inf_pos, self.import_stmt))
return names return names
def sys_path_with_modifications(self): def sys_path_with_modifications(self):

149
jedi/parsing.py
View File

@@ -63,12 +63,13 @@ class Simple(Base):
The super class for Scope, Import, Name and Statement. Every object in The super class for Scope, Import, Name and Statement. Every object in
the parser tree inherits from this class. the parser tree inherits from this class.
""" """
def __init__(self, start_pos, end_pos=(None, None)): def __init__(self, module, start_pos, end_pos=(None, None)):
self.start_pos = start_pos self._start_pos = start_pos
self.end_pos = end_pos self._end_pos = end_pos
self.parent = None self.parent = None
# use this attribute if parent should be something else than self. # use this attribute if parent should be something else than self.
self.set_parent = self self.set_parent = self
self.module = module
@Python3Method @Python3Method
def get_parent_until(self, classes=(), reverse=False, def get_parent_until(self, classes=(), reverse=False,
@@ -85,6 +86,24 @@ class Simple(Base):
scope = scope.parent scope = scope.parent
return scope return scope
@property
def start_pos(self):
return self.module.line_offset + self._start_pos[0], self._start_pos[1]
@start_pos.setter
def start_pos(self, value):
self._start_pos = value
@property
def end_pos(self):
if None in self._end_pos:
return self._end_pos
return self.module.line_offset + self._end_pos[0], self._end_pos[1]
@end_pos.setter
def end_pos(self, value):
self._end_pos = value
def __repr__(self): def __repr__(self):
code = self.get_code().replace('\n', ' ') code = self.get_code().replace('\n', ' ')
return "<%s: %s@%s>" % \ return "<%s: %s@%s>" % \
@@ -104,12 +123,12 @@ class Scope(Simple):
:param docstr: The docstring for the current Scope. :param docstr: The docstring for the current Scope.
:type docstr: str :type docstr: str
""" """
def __init__(self, start_pos, docstr=''): def __init__(self, module, start_pos):
super(Scope, self).__init__(start_pos) super(Scope, self).__init__(module, start_pos)
self.subscopes = [] self.subscopes = []
self.imports = [] self.imports = []
self.statements = [] self.statements = []
self.docstr = docstr self.docstr = ''
self.asserts = [] self.asserts = []
def add_scope(self, sub, decorators): def add_scope(self, sub, decorators):
@@ -248,14 +267,14 @@ class SubModule(Scope, Module):
of a module. of a module.
""" """
def __init__(self, path, start_pos, top_module=None): def __init__(self, path, start_pos, top_module=None):
super(SubModule, self).__init__(start_pos) super(SubModule, self).__init__(self, start_pos)
self.path = path self.path = path
self.global_vars = [] self.global_vars = []
self._name = None self._name = None
self.used_names = {} self.used_names = {}
self.temp_used_names = [] self.temp_used_names = []
# this may be changed depending on fast_parser # this may be changed depending on fast_parser
self._line_offset = 0 self.line_offset = 0
self.set_parent = top_module or self self.set_parent = top_module or self
@@ -289,7 +308,8 @@ class SubModule(Scope, Module):
self.path) self.path)
string = r.group(1) string = r.group(1)
names = [(string, (0, 0))] names = [(string, (0, 0))]
self._name = Name(names, self.start_pos, self.end_pos, self.set_parent) self._name = Name(self, names, self.start_pos, self.end_pos,
self.set_parent)
return self._name return self._name
def is_builtin(self): def is_builtin(self):
@@ -306,11 +326,9 @@ class Class(Scope):
:type supers: list :type supers: list
:param start_pos: The start position (line, column) of the class. :param start_pos: The start position (line, column) of the class.
:type start_pos: tuple(int, int) :type start_pos: tuple(int, int)
:param docstr: The docstring for the current Scope.
:type docstr: str
""" """
def __init__(self, name, supers, start_pos, docstr=''): def __init__(self, module, name, supers, start_pos):
super(Class, self).__init__(start_pos, docstr) super(Class, self).__init__(module, start_pos)
self.name = name self.name = name
name.parent = self.set_parent name.parent = self.set_parent
self.supers = supers self.supers = supers
@@ -344,8 +362,8 @@ class Function(Scope):
:param docstr: The docstring for the current Scope. :param docstr: The docstring for the current Scope.
:type docstr: str :type docstr: str
""" """
def __init__(self, name, params, start_pos, annotation): def __init__(self, module, name, params, start_pos, annotation):
Scope.__init__(self, start_pos) super(Function, self).__init__(module, start_pos)
self.name = name self.name = name
name.parent = self.set_parent name.parent = self.set_parent
self.params = params self.params = params
@@ -433,10 +451,10 @@ class Flow(Scope):
:param set_vars: Local variables used in the for loop (only there). :param set_vars: Local variables used in the for loop (only there).
:type set_vars: list :type set_vars: list
""" """
def __init__(self, command, inits, start_pos, set_vars=None): def __init__(self, module, command, inits, start_pos, set_vars=None):
self.next = None self.next = None
self.command = command self.command = command
super(Flow, self).__init__(start_pos, '') super(Flow, self).__init__(module, start_pos)
self._parent = None self._parent = None
# These have to be statements, because of with, which takes multiple. # These have to be statements, because of with, which takes multiple.
self.inits = inits self.inits = inits
@@ -510,8 +528,8 @@ class ForFlow(Flow):
""" """
Used for the for loop, because there are two statement parts. Used for the for loop, because there are two statement parts.
""" """
def __init__(self, inits, start_pos, set_stmt, is_list_comp=False): def __init__(self, module, inits, start_pos, set_stmt, is_list_comp=False):
super(ForFlow, self).__init__('for', inits, start_pos, super(ForFlow, self).__init__(module, 'for', inits, start_pos,
set_stmt.used_vars) set_stmt.used_vars)
self.set_stmt = set_stmt self.set_stmt = set_stmt
self.is_list_comp = is_list_comp self.is_list_comp = is_list_comp
@@ -546,9 +564,9 @@ class Import(Simple):
:param defunct: An Import is valid or not. :param defunct: An Import is valid or not.
:type defunct: bool :type defunct: bool
""" """
def __init__(self, start_pos, end_pos, namespace, alias=None, def __init__(self, module, start_pos, end_pos, namespace, alias=None,
from_ns=None, star=False, relative_count=0, defunct=False): from_ns=None, star=False, relative_count=0, defunct=False):
super(Import, self).__init__(start_pos, end_pos) super(Import, self).__init__(module, start_pos, end_pos)
self.namespace = namespace self.namespace = namespace
self.alias = alias self.alias = alias
@@ -590,8 +608,8 @@ class Import(Simple):
return [self.alias] return [self.alias]
if len(self.namespace) > 1: if len(self.namespace) > 1:
o = self.namespace o = self.namespace
n = Name([(o.names[0], o.start_pos)], o.start_pos, o.end_pos, n = Name(self.module, [(o.names[0], o.start_pos)], o.start_pos,
parent=o.parent) o.end_pos, parent=o.parent)
return [n] return [n]
else: else:
return [self.namespace] return [self.namespace]
@@ -630,9 +648,9 @@ class Statement(Simple):
:param start_pos: Position (line, column) of the Statement. :param start_pos: Position (line, column) of the Statement.
:type start_pos: tuple(int, int) :type start_pos: tuple(int, int)
""" """
def __init__(self, code, set_vars, used_funcs, used_vars, token_list, def __init__(self, module, code, set_vars, used_funcs, used_vars,
start_pos, end_pos): token_list, start_pos, end_pos):
super(Statement, self).__init__(start_pos, end_pos) super(Statement, self).__init__(module, start_pos, end_pos)
self.code = code self.code = code
self.used_funcs = used_funcs self.used_funcs = used_funcs
self.used_vars = used_vars self.used_vars = used_vars
@@ -845,10 +863,10 @@ class Param(Statement):
The class which shows definitions of params of classes and functions. The class which shows definitions of params of classes and functions.
But this is not to define function calls. But this is not to define function calls.
""" """
def __init__(self, code, set_vars, used_funcs, used_vars, def __init__(self, module, code, set_vars, used_funcs, used_vars,
token_list, start_pos, end_pos): token_list, start_pos, end_pos):
super(Param, self).__init__(code, set_vars, used_funcs, used_vars, super(Param, self).__init__(module, code, set_vars, used_funcs,
token_list, start_pos, end_pos) used_vars, token_list, start_pos, end_pos)
# this is defined by the parser later on, not at the initialization # this is defined by the parser later on, not at the initialization
# it is the position in the call (first argument, second...) # it is the position in the call (first argument, second...)
@@ -890,6 +908,15 @@ class Call(Base):
self.execution = None self.execution = None
self._parent_stmt = parent_stmt self._parent_stmt = parent_stmt
@property
def start_pos(self):
offset = self.parent_stmt.module.line_offset
return offset + self._start_pos[0], self._start_pos[1]
@start_pos.setter
def start_pos(self, value):
self._start_pos = value
@property @property
def parent_stmt(self): def parent_stmt(self):
if self._parent_stmt is not None: if self._parent_stmt is not None:
@@ -979,7 +1006,18 @@ class Array(Call):
self.values = values if values else [] self.values = values if values else []
self.arr_el_pos = [] self.arr_el_pos = []
self.keys = [] self.keys = []
self.end_pos = None self._end_pos = None, None
@property
def end_pos(self):
if None in self._end_pos:
return self._end_pos
offset = self.parent_stmt.module.line_offset
return offset + self._end_pos[0], self._end_pos[1]
@end_pos.setter
def end_pos(self, value):
self._end_pos = value
def add_field(self, start_pos): def add_field(self, start_pos):
""" """
@@ -1087,11 +1125,17 @@ class NamePart(str):
A string. Sometimes it is important to know if the string belongs to a name A string. Sometimes it is important to know if the string belongs to a name
or not. or not.
""" """
def __new__(cls, s, start_pos): def __new__(cls, s, parent, start_pos):
self = super(NamePart, cls).__new__(cls, s) self = super(NamePart, cls).__new__(cls, s)
self.start_pos = start_pos self._start_pos = start_pos
self.parent = parent
return self return self
@property
def start_pos(self):
offset = self.parent.module.line_offset
return offset + self._start_pos[0], self._start_pos[1]
@property @property
def end_pos(self): def end_pos(self):
return self.start_pos[0], self.start_pos[1] + len(self) return self.start_pos[0], self.start_pos[1] + len(self)
@@ -1104,10 +1148,10 @@ class Name(Simple):
So a name like "module.class.function" So a name like "module.class.function"
would result in an array of [module, class, function] would result in an array of [module, class, function]
""" """
def __init__(self, names, start_pos, end_pos, parent=None): def __init__(self, module, names, start_pos, end_pos, parent=None):
super(Name, self).__init__(start_pos, end_pos) super(Name, self).__init__(module, start_pos, end_pos)
self.names = tuple(n if isinstance(n, NamePart) else NamePart(*n) self.names = tuple(n if isinstance(n, NamePart) else
for n in names) NamePart(n[0], self, n[1]) for n in names)
if parent is not None: if parent is not None:
self.parent = parent self.parent = parent
@@ -1251,7 +1295,8 @@ class PyFuzzyParser(object):
break break
append((tok, self.start_pos)) append((tok, self.start_pos))
n = Name(names, first_pos, self.end_pos) if names else None n = Name(self.module, names, first_pos, self.end_pos) if names \
else None
return n, token_type, tok return n, token_type, tok
def _parseimportlist(self): def _parseimportlist(self):
@@ -1339,7 +1384,8 @@ class PyFuzzyParser(object):
if token_type != tokenize.NAME: if token_type != tokenize.NAME:
return None return None
fname = Name([(fname, self.start_pos)], self.start_pos, self.end_pos) fname = Name(self.module, [(fname, self.start_pos)], self.start_pos,
self.end_pos)
token_type, open = self.next() token_type, open = self.next()
if open != '(': if open != '(':
@@ -1361,7 +1407,7 @@ class PyFuzzyParser(object):
return None return None
# because of 2 line func param definitions # because of 2 line func param definitions
scope = Function(fname, params, first_pos, annotation) scope = Function(self.module, fname, params, first_pos, annotation)
if self.user_scope and scope != self.user_scope \ if self.user_scope and scope != self.user_scope \
and self.user_position > first_pos: and self.user_position > first_pos:
self.user_scope = scope self.user_scope = scope
@@ -1382,7 +1428,8 @@ class PyFuzzyParser(object):
% (self.start_pos[0], tokenize.tok_name[token_type], cname)) % (self.start_pos[0], tokenize.tok_name[token_type], cname))
return None return None
cname = Name([(cname, self.start_pos)], self.start_pos, self.end_pos) cname = Name(self.module, [(cname, self.start_pos)], self.start_pos,
self.end_pos)
super = [] super = []
token_type, next = self.next() token_type, next = self.next()
@@ -1395,7 +1442,7 @@ class PyFuzzyParser(object):
return None return None
# because of 2 line class initializations # because of 2 line class initializations
scope = Class(cname, super, first_pos) scope = Class(self.module, cname, super, first_pos)
if self.user_scope and scope != self.user_scope \ if self.user_scope and scope != self.user_scope \
and self.user_position > first_pos: and self.user_position > first_pos:
self.user_scope = scope self.user_scope = scope
@@ -1528,7 +1575,7 @@ class PyFuzzyParser(object):
for t in toks: for t in toks:
src += t[1] if isinstance(t, tuple) \ src += t[1] if isinstance(t, tuple) \
else t.get_code() else t.get_code()
st = Statement(src, [], [], [], st = Statement(self.module, src, [], [], [],
toks, first_pos, self.end_pos) toks, first_pos, self.end_pos)
for s in [st, middle, in_clause]: for s in [st, middle, in_clause]:
@@ -1578,8 +1625,8 @@ class PyFuzzyParser(object):
self.scope.add_docstr(self.last_token[1]) self.scope.add_docstr(self.last_token[1])
return None, tok return None, tok
else: else:
stmt = stmt_class(string, set_vars, used_funcs, used_vars, stmt = stmt_class(self.module, string, set_vars, used_funcs,
tok_list, first_pos, self.end_pos) used_vars, tok_list, first_pos, self.end_pos)
self._check_user_stmt(stmt) self._check_user_stmt(stmt)
if is_return: if is_return:
# add returns to the scope # add returns to the scope
@@ -1692,12 +1739,13 @@ class PyFuzzyParser(object):
elif tok == 'import': elif tok == 'import':
imports = self._parseimportlist() imports = self._parseimportlist()
for m, alias, defunct in imports: for m, alias, defunct in imports:
i = Import(first_pos, self.end_pos, m, alias, i = Import(self.module, first_pos, self.end_pos, m, alias,
defunct=defunct) defunct=defunct)
self._check_user_stmt(i) self._check_user_stmt(i)
self.scope.add_import(i) self.scope.add_import(i)
if not imports: if not imports:
i = Import(first_pos, self.end_pos, None, defunct=True) i = Import(self.module, first_pos, self.end_pos, None,
defunct=True)
self._check_user_stmt(i) self._check_user_stmt(i)
self.freshscope = False self.freshscope = False
elif tok == 'from': elif tok == 'from':
@@ -1725,8 +1773,9 @@ class PyFuzzyParser(object):
star = name is not None and name.names[0] == '*' star = name is not None and name.names[0] == '*'
if star: if star:
name = None name = None
i = Import(first_pos, self.end_pos, name, alias, mod, i = Import(self.module, first_pos, self.end_pos, name,
star, relative_count, defunct=defunct or defunct2) alias, mod, star, relative_count,
defunct=defunct or defunct2)
self._check_user_stmt(i) self._check_user_stmt(i)
self.scope.add_import(i) self.scope.add_import(i)
self.freshscope = False self.freshscope = False
@@ -1737,7 +1786,7 @@ class PyFuzzyParser(object):
statement, tok = self._parse_statement() statement, tok = self._parse_statement()
if tok == ':': if tok == ':':
s = [] if statement is None else [statement] s = [] if statement is None else [statement]
f = ForFlow(s, first_pos, set_stmt) f = ForFlow(self.module, s, first_pos, set_stmt)
self.scope = self.scope.add_statement(f) self.scope = self.scope.add_statement(f)
else: else:
debug.warning('syntax err, for flow started @%s', debug.warning('syntax err, for flow started @%s',
@@ -1776,7 +1825,7 @@ class PyFuzzyParser(object):
first = False first = False
if tok == ':': if tok == ':':
f = Flow(command, inits, first_pos) f = Flow(self.module, command, inits, first_pos)
if command in extended_flow: if command in extended_flow:
# the last statement has to be another part of # the last statement has to be another part of
# the flow statement, because a dedent releases the # the flow statement, because a dedent releases the