diff --git a/jedi/evaluate.py b/jedi/evaluate.py index f0474552..b2894186 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -714,7 +714,8 @@ class Generator(use_metaclass(cache.CachedMetaClass, parsing.Base)): none_pos = (0, 0) executes_generator = ('__next__', 'send') for n in ('close', 'throw') + executes_generator: - name = parsing.Name([(n, none_pos)], none_pos, none_pos) + name = parsing.Name(builtin.Builtin.scope, [(n, none_pos)], + none_pos, none_pos) if n in executes_generator: name.parent = self names.append(name) @@ -1332,7 +1333,8 @@ def follow_call_list(call_list): if isinstance(nested_lc, parsing.ListComprehension): # is nested LC input = nested_lc.stmt - loop = parsing.ForFlow([input], lc.stmt.start_pos, + module = input.get_parent_until() + loop = parsing.ForFlow(module, [input], lc.stmt.start_pos, lc.middle, True) if parent is None: loop.parent = lc.stmt.parent diff --git a/jedi/fast_parser.py b/jedi/fast_parser.py index 678a817a..7f25a3e0 100644 --- a/jedi/fast_parser.py +++ b/jedi/fast_parser.py @@ -10,9 +10,10 @@ parser_cache = {} class Module(parsing.Simple, parsing.Module): def __init__(self, parsers): self._end_pos = None, None - super(Module, self).__init__((1,0)) + super(Module, self).__init__(self, (1,0)) self.parsers = parsers self.reset_caches() + self.line_offset = 0 def reset_caches(self): """ This module does a whole lot of caching, because it uses different @@ -105,6 +106,16 @@ class Module(parsing.Simple, parsing.Module): raise NotImplementedError("Parser doesn't exist.") return self.parsers[0].module.is_builtin + @property + def start_pos(self): + """ overwrite start_pos of Simple """ + return 1, 0 + + @start_pos.setter + def start_pos(self): + """ ignore """ + pass + @property def end_pos(self): return self._end_pos @@ -172,7 +183,7 @@ class FastParser(use_metaclass(CachedFastParser)): self.user_position = user_position self.reset_caches() self.parsers = [] - self.module.parsers = self.parsers + self.module = Module(self.parsers) self._parse(code) @@ -189,12 +200,19 @@ class FastParser(use_metaclass(CachedFastParser)): parts[0] += parts[1] parts.pop(1) + self.parsers = [] + self.module.parsers = self.parsers + hashes = {p.hash:p for p in self.parsers} + line_offset = 0 start = 0 p = None - for s in parts: - lines = s.count('\n') + for code_part in parts: + lines = code_part.count('\n') if p is None or line_offset >= p.end_pos[0] - 2: + h = hash(code_part) + if h in hashes: + print(h) p = parsing.PyFuzzyParser(code[start:], self.module_path, self.user_position, line_offset=line_offset, stop_on_scope=True, @@ -203,7 +221,7 @@ class FastParser(use_metaclass(CachedFastParser)): p.module.parent = self.module self.parsers.append(p) line_offset += lines - start += len(s) + start += len(code_part) def reset_caches(self): self._user_scope = None diff --git a/jedi/helpers.py b/jedi/helpers.py index 71ac098b..99075ac1 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -166,7 +166,8 @@ def fast_parent_copy(obj): setattr(new_obj, key, new_elements[value]) except KeyError: pass - elif key in ['parent_stmt', 'parent_function', 'set_parent']: + elif key in ['parent_stmt', 'parent_function', 'set_parent', + 'module']: continue elif isinstance(value, list): setattr(new_obj, key, list_rec(value)) @@ -230,7 +231,7 @@ def scan_array_for_pos(arr, pos): elif s.execution is not None: end = s.execution.end_pos if s.execution.start_pos < pos and \ - (end is None or pos < end): + (None in end or pos < end): c, index, stop = scan_array_for_pos( s.execution, pos) if stop: diff --git a/jedi/imports.py b/jedi/imports.py index 9b277c34..501d6790 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -29,16 +29,16 @@ class ImportPath(parsing.Base): An ImportPath is the path of a `parsing.Import` object. """ class _GlobalNamespace(object): + def __init__(self): + self.start_pos = 0, 0 + self.line_offset = 0 + def get_defined_names(self): return [] def get_imports(self): return [] - @property - def start_pos(self): - return (0, 0) - def get_parent_until(self): return None @@ -88,8 +88,9 @@ class ImportPath(parsing.Base): # This is not an existing Import statement. Therefore, set position to # 0 (0 is not a valid line number). zero = (0, 0) - n = parsing.Name(i.namespace.names[1:], zero, zero, self.import_stmt) - new = parsing.Import(zero, zero, n) + names = i.namespace.names[1:] + n = parsing.Name(i.module, names, zero, zero, self.import_stmt) + new = parsing.Import(i.module, zero, zero, n) new.parent = parent debug.dbg('Generated a nested import: %s' % new) return new @@ -135,8 +136,8 @@ class ImportPath(parsing.Base): names = [] for module_loader, name, is_pkg in pkgutil.iter_modules(search_path): inf_pos = (float('inf'), float('inf')) - names.append(parsing.Name([(name, inf_pos)], inf_pos, inf_pos, - self.import_stmt)) + names.append(parsing.Name(self.GlobalNamespace, [(name, inf_pos)], + inf_pos, inf_pos, self.import_stmt)) return names def sys_path_with_modifications(self): diff --git a/jedi/parsing.py b/jedi/parsing.py index 02315ec3..e239b174 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -63,12 +63,13 @@ class Simple(Base): The super class for Scope, Import, Name and Statement. Every object in the parser tree inherits from this class. """ - def __init__(self, start_pos, end_pos=(None, None)): - self.start_pos = start_pos - self.end_pos = end_pos + def __init__(self, module, start_pos, end_pos=(None, None)): + self._start_pos = start_pos + self._end_pos = end_pos self.parent = None # use this attribute if parent should be something else than self. self.set_parent = self + self.module = module @Python3Method def get_parent_until(self, classes=(), reverse=False, @@ -85,6 +86,24 @@ class Simple(Base): scope = scope.parent return scope + @property + def start_pos(self): + return self.module.line_offset + self._start_pos[0], self._start_pos[1] + + @start_pos.setter + def start_pos(self, value): + self._start_pos = value + + @property + def end_pos(self): + if None in self._end_pos: + return self._end_pos + return self.module.line_offset + self._end_pos[0], self._end_pos[1] + + @end_pos.setter + def end_pos(self, value): + self._end_pos = value + def __repr__(self): code = self.get_code().replace('\n', ' ') return "<%s: %s@%s>" % \ @@ -104,12 +123,12 @@ class Scope(Simple): :param docstr: The docstring for the current Scope. :type docstr: str """ - def __init__(self, start_pos, docstr=''): - super(Scope, self).__init__(start_pos) + def __init__(self, module, start_pos): + super(Scope, self).__init__(module, start_pos) self.subscopes = [] self.imports = [] self.statements = [] - self.docstr = docstr + self.docstr = '' self.asserts = [] def add_scope(self, sub, decorators): @@ -248,14 +267,14 @@ class SubModule(Scope, Module): of a module. """ def __init__(self, path, start_pos, top_module=None): - super(SubModule, self).__init__(start_pos) + super(SubModule, self).__init__(self, start_pos) self.path = path self.global_vars = [] self._name = None self.used_names = {} self.temp_used_names = [] # this may be changed depending on fast_parser - self._line_offset = 0 + self.line_offset = 0 self.set_parent = top_module or self @@ -289,7 +308,8 @@ class SubModule(Scope, Module): self.path) string = r.group(1) names = [(string, (0, 0))] - self._name = Name(names, self.start_pos, self.end_pos, self.set_parent) + self._name = Name(self, names, self.start_pos, self.end_pos, + self.set_parent) return self._name def is_builtin(self): @@ -306,11 +326,9 @@ class Class(Scope): :type supers: list :param start_pos: The start position (line, column) of the class. :type start_pos: tuple(int, int) - :param docstr: The docstring for the current Scope. - :type docstr: str """ - def __init__(self, name, supers, start_pos, docstr=''): - super(Class, self).__init__(start_pos, docstr) + def __init__(self, module, name, supers, start_pos): + super(Class, self).__init__(module, start_pos) self.name = name name.parent = self.set_parent self.supers = supers @@ -344,8 +362,8 @@ class Function(Scope): :param docstr: The docstring for the current Scope. :type docstr: str """ - def __init__(self, name, params, start_pos, annotation): - Scope.__init__(self, start_pos) + def __init__(self, module, name, params, start_pos, annotation): + super(Function, self).__init__(module, start_pos) self.name = name name.parent = self.set_parent self.params = params @@ -433,10 +451,10 @@ class Flow(Scope): :param set_vars: Local variables used in the for loop (only there). :type set_vars: list """ - def __init__(self, command, inits, start_pos, set_vars=None): + def __init__(self, module, command, inits, start_pos, set_vars=None): self.next = None self.command = command - super(Flow, self).__init__(start_pos, '') + super(Flow, self).__init__(module, start_pos) self._parent = None # These have to be statements, because of with, which takes multiple. self.inits = inits @@ -510,8 +528,8 @@ class ForFlow(Flow): """ Used for the for loop, because there are two statement parts. """ - def __init__(self, inits, start_pos, set_stmt, is_list_comp=False): - super(ForFlow, self).__init__('for', inits, start_pos, + def __init__(self, module, inits, start_pos, set_stmt, is_list_comp=False): + super(ForFlow, self).__init__(module, 'for', inits, start_pos, set_stmt.used_vars) self.set_stmt = set_stmt self.is_list_comp = is_list_comp @@ -546,9 +564,9 @@ class Import(Simple): :param defunct: An Import is valid or not. :type defunct: bool """ - def __init__(self, start_pos, end_pos, namespace, alias=None, + def __init__(self, module, start_pos, end_pos, namespace, alias=None, from_ns=None, star=False, relative_count=0, defunct=False): - super(Import, self).__init__(start_pos, end_pos) + super(Import, self).__init__(module, start_pos, end_pos) self.namespace = namespace self.alias = alias @@ -590,8 +608,8 @@ class Import(Simple): return [self.alias] if len(self.namespace) > 1: o = self.namespace - n = Name([(o.names[0], o.start_pos)], o.start_pos, o.end_pos, - parent=o.parent) + n = Name(self.module, [(o.names[0], o.start_pos)], o.start_pos, + o.end_pos, parent=o.parent) return [n] else: return [self.namespace] @@ -630,9 +648,9 @@ class Statement(Simple): :param start_pos: Position (line, column) of the Statement. :type start_pos: tuple(int, int) """ - def __init__(self, code, set_vars, used_funcs, used_vars, token_list, - start_pos, end_pos): - super(Statement, self).__init__(start_pos, end_pos) + def __init__(self, module, code, set_vars, used_funcs, used_vars, + token_list, start_pos, end_pos): + super(Statement, self).__init__(module, start_pos, end_pos) self.code = code self.used_funcs = used_funcs self.used_vars = used_vars @@ -845,10 +863,10 @@ class Param(Statement): The class which shows definitions of params of classes and functions. But this is not to define function calls. """ - def __init__(self, code, set_vars, used_funcs, used_vars, + def __init__(self, module, code, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos): - super(Param, self).__init__(code, set_vars, used_funcs, used_vars, - token_list, start_pos, end_pos) + super(Param, self).__init__(module, code, set_vars, used_funcs, + used_vars, token_list, start_pos, end_pos) # this is defined by the parser later on, not at the initialization # it is the position in the call (first argument, second...) @@ -890,6 +908,15 @@ class Call(Base): self.execution = None self._parent_stmt = parent_stmt + @property + def start_pos(self): + offset = self.parent_stmt.module.line_offset + return offset + self._start_pos[0], self._start_pos[1] + + @start_pos.setter + def start_pos(self, value): + self._start_pos = value + @property def parent_stmt(self): if self._parent_stmt is not None: @@ -979,7 +1006,18 @@ class Array(Call): self.values = values if values else [] self.arr_el_pos = [] self.keys = [] - self.end_pos = None + self._end_pos = None, None + + @property + def end_pos(self): + if None in self._end_pos: + return self._end_pos + offset = self.parent_stmt.module.line_offset + return offset + self._end_pos[0], self._end_pos[1] + + @end_pos.setter + def end_pos(self, value): + self._end_pos = value def add_field(self, start_pos): """ @@ -1087,11 +1125,17 @@ class NamePart(str): A string. Sometimes it is important to know if the string belongs to a name or not. """ - def __new__(cls, s, start_pos): + def __new__(cls, s, parent, start_pos): self = super(NamePart, cls).__new__(cls, s) - self.start_pos = start_pos + self._start_pos = start_pos + self.parent = parent return self + @property + def start_pos(self): + offset = self.parent.module.line_offset + return offset + self._start_pos[0], self._start_pos[1] + @property def end_pos(self): return self.start_pos[0], self.start_pos[1] + len(self) @@ -1104,10 +1148,10 @@ class Name(Simple): So a name like "module.class.function" would result in an array of [module, class, function] """ - def __init__(self, names, start_pos, end_pos, parent=None): - super(Name, self).__init__(start_pos, end_pos) - self.names = tuple(n if isinstance(n, NamePart) else NamePart(*n) - for n in names) + def __init__(self, module, names, start_pos, end_pos, parent=None): + super(Name, self).__init__(module, start_pos, end_pos) + self.names = tuple(n if isinstance(n, NamePart) else + NamePart(n[0], self, n[1]) for n in names) if parent is not None: self.parent = parent @@ -1251,7 +1295,8 @@ class PyFuzzyParser(object): break append((tok, self.start_pos)) - n = Name(names, first_pos, self.end_pos) if names else None + n = Name(self.module, names, first_pos, self.end_pos) if names \ + else None return n, token_type, tok def _parseimportlist(self): @@ -1339,7 +1384,8 @@ class PyFuzzyParser(object): if token_type != tokenize.NAME: return None - fname = Name([(fname, self.start_pos)], self.start_pos, self.end_pos) + fname = Name(self.module, [(fname, self.start_pos)], self.start_pos, + self.end_pos) token_type, open = self.next() if open != '(': @@ -1361,7 +1407,7 @@ class PyFuzzyParser(object): return None # because of 2 line func param definitions - scope = Function(fname, params, first_pos, annotation) + scope = Function(self.module, fname, params, first_pos, annotation) if self.user_scope and scope != self.user_scope \ and self.user_position > first_pos: self.user_scope = scope @@ -1382,7 +1428,8 @@ class PyFuzzyParser(object): % (self.start_pos[0], tokenize.tok_name[token_type], cname)) return None - cname = Name([(cname, self.start_pos)], self.start_pos, self.end_pos) + cname = Name(self.module, [(cname, self.start_pos)], self.start_pos, + self.end_pos) super = [] token_type, next = self.next() @@ -1395,7 +1442,7 @@ class PyFuzzyParser(object): return None # because of 2 line class initializations - scope = Class(cname, super, first_pos) + scope = Class(self.module, cname, super, first_pos) if self.user_scope and scope != self.user_scope \ and self.user_position > first_pos: self.user_scope = scope @@ -1528,7 +1575,7 @@ class PyFuzzyParser(object): for t in toks: src += t[1] if isinstance(t, tuple) \ else t.get_code() - st = Statement(src, [], [], [], + st = Statement(self.module, src, [], [], [], toks, first_pos, self.end_pos) for s in [st, middle, in_clause]: @@ -1578,8 +1625,8 @@ class PyFuzzyParser(object): self.scope.add_docstr(self.last_token[1]) return None, tok else: - stmt = stmt_class(string, set_vars, used_funcs, used_vars, - tok_list, first_pos, self.end_pos) + stmt = stmt_class(self.module, string, set_vars, used_funcs, + used_vars, tok_list, first_pos, self.end_pos) self._check_user_stmt(stmt) if is_return: # add returns to the scope @@ -1692,12 +1739,13 @@ class PyFuzzyParser(object): elif tok == 'import': imports = self._parseimportlist() for m, alias, defunct in imports: - i = Import(first_pos, self.end_pos, m, alias, + i = Import(self.module, first_pos, self.end_pos, m, alias, defunct=defunct) self._check_user_stmt(i) self.scope.add_import(i) if not imports: - i = Import(first_pos, self.end_pos, None, defunct=True) + i = Import(self.module, first_pos, self.end_pos, None, + defunct=True) self._check_user_stmt(i) self.freshscope = False elif tok == 'from': @@ -1725,8 +1773,9 @@ class PyFuzzyParser(object): star = name is not None and name.names[0] == '*' if star: name = None - i = Import(first_pos, self.end_pos, name, alias, mod, - star, relative_count, defunct=defunct or defunct2) + i = Import(self.module, first_pos, self.end_pos, name, + alias, mod, star, relative_count, + defunct=defunct or defunct2) self._check_user_stmt(i) self.scope.add_import(i) self.freshscope = False @@ -1737,7 +1786,7 @@ class PyFuzzyParser(object): statement, tok = self._parse_statement() if tok == ':': s = [] if statement is None else [statement] - f = ForFlow(s, first_pos, set_stmt) + f = ForFlow(self.module, s, first_pos, set_stmt) self.scope = self.scope.add_statement(f) else: debug.warning('syntax err, for flow started @%s', @@ -1776,7 +1825,7 @@ class PyFuzzyParser(object): first = False if tok == ':': - f = Flow(command, inits, first_pos) + f = Flow(self.module, command, inits, first_pos) if command in extended_flow: # the last statement has to be another part of # the flow statement, because a dedent releases the