diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 83f3ad77..831353b5 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -167,8 +167,8 @@ class Script(object): # Allow access on _definition here, because it's a # public API and we don't want to make the internal # Name object public. - if p._name.get_definition().stars == 0: # no *args/**kwargs - completions.append((p._name.parent, p)) + if p._definition.stars == 0: # no *args/**kwargs + completions.append((p._name, p._name)) if not path and not isinstance(user_stmt, pr.Import): # add keywords @@ -179,17 +179,15 @@ class Script(object): comps = [] comp_dct = {} for c, s in set(completions): - # TODO Remove this line. c should be a namepart even before that. - c = c.names[-1] n = str(c) if settings.case_insensitive_completion \ and n.lower().startswith(like.lower()) \ or n.startswith(like): if not filter_private_variable(s, user_stmt or self._parser.user_scope(), n): - if isinstance(c.parent.parent, (pr.Function, pr.Class)): + if isinstance(c.parent, (pr.Function, pr.Class)): # TODO I think this is a hack. It should be an # er.Function/er.Class before that. - c = er.wrap(self._evaluator, c.parent.parent).name.names[-1] + c = er.wrap(self._evaluator, c.parent).name new = classes.Completion(self._evaluator, c, needs_dot, len(like), s) k = (new.name, new.complete) # key if k in comp_dct and settings.no_completion_duplicates: @@ -395,10 +393,9 @@ class Script(object): definitions = set(self._prepare_goto(goto_path)) definitions = resolve_import_paths(definitions) - names = [s if isinstance(s, pr.Name) else s.name for s in definitions + names = [s.name for s in definitions if s is not imports.ImportWrapper.GlobalNamespace] - defs = [classes.Definition(self._evaluator, name.names[-1]) - for name in names] + defs = [classes.Definition(self._evaluator, name) for name in names] return helpers.sorted_definitions(set(defs)) def goto_assignments(self): @@ -432,7 +429,7 @@ class Script(object): and d.start_pos == (0, 0): i = imports.ImportWrapper(self._evaluator, d.parent).follow(is_goto=True) definitions.remove(d) - definitions |= follow_inexistent_imports(i.names[-1]) + definitions |= follow_inexistent_imports(i) return definitions goto_path = self._user_context.get_path_under_cursor() @@ -455,7 +452,7 @@ class Script(object): if next(context) in ('class', 'def'): # The cursor is on a class/function name. user_scope = self._parser.user_scope() - definitions = set([user_scope.name.names[-1]]) + definitions = set([user_scope.name]) elif isinstance(user_stmt, pr.Import): s, name_part = helpers.get_on_import_stmt(self._evaluator, self._user_context, user_stmt) @@ -467,7 +464,7 @@ class Script(object): if add_import_name: import_name = user_stmt.get_defined_names() # imports have only one name - np = import_name[0].names[-1] + np = import_name[0] if not user_stmt.star and unicode(name_part) == unicode(np): definitions.append(np) else: @@ -477,8 +474,9 @@ class Script(object): if isinstance(user_stmt, pr.ExprStmt): for name in user_stmt.get_defined_names(): if name.start_pos <= self._pos <= name.end_pos \ - and len(name.names) == 1: - return [name.names[0]] + and (not isinstance(name.parent, pr.Call) + or name.parent.next is None): + return [name] defs = self._evaluator.goto(stmt, call_path) definitions = follow_inexistent_imports(defs) @@ -504,16 +502,6 @@ class Script(object): # Without a definition for a name we cannot find references. return [] - # Once Script._goto works correct, we can probably remove this - # branch. - if isinstance(user_stmt, pr.ExprStmt): - c = user_stmt.expression_list()[0] - if not isinstance(c, unicode) and self._pos < c.start_pos: - # The lookup might be before `=` - definitions = [v.names[-1] for v in user_stmt.get_defined_names() - if unicode(v.names[-1]) == - list(definitions)[0].get_code()] - if not isinstance(user_stmt, pr.Import): # import case is looked at with add_import_name option definitions = usages.usages_add_import_modules(self._evaluator, @@ -572,7 +560,7 @@ class Script(object): key_name = unicode(detail[0][0].name) except (IndexError, AttributeError): pass - return [classes.CallSignature(self._evaluator, o.name.names[-1], call, index, key_name) + return [classes.CallSignature(self._evaluator, o.name, call, index, key_name) for o in origins if hasattr(o, 'py__call__')] def _analysis(self): @@ -583,7 +571,7 @@ class Script(object): iw = imports.ImportWrapper(self._evaluator, i, nested_resolve=True).follow() if i.is_nested() and any(not isinstance(i, pr.Module) for i in iw): - analysis.add(self._evaluator, 'import-error', i.namespace.names[-1]) + analysis.add(self._evaluator, 'import-error', i.namespace_names[-1]) for stmt in sorted(stmts, key=lambda obj: obj.start_pos): if not (isinstance(stmt.parent, pr.ForFlow) and stmt.parent.set_stmt == stmt): diff --git a/jedi/api/classes.py b/jedi/api/classes.py index e5007d45..68ddd424 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -36,8 +36,7 @@ def defined_names(evaluator, scope): pair = next(get_names_of_scope(evaluator, scope, star_search=False, include_builtin=False), None) names = pair[1] if pair else [] - names = [n for n in names if isinstance(n, pr.Import) or (len(n) == 1)] - return [Definition(evaluator, d.names[-1]) for d in sorted(names, key=lambda s: s.start_pos)] + return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)] class BaseDefinition(object): @@ -168,16 +167,11 @@ class BaseDefinition(object): def _path(self): """The module path.""" path = [] - - def insert_nonnone(x): - if x: - path.insert(0, x) - par = self._definition while par is not None: if isinstance(par, pr.Import): - insert_nonnone(par.namespace) - insert_nonnone(par.from_ns) + path += par.from_names + path += par.namespace_names if par.relative_count == 0: break with common.ignored(AttributeError): @@ -376,12 +370,12 @@ class BaseDefinition(object): params = sub.params[1:] # ignore self except KeyError: return [] - return [_Param(self._evaluator, p.get_name().names[-1]) for p in params] + return [_Param(self._evaluator, p.get_name()) for p in params] def parent(self): scope = self._definition.get_parent_scope() non_flow = scope.get_parent_until(pr.Flow, reverse=True) - return Definition(self._evaluator, non_flow.name.names[-1]) + return Definition(self._evaluator, non_flow.name) def __repr__(self): return "<%s %s>" % (type(self).__name__, self.description) @@ -539,7 +533,7 @@ class Completion(BaseDefinition): it's just PITA-slow. """ defs = self._follow_statements_imports() - return [Definition(self._evaluator, d.name.names[-1]) for d in defs] + return [Definition(self._evaluator, d.name) for d in defs] class Definition(use_metaclass(CachedMetaClass, BaseDefinition)): diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 407ae738..dba84dc8 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -28,14 +28,14 @@ def get_on_import_stmt(evaluator, user_context, user_stmt, is_like_search=False) import_names = user_stmt.get_all_import_names() kill_count = -1 cur_name_part = None - for i in import_names: - if user_stmt.alias == i: + for name in import_names: + if user_stmt.alias == name: continue - for name_part in i.names: - if name_part.end_pos >= user_context.position: - if not cur_name_part: - cur_name_part = name_part - kill_count += 1 + + if name.end_pos >= user_context.position: + if not cur_name_part: + cur_name_part = name + kill_count += 1 context = user_context.get_context() just_from = next(context) == 'from' diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index 318dc7a7..6b4f68b6 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -48,8 +48,7 @@ class LazyName(helpers.FakeName): module = obj else: class FakeParent(pr.Base): - parent = None # To avoid having no parent for NamePart. - path = None + parent = compiled.builtin names = [] try: @@ -67,8 +66,7 @@ class LazyName(helpers.FakeName): module = builtins else: module = __import__(module_name) - fake_name = helpers.FakeName(names, FakeParent()) - parser_path = fake_name.names + parser_path = [helpers.FakeName(n, FakeParent()) for n in names] raw_module = get_module(self._value) try: diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index e5f63836..2a54ba2d 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -1,7 +1,6 @@ import pydoc import keyword -from jedi.parser.representation import NamePart from jedi._compatibility import is_py3 from jedi import common from jedi.evaluate import compiled diff --git a/jedi/api/usages.py b/jedi/api/usages.py index e6684299..ec0a6c0b 100644 --- a/jedi/api/usages.py +++ b/jedi/api/usages.py @@ -8,7 +8,7 @@ from jedi.evaluate import helpers def usages(evaluator, definitions, mods): """ - :param definitions: list of NameParts + :param definitions: list of Name """ def compare_array(definitions): """ `definitions` are being compared by module/start_pos, because @@ -25,8 +25,7 @@ def usages(evaluator, definitions, mods): while not stmt.parent.is_scope(): stmt = stmt.parent # New definition, call cannot be a part of stmt - if len(call.name) == 1 and call.next is None \ - and call.name in stmt.get_defined_names(): + if call.next is None and call.name in stmt.get_defined_names(): # Class params are not definitions (like function params). They # are super classes, that need to be resolved. if not (isinstance(stmt, pr.Param) and isinstance(stmt.parent, pr.Class)): @@ -35,7 +34,7 @@ def usages(evaluator, definitions, mods): follow = [] # There might be multiple search_name's in one call_path call_path = list(call.generate_call_path()) for i, name in enumerate(call_path): - # name is `pr.NamePart`. + # name is `pr.Name`. if u(name) == search_name: follow.append(call_path[:i + 1]) @@ -70,11 +69,10 @@ def usages(evaluator, definitions, mods): if isinstance(stmt, pr.Import): count = 0 imps = [] - for i in stmt.get_all_import_names(): - for name_part in i.names: - count += 1 - if unicode(name_part) == search_name: - imps.append((count, name_part)) + for name in stmt.get_all_import_names(): + count += 1 + if unicode(name) == search_name: + imps.append((count, name)) for used_count, name_part in imps: i = imports.ImportWrapper(evaluator, stmt, kill_count=count - used_count, diff --git a/jedi/cache.py b/jedi/cache.py index dae5c085..088f7cdd 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -243,7 +243,7 @@ def save_parser(path, name, parser, pickling=True): class ParserPickling(object): - version = 17 + version = 18 """ Version number (integer) for file system cache. diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 1d1a8bed..3655ef47 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -69,7 +69,7 @@ backtracking algorithm. .. todo:: nonlocal statement, needed or can be ignored? (py3k) """ import copy -import itertools +from itertools import tee, chain from jedi._compatibility import next, hasattr, unicode from jedi.parser import representation as pr @@ -172,20 +172,7 @@ class Evaluator(object): return precedence.process_precedence_element(self, p) or [] def eval_statement_element(self, element): - if pr.Array.is_type(element, pr.Array.NOARRAY): - try: - lst_cmp = element[0].expression_list()[0] - if not isinstance(lst_cmp, pr.ListComprehension): - raise IndexError - except IndexError: - r = list(itertools.chain.from_iterable(self.eval_statement(s) - for s in element)) - else: - r = [iterable.GeneratorComprehension(self, lst_cmp)] - call_path = element.generate_call_path() - next(call_path, None) # the first one has been used already - return self.follow_path(call_path, r, element.parent) - elif isinstance(element, pr.ListComprehension): + if isinstance(element, pr.ListComprehension): return self.eval_statement(element.stmt) elif isinstance(element, pr.Lambda): return [er.Function(self, element)] @@ -219,9 +206,20 @@ class Evaluator(object): current = next(path) if isinstance(current, pr.Array): - types = [iterable.Array(self, current)] + if current.type == pr.Array.NOARRAY: + try: + lst_cmp = current[0].expression_list()[0] + if not isinstance(lst_cmp, pr.ListComprehension): + raise IndexError + except IndexError: + types = list(chain.from_iterable(self.eval_statement(s) + for s in current)) + else: + types = [iterable.GeneratorComprehension(self, lst_cmp)] + else: + types = [iterable.Array(self, current)] else: - if isinstance(current, pr.NamePart): + if isinstance(current, pr.Name): # This is the first global lookup. types = self.find_types(scope, current, position=position, search_global=True) @@ -241,7 +239,7 @@ class Evaluator(object): to follow a call like ``module.a_type.Foo.bar`` (in ``from_somewhere``). """ results_new = [] - iter_paths = itertools.tee(path, len(types)) + iter_paths = tee(path, len(types)) for i, typ in enumerate(types): fp = self._follow_path(iter_paths[i], typ, call_scope) @@ -320,12 +318,26 @@ class Evaluator(object): return types def goto(self, stmt, call_path): + if isinstance(stmt, pr.Import): + # Nowhere to goto for aliases + if stmt.alias == call_path[0]: + return [call_path[0]] + + names = stmt.get_all_import_names() + if stmt.alias: + names = names[:-1] + # Filter names that are after our Name + removed_names = len(names) - names.index(call_path[0]) - 1 + i = imports.ImportWrapper(self, stmt, kill_count=removed_names, + nested_resolve=True) + return i.follow(is_goto=True) + # Return the name defined in the call_path, if it's part of the # statement name definitions. Only return, if it's one name and one # name only. Otherwise it's a mixture between a definition and a # reference. In this case it's just a definition. So we stay on it. - if len(call_path) == 1 and isinstance(call_path[0], pr.NamePart) \ - and call_path[0] in [d.names[-1] for d in stmt.get_defined_names()]: + if len(call_path) == 1 and isinstance(call_path[0], pr.Name) \ + and call_path[0] in stmt.get_defined_names(): # Named params should get resolved to their param definitions. if pr.Array.is_type(stmt.parent, pr.Array.TUPLE, pr.Array.NOARRAY) \ and stmt.parent.previous: @@ -337,9 +349,15 @@ class Evaluator(object): param_names = [] named_param_name = stmt.get_defined_names()[0] for typ in self.eval_call(call): - for param in typ.params: + if isinstance(typ, er.Class): + params = [] + for init_method in typ.py__getattribute__('__init__'): + params += init_method.params + else: + params = typ.params + for param in params: if unicode(param.get_name()) == unicode(named_param_name): - param_names.append(param.get_name().names[-1]) + param_names.append(param.get_name()) return param_names return [call_path[0]] @@ -364,7 +382,7 @@ class Evaluator(object): def filter_private_variable(scope, call_scope, var_name): """private variables begin with a double underline `__`""" - var_name = str(var_name) # var_name could be a NamePart + var_name = str(var_name) # var_name could be a Name if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): s = call_scope.get_parent_until((pr.Class, er.Instance, compiled.CompiledObject)) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 05557eb8..be838150 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -240,7 +240,6 @@ class CompiledName(FakeName): super(CompiledName, self).__init__(name) self._obj = obj self.name = name - self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. def __repr__(self): try: diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index aa53a8d3..52a4d3f3 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -87,7 +87,7 @@ def search_params(evaluator, param): # Need to take right index, because there could be a # func usage before. - call_path_simple = [unicode(d) if isinstance(d, pr.NamePart) + call_path_simple = [unicode(d) if isinstance(d, pr.Name) else d for d in call_path] i = listRightIndex(call_path_simple, func_name) before, after = call_path[:i], call_path[i + 1:] @@ -121,7 +121,7 @@ def search_params(evaluator, param): for params in get_posibilities(evaluator, module, func_name): for p in params: if str(p) == param_name: - result += evaluator.eval_statement(p.parent) + result += evaluator.eval_statement(p.get_definition()) return result func = param.get_parent_until(pr.Function) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 842809fc..54b74302 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -42,7 +42,7 @@ class NameFinder(object): types = self._names_to_types(names, resolve_decorator) if not names and not types \ - and not (isinstance(self.name_str, pr.NamePart) + and not (isinstance(self.name_str, pr.Name) and isinstance(self.name_str.parent.parent, pr.Param)): if not isinstance(self.name_str, (str, unicode)): # TODO Remove? if search_global: @@ -102,18 +102,18 @@ class NameFinder(object): or isinstance(scope, compiled.CompiledObject) \ or isinstance(stmt, pr.ExprStmt) and stmt.is_global(): # Always reachable. - names.append(name.names[-1]) + names.append(name) else: check = flow_analysis.break_check(self._evaluator, name_list_scope, er.wrap(self._evaluator, scope), self.scope) if check is not flow_analysis.UNREACHABLE: - names.append(name.names[-1]) + names.append(name) if check is flow_analysis.REACHABLE: break - if names and self._is_name_break_scope(name, stmt): + if names and self._is_name_break_scope(stmt): if self._does_scope_break_immediately(scope, name_list_scope): break else: @@ -139,16 +139,16 @@ class NameFinder(object): evaluation, so remove them already here! """ for n in names: - definition = n.parent.parent + definition = n.parent if isinstance(definition, (pr.Function, pr.Class, pr.Module)): - yield er.wrap(self._evaluator, definition).name.names[-1] + yield er.wrap(self._evaluator, definition).name else: yield n def _check_getattr(self, inst): """Checks for both __getattr__ and __getattribute__ methods""" result = [] - # str is important to lose the NamePart! + # str is important, because it shouldn't be `Name`! name = compiled.create(self._evaluator, str(self.name_str)) with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', [name]) @@ -161,12 +161,12 @@ class NameFinder(object): result = inst.execute_subscope_by_name('__getattribute__', [name]) return result - def _is_name_break_scope(self, name, stmt): + def _is_name_break_scope(self, stmt): """ Returns True except for nested imports and instance variables. """ if stmt.isinstance(pr.ExprStmt): - if isinstance(name, er.InstanceElement) and not name.is_class_var: + if isinstance(stmt, er.InstanceElement) and not stmt.is_class_var: return False elif isinstance(stmt, pr.Import) and stmt.is_nested(): return False @@ -219,7 +219,7 @@ class NameFinder(object): evaluator = self._evaluator # Add isinstance and other if/assert knowledge. - if isinstance(self.name_str, pr.NamePart): + if isinstance(self.name_str, pr.Name): flow_scope = self.name_str.parent.parent # Ignore FunctionExecution parents for now. until = flow_scope.get_parent_until(er.FunctionExecution) @@ -281,7 +281,7 @@ class NameFinder(object): if isinstance(p, pr.Flow) and p.command == 'except' and p.inputs: as_names = p.inputs[0].as_names try: - if as_names[0].names[-1] == name: + if as_names[0] == name: # TODO check for types that are not classes and add it to # the static analysis report. types = list(chain.from_iterable( @@ -395,7 +395,7 @@ def check_flow_information(evaluator, flow, search_name_part, pos): return result -def _check_isinstance_type(evaluator, stmt, search_name_part): +def _check_isinstance_type(evaluator, stmt, search_name): try: expression_list = stmt.expression_list() # this might be removed if we analyze and, etc @@ -412,8 +412,12 @@ def _check_isinstance_type(evaluator, stmt, search_name_part): assert len(classes) == 1 assert isinstance(obj[0], pr.Call) - # names fit? - assert unicode(obj[0].name) == unicode(search_name_part.parent) + prev = search_name.parent + while prev.previous is not None: + prev = prev.previous + # Do a simple get_code comparison. They should just have the same code, + # and everything will be all right. + assert obj[0].get_code() == prev.get_code() assert isinstance(classes[0], pr.StatementElement) # can be type or tuple except AssertionError: return [] @@ -583,7 +587,7 @@ def find_assignments(lhs, results, seek_name): """ if isinstance(lhs, pr.Array): return _assign_tuples(lhs, results, seek_name) - elif unicode(lhs.name.names[-1]) == seek_name: + elif unicode(lhs.name) == seek_name: return results else: return [] diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 29eb19b2..f8f6b148 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -1,6 +1,7 @@ import copy from itertools import chain +from jedi._compatibility import unicode from jedi.parser import representation as pr from jedi import debug @@ -14,7 +15,7 @@ def deep_ast_copy(obj, new_elements_default=None): return key_value[0] not in ('_expression_list', '_assignment_details') new_elements = new_elements_default or {} - accept = (pr.Simple, pr.NamePart, pr.KeywordStatement) + accept = (pr.Simple, pr.Name, pr.KeywordStatement) def recursion(obj): # If it's already in the cache, just return it. @@ -50,6 +51,8 @@ def deep_ast_copy(obj, new_elements_default=None): # because there are several references that don't walk the whole # tree in there. items = sorted(items, key=sort_stmt) + else: + items = sorted(items, key=lambda x: x[0] == '_names_dict') # Actually copy and set attributes. new_obj = copy.copy(obj) @@ -67,13 +70,17 @@ def deep_ast_copy(obj, new_elements_default=None): pass elif key in ['parent_function', 'use_as_parent', '_sub_module']: continue + elif key == '_names_dict': + d = dict((k, sequence_recursion(v)) for k, v in value.items()) + setattr(new_obj, key, d) elif isinstance(value, (list, tuple)): - setattr(new_obj, key, list_or_tuple_rec(value)) + setattr(new_obj, key, sequence_recursion(value)) elif isinstance(value, accept): setattr(new_obj, key, recursion(value)) + return new_obj - def list_or_tuple_rec(array_obj): + def sequence_recursion(array_obj): if isinstance(array_obj, tuple): copied_array = list(array_obj) else: @@ -82,7 +89,7 @@ def deep_ast_copy(obj, new_elements_default=None): if isinstance(el, accept): copied_array[i] = recursion(el) elif isinstance(el, (tuple, list)): - copied_array[i] = list_or_tuple_rec(el) + copied_array[i] = sequence_recursion(el) if isinstance(array_obj, tuple): return tuple(copied_array) @@ -196,9 +203,7 @@ def scan_statement_for_calls(stmt, search_name, assignment_details=False): if isinstance(s_new, pr.Array): result += scan_array(s_new, search_name) else: - n = s_new.name - if isinstance(n, pr.Name) \ - and search_name in [str(x) for x in n.names]: + if search_name == unicode(s_new.name): result.append(c) s_new = s_new.next @@ -217,7 +222,7 @@ def get_module_name_parts(module): def scope_name_parts(scope): for s in scope.subscopes: # Yield the name parts, not names. - yield s.name.names[0] + yield s.name for need_yield_from in scope_name_parts(s): yield need_yield_from @@ -226,7 +231,7 @@ def get_module_name_parts(module): for stmt_or_import in statements_or_imports: if isinstance(stmt_or_import, pr.Import): for name in stmt_or_import.get_all_import_names(): - name_parts.update(name.names) + name_parts.add(name) else: # Running this ensures that all the expression lists are generated # and the parents are all set. (Important for Lambdas) Howeer, this @@ -238,7 +243,7 @@ def get_module_name_parts(module): # all the name_parts. for tok in stmt_or_import._token_list: if isinstance(tok, pr.Name): - name_parts.update(tok.names) + name_parts.add(tok) return name_parts @@ -298,18 +303,14 @@ class FakeStatement(pr.ExprStmt): class FakeImport(pr.Import): def __init__(self, name, parent, level=0): p = 0, 0 - super(FakeImport, self).__init__(FakeSubModule, p, p, name, + super(FakeImport, self).__init__(FakeSubModule, p, p, [name], relative_count=level) self.parent = parent class FakeName(pr.Name): - def __init__(self, name_or_names, parent=None, start_pos=(0, 0)): - if isinstance(name_or_names, list): - names = [(n, start_pos) for n in name_or_names] - else: - names = [(name_or_names, start_pos)] - super(FakeName, self).__init__(FakeSubModule, names, start_pos, start_pos, parent) + def __init__(self, name_str, parent=None, start_pos=(0, 0)): + super(FakeName, self).__init__(FakeSubModule, name_str, parent, start_pos) def get_definition(self): return self.parent diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index b21088ab..4582dc29 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -67,13 +67,13 @@ class ImportWrapper(pr.Base): # rest is import_path resolution import_path = [] - if import_stmt.from_ns: - import_path += import_stmt.from_ns.names - if import_stmt.namespace: + if import_stmt.from_names: + import_path += import_stmt.from_names + if import_stmt.namespace_names: if self.import_stmt.is_nested() and not nested_resolve: - import_path.append(import_stmt.namespace.names[0]) + import_path.append(import_stmt.namespace_names[0]) else: - import_path += import_stmt.namespace.names + import_path += import_stmt.namespace_names for i in range(kill_count + int(is_like_search)): if import_path: @@ -110,6 +110,7 @@ class ImportWrapper(pr.Base): m = _load_module(rel_path) names += m.get_defined_names() else: + # flask if self.import_path == ('flask', 'ext'): # List Flask extensions like ``flask_foo`` for mod in self._get_module_names(): @@ -122,6 +123,8 @@ class ImportWrapper(pr.Base): flaskext = os.path.join(dir, 'flaskext') if os.path.isdir(flaskext): names += self._get_module_names([flaskext]) + + # namespace packages if on_import_stmt and isinstance(scope, pr.Module) \ and scope.path.endswith('__init__.py'): pkg_path = os.path.dirname(scope.path) @@ -136,18 +139,18 @@ class ImportWrapper(pr.Base): # ``sys.modules`` modification. names.append(self._generate_name('path')) continue + + if not self.import_stmt.from_names or self.is_partial_import: + # from_names must be defined to access module + # values plus a partial import means that there + # is something after the import, which + # automatically implies that there must not be + # any non-module scope. + continue from jedi.evaluate import finder for s, scope_names in finder.get_names_of_scope(self._evaluator, scope, include_builtin=False): for n in scope_names: - if self.import_stmt.from_ns is None \ - or self.is_partial_import: - # from_ns must be defined to access module - # values plus a partial import means that there - # is something after the import, which - # automatically implies that there must not be - # any non-module scope. - continue names.append(n) return names @@ -179,55 +182,57 @@ class ImportWrapper(pr.Base): # check recursion return [] - if self.import_path: - try: - module, rest = self._importer.follow_file_system() - except ModuleNotFound as e: - analysis.add(self._evaluator, 'import-error', e.name_part) - return [] + try: + if self.import_path: + try: + module, rest = self._importer.follow_file_system() + except ModuleNotFound as e: + analysis.add(self._evaluator, 'import-error', e.name_part) + return [] - if module is None: - return [] + if module is None: + return [] - if self.import_stmt.is_nested() and not self.nested_resolve: - scopes = [NestedImportModule(module, self.import_stmt)] - else: - scopes = [module] - - star_imports = remove_star_imports(self._evaluator, module) - if star_imports: - scopes = [StarImportModule(scopes[0], star_imports)] - - # goto only accepts Names or NameParts - if is_goto and not rest: - scopes = [s.name.names[-1] for s in scopes] - - # follow the rest of the import (not FS -> classes, functions) - if len(rest) > 1 or rest and self.is_like_search: - scopes = [] - if ('os', 'path') == self.import_path[:2] \ - and not self._is_relative_import(): - # This is a huge exception, we follow a nested import - # ``os.path``, because it's a very important one in Python - # that is being achieved by messing with ``sys.modules`` in - # ``os``. - scopes = self._evaluator.follow_path(iter(rest), [module], module) - elif rest: - if is_goto: - scopes = list(chain.from_iterable( - self._evaluator.find_types(s, rest[0], is_goto=True) - for s in scopes)) + if self.import_stmt.is_nested() and not self.nested_resolve: + scopes = [NestedImportModule(module, self.import_stmt)] else: - scopes = list(chain.from_iterable( - self._evaluator.follow_path(iter(rest), [s], s) - for s in scopes)) - else: - scopes = [ImportWrapper.GlobalNamespace] - debug.dbg('after import: %s', scopes) - if not scopes: - analysis.add(self._evaluator, 'import-error', - self._importer.import_path[-1]) - self._evaluator.recursion_detector.pop_stmt() + scopes = [module] + + star_imports = remove_star_imports(self._evaluator, module) + if star_imports: + scopes = [StarImportModule(scopes[0], star_imports)] + + # goto only accepts `Name` + if is_goto and not rest: + scopes = [s.name for s in scopes] + + # follow the rest of the import (not FS -> classes, functions) + if len(rest) > 1 or rest and self.is_like_search: + scopes = [] + if ('os', 'path') == self.import_path[:2] \ + and not self._is_relative_import(): + # This is a huge exception, we follow a nested import + # ``os.path``, because it's a very important one in Python + # that is being achieved by messing with ``sys.modules`` in + # ``os``. + scopes = self._evaluator.follow_path(iter(rest), [module], module) + elif rest: + if is_goto: + scopes = list(chain.from_iterable( + self._evaluator.find_types(s, rest[0], is_goto=True) + for s in scopes)) + else: + scopes = list(chain.from_iterable( + self._evaluator.follow_path(iter(rest), [s], s) + for s in scopes)) + else: + scopes = [ImportWrapper.GlobalNamespace] + debug.dbg('after import: %s', scopes) + if not scopes: + analysis.add(self._evaluator, 'import-error', + self._importer.import_path[-1]) + finally: + self._evaluator.recursion_detector.pop_stmt() return scopes @@ -244,12 +249,12 @@ class NestedImportModule(pr.Module): # This is not an existing Import statement. Therefore, set position to # 0 (0 is not a valid line number). zero = (0, 0) - names = [unicode(name_part) for name_part in i.namespace.names[1:]] + names = [unicode(name_part) for name_part in i.namespace_names[1:]] name = helpers.FakeName(names, self._nested_import) new = pr.Import(i._sub_module, zero, zero, name) new.parent = self._module debug.dbg('Generated a nested import: %s', new) - return helpers.FakeName(str(i.namespace.names[1]), new) + return helpers.FakeName(str(i.namespace_names[1]), new) def _get_defined_names(self): """ @@ -330,7 +335,7 @@ class _Importer(object): self.file_path = os.path.dirname(path) if path is not None else None def str_import_path(self): - """Returns the import path as pure strings instead of NameParts.""" + """Returns the import path as pure strings instead of `Name`.""" return tuple(str(name_part) for name_part in self.import_path) def get_relative_path(self): @@ -372,12 +377,12 @@ class _Importer(object): pos = (part._line, part._column) try: self.import_path = ( - pr.NamePart(FakeSubModule, 'flask_' + str(part), part.parent, pos), + pr.Name(FakeSubModule, 'flask_' + str(part), part.parent, pos), ) + orig_path[3:] return self._real_follow_file_system() except ModuleNotFound as e: self.import_path = ( - pr.NamePart(FakeSubModule, 'flaskext', part.parent, pos), + pr.Name(FakeSubModule, 'flaskext', part.parent, pos), ) + orig_path[2:] return self._real_follow_file_system() return self._real_follow_file_system() @@ -606,5 +611,5 @@ def get_modules_containing_name(mods, name): for p in sorted(paths): # make testing easier, sort it - same results on every interpreter c = check_python_file(p) - if c is not None and c not in mods: + if c is not None and c not in mods and not isinstance(c, compiled.CompiledObject): yield c diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 15b4c972..d576d848 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -193,14 +193,14 @@ class Array(use_metaclass(CachedMetaClass, IterableWrapper)): def scope_names_generator(self, position=None): """ - This method generates all `ArrayMethod` for one pr.Array. It returns e.g. for a list: append, pop, ... """ # `array.type` is a string with the type, e.g. 'list'. scope = self._evaluator.find_types(compiled.builtin, self._array.type)[0] scope = self._evaluator.execute(scope)[0] # builtins only have one class + from jedi.evaluate.representation import get_instance_el for _, names in scope.scope_names_generator(): - yield self, [ArrayMethod(n) for n in names] + yield self, [get_instance_el(self._evaluator, self, n) for n in names] @common.safe_property def parent(self): @@ -225,34 +225,6 @@ class Array(use_metaclass(CachedMetaClass, IterableWrapper)): return "" % (type(self).__name__, self._array) -class ArrayMethod(IterableWrapper): - """ - A name, e.g. `list.append`, it is used to access the original array - methods. - """ - def __init__(self, name): - super(ArrayMethod, self).__init__() - self.name = name - - @property - @underscore_memoization - def names(self): - # TODO remove this method, we need the ArrayMethod input to be a NamePart. - return [pr.NamePart(self.name._sub_module, unicode(n), self, n.start_pos) for n in self.name.names] - - def __getattr__(self, name): - # Set access privileges: - if name not in ['parent', 'start_pos', 'end_pos', 'get_code', 'get_definition']: - raise AttributeError('Strange access on %s: %s.' % (self, name)) - return getattr(self.name, name) - - def get_parent_until(self): - return compiled.builtin - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self.name) - - class MergedArray(Array): def __init__(self, evaluator, arrays): super(MergedArray, self).__init__(evaluator, arrays[-1]._array) @@ -342,7 +314,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): result = [] for c in calls: call_path = list(c.generate_call_path()) - call_path_simple = [unicode(n) if isinstance(n, pr.NamePart) else n + call_path_simple = [unicode(n) if isinstance(n, pr.Name) else n for n in call_path] separate_index = call_path_simple.index(add_name) if add_name == call_path_simple[-1] or separate_index == 0: diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index f5ef8f50..3ac8ad9f 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -1,9 +1,10 @@ import copy from jedi._compatibility import unicode, zip_longest +from jedi import debug +from jedi import common from jedi.parser import representation as pr from jedi.evaluate import iterable -from jedi import common from jedi.evaluate import helpers from jedi.evaluate import analysis @@ -294,7 +295,7 @@ def _iterate_star_args(evaluator, array, expression_list, func): for field_stmt in array.iter_content(): yield helpers.FakeStatement([field_stmt]) elif isinstance(array, Instance) and array.name.get_code() == 'tuple': - pass + debug.warning('Ignored a tuple *args input %s' % array) else: if expression_list: m = "TypeError: %s() argument after * must be a sequence, not %s" \ @@ -320,6 +321,7 @@ def _star_star_dict(evaluator, array, expression_list, func): elif isinstance(call, pr.Call): key = call.name else: + debug.warning('Ignored complicated **kwargs stmt %s' % call) continue # We ignore complicated statements here, for now. # If the string is a duplicate, we don't care it's illegal Python @@ -359,8 +361,6 @@ def _gen_param_name_copy(func, var_args, param, keys=(), values=(), array_type=N new_param.set_expression_list([arr]) name = copy.copy(param.get_name()) - name.names = [copy.copy(name.names[0])] - name.names[0].parent = name name.parent = new_param return name diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 14d19b27..f5f28d99 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -164,12 +164,14 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): # because to follow them and their self variables is too # complicated. sub = self._get_method_execution(sub) - for n in sub.get_defined_names(): - # Only names with the selfname are being added. - # It is also important, that they have a len() of 2, - # because otherwise, they are just something else - if unicode(n.names[0]) == self_name and len(n.names) == 2: - add_self_dot_name(n) + for per_name_list in sub.get_names_dict().values(): + for call in per_name_list: + if unicode(call.name) == self_name \ + and isinstance(call.next, pr.Call) \ + and call.next.next is None: + names.append(get_instance_el(self._evaluator, self, call.next.name)) + #if unicode(n.names[0]) == self_name and len(n.names) == 2: + # add_self_dot_name(n) for s in self.base.py__bases__(self._evaluator): if not isinstance(s, compiled.CompiledObject): @@ -243,7 +245,12 @@ def get_instance_el(evaluator, instance, var, is_class_var=False): untouched. """ if isinstance(var, (Instance, compiled.CompiledObject, pr.Operator, Token, - pr.Module, FunctionExecution)): + pr.Module, FunctionExecution, pr.Name)): + if isinstance(var, pr.Name): + # TODO temp solution, remove later, Name should never get + # here? + par = get_instance_el(evaluator, instance, var.parent, is_class_var) + return pr.Name(var._sub_module, unicode(var), par, var.start_pos) return var var = wrap(evaluator, var) @@ -275,6 +282,9 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): return par def get_parent_until(self, *args, **kwargs): + if isinstance(self.var, pr.Name): + # TODO Name should never even be InstanceElements + return pr.Simple.get_parent_until(self.parent, *args, **kwargs) return pr.Simple.get_parent_until(self, *args, **kwargs) def get_definition(self): @@ -291,12 +301,6 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): return [get_instance_el(self._evaluator, self.instance, command, self.is_class_var) for command in self.var.expression_list()] - @property - @underscore_memoization - def names(self): - return [pr.NamePart(helpers.FakeSubModule, unicode(n), self, n.start_pos) - for n in self.var.names] - @property @underscore_memoization def name(self): @@ -390,6 +394,9 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): def py__call__(self, evaluator, params): return [Instance(evaluator, self, params)] + def py__getattribute__(self, name): + return self._evaluator.find_types(self, name) + def scope_names_generator(self, position=None, add_class_vars=True): def in_iterable(name, iterable): """ checks if the name is in the variable 'iterable'. """ @@ -522,6 +529,21 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): return "" % (type(self).__name__, self.base_func, dec) +class LazyDict(object): + def __init__(self, old_dct, copy_func): + self._copy_func = copy_func + self._old_dct = old_dct + + def __getitem__(self, key): + return self._copy_func(self._old_dct[key]) + + @underscore_memoization + def values(self): + # TODO REMOVE this. Not necessary with correct name lookups. + for calls in self._old_dct.values(): + yield self._copy_func(calls) + + class FunctionExecution(Executed): """ This class is used to evaluate functions and their returns. @@ -570,6 +592,10 @@ class FunctionExecution(Executed): break return types + @underscore_memoization + def get_names_dict(self): + return LazyDict(self.base.get_names_dict(), self._copy_list) + @memoize_default(default=()) def _get_params(self): """ @@ -591,15 +617,13 @@ class FunctionExecution(Executed): names = pr.filter_after_position(pr.Scope.get_defined_names(self), position) yield self, self._get_params() + names - def _copy_list(self, list_name): + def _copy_list(self, lst): """ Copies a list attribute of a parser Function. Copying is very expensive, because it is something like `copy.deepcopy`. However, these copied objects can be used for the executions, as if they were in the execution. """ - # Copy all these lists into this local function. - lst = getattr(self.base, list_name) objects = [] for element in lst: self._scope_copy(element.parent) @@ -622,22 +646,22 @@ class FunctionExecution(Executed): @common.safe_property @memoize_default([]) def returns(self): - return self._copy_list('returns') + return self._copy_list(self.base.returns) @common.safe_property @memoize_default([]) def asserts(self): - return self._copy_list('asserts') + return self._copy_list(self.base.asserts) @common.safe_property @memoize_default([]) def statements(self): - return self._copy_list('statements') + return self._copy_list(self.base.statements) @common.safe_property @memoize_default([]) def subscopes(self): - return self._copy_list('subscopes') + return self._copy_list(self.base.subscopes) def get_statement_for_position(self, pos): return pr.Scope.get_statement_for_position(self, pos) @@ -667,6 +691,11 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, pr.Module, Wrapper)): # All the additional module attributes are strings. return [helpers.LazyName(n, parent_callback) for n in names] + @property + @memoize_default() + def name(self): + return pr.Name(self, unicode(self.base.name), self, (1, 0)) + @memoize_default() def _sub_modules(self): """ @@ -683,6 +712,14 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, pr.Module, Wrapper)): imp = helpers.FakeImport(name, self, level=1) name.parent = imp names.append(name) + + # TODO add something like this in the future, its cleaner than the + # import hacks. + # ``os.path`` is a hardcoded exception, because it's a + # ``sys.modules`` modification. + #if str(self.name) == 'os': + # names.append(helpers.FakeName('path', parent=self)) + return names def __getattr__(self, name): diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 74d4b98b..170fa6fe 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -60,7 +60,7 @@ def _paths_from_assignment(evaluator, statement): for exp_list, operator in statement.assignment_details: if len(exp_list) != 1 or not isinstance(exp_list[0], pr.Call): continue - if unicode(exp_list[0].name) != 'sys.path': + if exp_list[0].names() != ['sys', 'path']: continue # TODO at this point we ignore all ways what could be assigned to # sys.path or an execution of it. Here we could do way more @@ -88,17 +88,15 @@ def _paths_from_insert(module_path, exe): def _paths_from_call_expression(module_path, call): """ extract the path from either "sys.path.append" or "sys.path.insert" """ - if not call.next_is_execution(): - return + names = call.names() + if names[:3] != ['sys', 'path', 'append'] and names[:3] != ['sys', 'path', 'insert']: + return [] + if not call.next.next.next_is_execution(): + return [] - n = call.name - if not isinstance(n, pr.Name) or len(n.names) != 3: - return - names = [unicode(x) for x in n.names] - if names[:2] != ['sys', 'path']: - return cmd = names[2] - exe = call.next + exe = call.next.next.next + path = None if cmd == 'insert' and len(exe) == 2: path = _paths_from_insert(module_path, exe) elif cmd == 'append' and len(exe) == 1: diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 66b05555..1ef45aff 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -100,38 +100,46 @@ class Parser(object): except KeyError: self.module.used_names[tok_name] = set([simple]) self.module.temp_used_names = [] + if isinstance(simple, pr.Statement): + for name, calls in simple.get_names_dict().items(): + self._scope.add_name_calls(name, calls) - def _parse_dot_name(self, pre_used_token=None): + def _parse_dotted_name(self, pre_used_token=None): """ The dot name parser parses a name, variable or function and returns their names. + Just used for parsing imports. :return: tuple of Name, next_token """ - def append(el): - names.append(el) - self.module.temp_used_names.append(el[0]) + def append(tok): + names.append(pr.Name(self.module, tok.string, None, tok.start_pos)) + self.module.temp_used_names.append(tok.string) names = [] tok = next(self._gen) if pre_used_token is None else pre_used_token if tok.type != tokenize.NAME and tok.string != '*': - return None, tok + return [], tok - first_pos = tok.start_pos - append((tok.string, first_pos)) + append(tok) while True: - end_pos = tok.end_pos tok = next(self._gen) if tok.string != '.': break tok = next(self._gen) if tok.type != tokenize.NAME: break - append((tok.string, tok.start_pos)) + append(tok) - n = pr.Name(self.module, names, first_pos, end_pos) if names else None - return n, tok + return names, tok + + def _parse_name(self, pre_used_token=None): + tok = next(self._gen) if pre_used_token is None else pre_used_token + self.module.temp_used_names.append(tok.string) + if tok.type != tokenize.NAME: + return None, tok + return pr.Name(self.module, tok.string, None, tok.start_pos), next(self._gen) def _parse_import_list(self): """ @@ -161,13 +169,13 @@ class Parser(object): tok = next(self._gen) if brackets and tok.type == tokenize.NEWLINE: tok = next(self._gen) - i, tok = self._parse_dot_name(tok) - if not i: + names, tok = self._parse_dotted_name(tok) + if not names: defunct = True - name2 = None + alias = None if tok.string == 'as': - name2, tok = self._parse_dot_name() - imports.append((i, name2, defunct)) + alias, tok = self._parse_name() + imports.append((names, alias, defunct)) while tok.string not in continue_kw: tok = next(self._gen) if not (tok.string == "," or brackets and tok.type == tokenize.NEWLINE): @@ -224,10 +232,8 @@ class Parser(object): if tok.type != tokenize.NAME: return None - fname = pr.Name(self.module, [(tok.string, tok.start_pos)], tok.start_pos, - tok.end_pos) + fname, tok = self._parse_name(tok) - tok = next(self._gen) if tok.string != '(': return None params = self._parse_parentheses(is_class=False) @@ -246,7 +252,7 @@ class Parser(object): if colon.string != ':': return None - # because of 2 line func param definitions + # Because of 2 line func param definitions return pr.Function(self.module, fname, params, first_pos, annotation) def _parse_class(self): @@ -264,11 +270,9 @@ class Parser(object): cname.start_pos[0], tokenize.tok_name[cname.type], cname.string) return None - cname = pr.Name(self.module, [(cname.string, cname.start_pos)], - cname.start_pos, cname.end_pos) + cname, _next = self._parse_name(cname) superclasses = [] - _next = next(self._gen) if _next.string == '(': superclasses = self._parse_parentheses(is_class=True) _next = next(self._gen) @@ -342,7 +346,7 @@ class Parser(object): if tok.string == 'as': tok = next(self._gen) if tok.type == tokenize.NAME: - n, tok = self._parse_dot_name(self._gen.current) + n, tok = self._parse_name(self._gen.current) if n: set_vars.append(n) as_names.append(n) @@ -354,11 +358,7 @@ class Parser(object): elif in_lambda_param and tok.string == ':': in_lambda_param = False elif tok.type == tokenize.NAME and not is_kw: - n, tok = self._parse_dot_name(self._gen.current) - # removed last entry, because we add Name - tok_list.pop() - if n: - tok_list.append(n) + tok_list[-1], tok = self._parse_name(tok) continue elif tok.string in opening_brackets: level += 1 @@ -460,10 +460,10 @@ class Parser(object): # import stuff elif tok_str == 'import': imports = self._parse_import_list() - for count, (m, alias, defunct) in enumerate(imports): - e = (alias or m or self._gen.previous).end_pos + for count, (names, alias, defunct) in enumerate(imports): + e = (alias or names and names[-1] or self._gen.previous).end_pos end_pos = self._gen.previous.end_pos if count + 1 == len(imports) else e - i = pr.Import(self.module, first_pos, end_pos, m, + i = pr.Import(self.module, first_pos, end_pos, names, alias, defunct=defunct) self._check_user_stmt(i) self._scope.add_import(i) @@ -482,26 +482,26 @@ class Parser(object): break relative_count += 1 # the from import - mod, tok = self._parse_dot_name(self._gen.current) + from_names, tok = self._parse_dotted_name(self._gen.current) tok_str = tok.string - if str(mod) == 'import' and relative_count: + if len(from_names) == 1 and str(from_names[0]) == 'import' and relative_count: self._gen.push_last_back() tok_str = 'import' - mod = None - if not mod and not relative_count or tok_str != "import": + from_names = [] + if not from_names and not relative_count or tok_str != "import": debug.warning("from: syntax error@%s", tok.start_pos[0]) defunct = True if tok_str != 'import': self._gen.push_last_back() - names = self._parse_import_list() - for count, (name, alias, defunct2) in enumerate(names): - star = name is not None and unicode(name.names[0]) == '*' + imports = self._parse_import_list() + for count, (names, alias, defunct2) in enumerate(imports): + star = names and unicode(names[-1]) == '*' if star: - name = None - e = (alias or name or self._gen.previous).end_pos - end_pos = self._gen.previous.end_pos if count + 1 == len(names) else e - i = pr.Import(self.module, first_pos, end_pos, name, - alias, mod, star, relative_count, + names = [] + e = (alias or names and names[-1] or self._gen.previous).end_pos + #end_pos = self._gen.previous.end_pos if count + 1 == len(names) else e + i = pr.Import(self.module, first_pos, e, names, + alias, from_names, star, relative_count, defunct=defunct or defunct2) self._check_user_stmt(i) self._scope.add_import(i) @@ -536,7 +536,7 @@ class Parser(object): if command == 'except' and tok.string == ',': # the except statement defines a var # this is only true for python 2 - n, tok = self._parse_dot_name() + n, tok = self._parse_name() if n: n.parent = statement statement.as_names.append(n) diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index 85a10a8d..438903a1 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -37,6 +37,7 @@ See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. import os import re from inspect import cleandoc +from collections import defaultdict from jedi._compatibility import (next, Python3Method, encoding, unicode, is_py3, u, literal_eval, use_metaclass) @@ -231,6 +232,14 @@ class IsScope(use_metaclass(IsScopeMeta)): pass +def _return_empty_list(): + """ + Necessary for pickling. It needs to be reachable for pickle, cannot + be a lambda or a closure. + """ + return [] + + class Scope(Simple, DocstringMixin): """ Super class for the parser tree, which represents the state of a python @@ -243,7 +252,7 @@ class Scope(Simple, DocstringMixin): :type start_pos: tuple(int, int) """ __slots__ = ('subscopes', 'imports', 'statements', '_doc_token', 'asserts', - 'returns', 'is_generator') + 'returns', 'is_generator', '_names_dict') def __init__(self, module, start_pos): super(Scope, self).__init__(module, start_pos) @@ -255,11 +264,19 @@ class Scope(Simple, DocstringMixin): # Needed here for fast_parser, because the fast_parser splits and # returns will be in "normal" modules. self.returns = [] + self._names_dict = defaultdict(_return_empty_list) self.is_generator = False def is_scope(self): return True + def add_name_calls(self, name, calls): + """Add a name to the names_dict.""" + self._names_dict[name] += calls + + def get_names_dict(self): + return self._names_dict + def add_scope(self, sub, decorators): sub.parent = self.use_as_parent sub.decorators = decorators @@ -468,8 +485,7 @@ class SubModule(Scope, Module): string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) # Positions are not real, but a module starts at (1, 0) p = (1, 0) - names = [(string, p)] - return Name(self, names, p, p, self.use_as_parent) + return Name(self, string, self.use_as_parent, p) @property def has_explicit_absolute_import(self): @@ -478,10 +494,10 @@ class SubModule(Scope, Module): is a ``__future__`` import. """ for imp in self.imports: - if imp.from_ns is None or imp.namespace is None: + if not imp.from_names or not imp.namespace_names: continue - namespace, feature = imp.from_ns.names[0], imp.namespace.names[0] + namespace, feature = imp.from_names[0], imp.namespace_names[0] if unicode(namespace) == "__future__" and unicode(feature) == "absolute_import": return True @@ -529,9 +545,9 @@ class Class(Scope): if self._doc_token is not None: docstr = self.raw_doc for sub in self.subscopes: - if unicode(sub.name.names[-1]) == '__init__': + if unicode(sub.name) == '__init__': return '%s\n\n%s' % ( - sub.get_call_signature(funcname=self.name.names[-1]), docstr) + sub.get_call_signature(funcname=self.name), docstr) return docstr def scope_names_generator(self, position=None): @@ -597,7 +613,7 @@ class Function(Scope): :rtype: str """ - l = unicode(funcname or self.name.names[-1]) + '(' + l = unicode(funcname or self.name) + '(' lines = [] for (i, p) in enumerate(self.params): code = p.get_code(False) @@ -672,6 +688,15 @@ class Flow(Scope): s.parent = self.use_as_parent self.set_vars = [] + def add_name_calls(self, name, calls): + """Add a name to the names_dict.""" + parent = self.parent + if isinstance(parent, Module): + # TODO this also looks like code smell. Look for opportunities to + # remove. + parent = self._sub_module + parent.add_name_calls(name, calls) + @property def parent(self): return self._parent @@ -769,27 +794,28 @@ class Import(Simple): :param start_pos: Position (line, column) of the Import. :type start_pos: tuple(int, int) - :param namespace: The import, can be empty if a star is given - :type namespace: Name + :param namespace_names: The import, can be empty if a star is given + :type namespace_names: list of Name :param alias: The alias of a namespace(valid in the current namespace). - :type alias: Name - :param from_ns: Like the namespace, can be equally used. - :type from_ns: Name + :type alias: list of Name + :param from_names: Like the namespace, can be equally used. + :type from_names: list of Name :param star: If a star is used -> from time import *. :type star: bool :param defunct: An Import is valid or not. :type defunct: bool """ - def __init__(self, module, start_pos, end_pos, namespace, alias=None, - from_ns=None, star=False, relative_count=0, defunct=False): + def __init__(self, module, start_pos, end_pos, namespace_names, alias=None, + from_names=(), star=False, relative_count=0, defunct=False): super(Import, self).__init__(module, start_pos, end_pos) - self.namespace = namespace + self.namespace_names = namespace_names self.alias = alias - self.from_ns = from_ns - for n in namespace, alias, from_ns: - if n: - n.parent = self.use_as_parent + if self.alias: + alias.parent = self + self.from_names = from_names + for n in namespace_names + list(from_names): + n.parent = self.use_as_parent self.star = star self.relative_count = relative_count @@ -798,20 +824,18 @@ class Import(Simple): def get_code(self, new_line=True): # in case one of the names is None alias = self.alias or '' - namespace = self.namespace or '' - from_ns = self.from_ns or '' + ns_str = '.'.join(unicode(n) for n in self.namespace_names) if self.alias: - ns_str = "%s as %s" % (namespace, alias) - else: - ns_str = unicode(namespace) + ns_str = "%s as %s" % (ns_str, alias) nl = '\n' if new_line else '' - if self.from_ns or self.relative_count: + if self.from_names or self.relative_count: if self.star: ns_str = '*' dots = '.' * self.relative_count - return "from %s%s import %s%s" % (dots, from_ns, ns_str, nl) + from_txt = '.'.join(unicode(n) for n in self.from_names) + return "from %s%s import %s%s" % (dots, from_txt, ns_str, nl) else: return "import %s%s" % (ns_str, nl) @@ -822,21 +846,18 @@ class Import(Simple): return [self] if self.alias: return [self.alias] - if len(self.namespace) > 1: - o = self.namespace - n = Name(self._sub_module, [(unicode(o.names[0]), o.start_pos)], - o.start_pos, o.end_pos, parent=o.parent) - return [n] + if len(self.namespace_names) > 1: + return [self.namespace_names[0]] else: - return [self.namespace] + return self.namespace_names def get_all_import_names(self): n = [] - if self.from_ns: - n.append(self.from_ns) - if self.namespace: - n.append(self.namespace) - if self.alias: + if self.from_names: + n += self.from_names + if self.namespace_names: + n += self.namespace_names + if self.alias is not None: n.append(self.alias) return n @@ -847,8 +868,8 @@ class Import(Simple): import foo.bar """ - return not self.alias and not self.from_ns and self.namespace is not None \ - and len(self.namespace.names) > 1 + return not self.alias and not self.from_names \ + and len(self.namespace_names) > 1 class KeywordStatement(Base): @@ -911,9 +932,6 @@ class Statement(Simple, DocstringMixin): self._token_list = token_list self._names_are_set_vars = names_are_set_vars if set_name_parents: - for t in token_list: - if isinstance(t, Name): - t.parent = self.use_as_parent for n in as_names: n.parent = self.use_as_parent self._doc_token = None @@ -950,7 +968,7 @@ class Statement(Simple, DocstringMixin): return code def get_defined_names(self): - """ Get the names for the statement. """ + """Get the names for the statement.""" if self._set_vars is None: def search_calls(calls): @@ -959,15 +977,11 @@ class Statement(Simple, DocstringMixin): for stmt in call: search_calls(stmt.expression_list()) elif isinstance(call, Call): - c = call # Check if there's an execution in it, if so this is # not a set_var. - while c: - if isinstance(c.next, Array): - break - c = c.next - else: + if not call.next: self._set_vars.append(call.name) + continue self._set_vars = [] for calls, operation in self.assignment_details: @@ -978,6 +992,37 @@ class Statement(Simple, DocstringMixin): search_calls(self.expression_list()) return self._set_vars + self.as_names + def get_names_dict(self): + """The future of name resolution. Returns a dict(str -> Call).""" + dct = defaultdict(lambda: []) + + def search_calls(calls): + for call in calls: + if isinstance(call, Array) and call.type != Array.DICT: + for stmt in call: + search_calls(stmt.expression_list()) + elif isinstance(call, Call): + c = call + # Check if there's an execution in it, if so this is + # not a set_var. + while True: + if c.next is None or isinstance(c.next, Array): + break + c = c.next + dct[unicode(c.name)].append(call) + + for calls, operation in self.assignment_details: + search_calls(calls) + + if not self.assignment_details and self._names_are_set_vars: + # In the case of Param, it's also a defining name without ``=`` + search_calls(self.expression_list()) + + for as_name in self.as_names: + dct[unicode(as_name)].append(Call(self._sub_module, as_name, + as_name.start_pos, as_name.end_pos, self)) + return dct + def is_global(self): p = self.parent return isinstance(p, KeywordStatement) and p.name == 'global' @@ -1050,7 +1095,7 @@ class Statement(Simple, DocstringMixin): return arr, break_tok def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(), - break_on_assignment=False, stmt_class=Statement, + break_on_assignment=False, stmt_class=ArrayStmt, allow_comma=False): token_list = [] level = 0 @@ -1064,7 +1109,7 @@ class Statement(Simple, DocstringMixin): first = False if isinstance(tok, Base): - # the token is a Name, which has already been parsed + # The token is a Name, which has already been parsed. if not level: if isinstance(tok, ListComprehension): # it's not possible to set it earlier @@ -1134,13 +1179,10 @@ class Statement(Simple, DocstringMixin): added_breaks=added_breaks) if stmt is not None: - for t in stmt._token_list: - if isinstance(t, Name): - t.parent = stmt stmt._names_are_set_vars = names_are_set_vars return stmt, tok - st = Statement(self._sub_module, token_list, start_pos, + st = ArrayStmt(self._sub_module, token_list, start_pos, end_pos, set_name_parents=False) middle, tok = parse_stmt_or_arr(token_iterator, ['in'], True) @@ -1170,7 +1212,7 @@ class Statement(Simple, DocstringMixin): next(token_iterator, None) continue else: - # the token is a Name, which has already been parsed + # The token is a Name, which has already been parsed tok_str = tok token_type = None @@ -1190,7 +1232,7 @@ class Statement(Simple, DocstringMixin): continue is_literal = token_type in (tokenize.STRING, tokenize.NUMBER) - if isinstance(tok_str, Name) or is_literal: + if is_literal or isinstance(tok, Name): cls = Literal if is_literal else Call call = cls(self._sub_module, tok_str, tok.start_pos, tok.end_pos, self) @@ -1213,7 +1255,7 @@ class Statement(Simple, DocstringMixin): is_chain = True elif tok_str == ',' and result: # implies a tuple # expression is now an array not a statement anymore - stmt = Statement(self._sub_module, result, result[0].start_pos, + stmt = ArrayStmt(self._sub_module, result, result[0].start_pos, tok.end_pos, self.parent, set_name_parents=False) stmt._expression_list = result arr, break_tok = parse_array(token_iterator, Array.TUPLE, @@ -1247,6 +1289,14 @@ class ExprStmt(Statement): """ +class ArrayStmt(Statement): + """ + This class exists temporarily. Like ``ExprStatement``, this exists to + distinguish between real statements and stuff that is defined in those + statements. + """ + + class Param(ExprStmt): """ The class which shows definitions of params of classes and functions. @@ -1308,8 +1358,7 @@ class StatementElement(Simple): def generate_call_path(self): """ Helps to get the order in which statements are executed. """ try: - for name_part in self.name.names: - yield name_part + yield self.name except AttributeError: yield self if self.next is not None: @@ -1318,7 +1367,7 @@ class StatementElement(Simple): def get_code(self): if self.next is not None: - s = '.' if isinstance(self, Array) else '' + s = '.' if not isinstance(self.next, Array) else '' return s + self.next.get_code() return '' @@ -1334,6 +1383,21 @@ class Call(StatementElement): def get_code(self): return self.name.get_code() + super(Call, self).get_code() + def names(self): + """ + Generate an array of string names. If a call is not just names, + raise an error. + """ + def check(call): + while call is not None: + if not isinstance(call, Call): # Could be an Array. + break + yield unicode(call.name) + call = call.next + + return list(check(self)) + + def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.name) @@ -1450,7 +1514,7 @@ class Array(StatementElement): return "<%s: %s%s>" % (type(self).__name__, typ, self.values) -class NamePart(object): +class Name(object): """ A string. Sometimes it is important to know if the string belongs to a name or not. @@ -1474,13 +1538,14 @@ class NamePart(object): return self._string def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self._string) + return "<%s: %s@%s,%s>" % (type(self).__name__, self._string, + self.start_pos[0], self.start_pos[1]) def get_code(self): return self._string def get_definition(self): - return self.parent.get_definition() + return self.get_parent_until((ArrayStmt, StatementElement), reverse=True) def get_parent_until(self, *args, **kwargs): return self.parent.get_parent_until(*args, **kwargs) @@ -1498,48 +1563,6 @@ class NamePart(object): return self.start_pos[0], self.start_pos[1] + len(self._string) -class Name(Simple): - """ - Used to define names in python. - Which means the whole namespace/class/function stuff. - So a name like "module.class.function" - would result in an array of [module, class, function] - """ - __slots__ = ('names', '_get_code') - - def __init__(self, module, names, start_pos, end_pos, parent=None): - super(Name, self).__init__(module, start_pos, end_pos, parent) - # Cache get_code, because it's used quite often for comparisons - # (seen by using the profiler). - self._get_code = ".".join(n[0] for n in names) - - names = tuple(NamePart(module, n[0], self, n[1]) for n in names) - self.names = names - - def get_code(self): - """ Returns the names in a full string format """ - return self._get_code - - def get_definition(self): - # TODO This is way to complicated, simplify this with a new parser. - return self.get_parent_until((ExprStmt, IsScope, Import)) - - @property - def end_pos(self): - return self.names[-1].end_pos - - @property - def docstr(self): - """Return attribute docstring (PEP 257) if exists.""" - return self.parent.docstr - - def __str__(self): - return self.get_code() - - def __len__(self): - return len(self.names) - - class ListComprehension(ForFlow): """ Helper class for list comprehensions """ def __init__(self, module, stmt, middle, input, parent): diff --git a/test/completion/dynamic.py b/test/completion/dynamic.py index da1723bd..18bb0ac3 100644 --- a/test/completion/dynamic.py +++ b/test/completion/dynamic.py @@ -2,7 +2,9 @@ This is used for dynamic object completion. Jedi tries to guess the types with a backtracking approach. """ -def func(a): +def func(a, default_arg=2): + #? int() + default_arg #? int() str() return a diff --git a/test/completion/usages.py b/test/completion/usages.py index 447fa87c..e7b762d3 100644 --- a/test/completion/usages.py +++ b/test/completion/usages.py @@ -185,7 +185,7 @@ class TestClass(Super): TestClass.base_var - #< 13 (5,13), (0,13) + #< 13 (5,13), (0,13), (-24,13) self.instance_var = 3 #< 9 (0,8), diff --git a/test/test_api/test_api_classes.py b/test/test_api/test_api_classes.py index 404aa849..f327113c 100644 --- a/test/test_api/test_api_classes.py +++ b/test/test_api/test_api_classes.py @@ -148,6 +148,28 @@ def test_signature_params(): check(Script(s + '\nbar=foo\nbar').goto_assignments()) +class TestIsDefinition(TestCase): + def _def(self, source, index=-1): + return names(dedent(source), references=True, all_scopes=True)[index] + + def test_name(self): + d = self._def('name') + assert d.name == 'name' + assert not d.is_definition() + + def test_stmt(self): + src = 'a = f(x)' + d = self._def(src, 0) + assert d.name == 'a' + assert d.is_definition() + d = self._def(src, 1) + assert d.name == 'f' + assert not d.is_definition() + d = self._def(src) + assert d.name == 'x' + assert not d.is_definition() + + class TestParent(TestCase): def _parent(self, source, line=None, column=None): defs = Script(dedent(source), line, column).goto_assignments() @@ -219,3 +241,60 @@ class TestGotoAssignments(TestCase): param = bar.goto_assignments()[0] assert param.start_pos == (1, 13) assert param.type == 'param' + + def test_class_call(self): + src = 'from threading import Thread; Thread(group=1)' + n = names(src, references=True)[-1] + assert n.name == 'group' + param_def = n.goto_assignments()[0] + assert param_def.name == 'group' + assert param_def.type == 'param' + + def test_parentheses(self): + n = names('("").upper', references=True)[-1] + assert n.goto_assignments()[0].name == 'upper' + + def test_import(self): + nms = names('from json import load', references=True) + assert nms[0].name == 'json' + assert nms[0].type == 'import' + n = nms[0].goto_assignments()[0] + assert n.name == 'json' + assert n.type == 'module' + + assert nms[1].name == 'load' + assert nms[1].type == 'import' + n = nms[1].goto_assignments()[0] + assert n.name == 'load' + assert n.type == 'function' + + nms = names('import os; os.path', references=True) + assert nms[0].name == 'os' + assert nms[0].type == 'import' + n = nms[0].goto_assignments()[0] + assert n.name == 'os' + assert n.type == 'module' + + n = nms[2].goto_assignments()[0] + assert n.name == 'path' + assert n.type == 'import' + + nms = names('import os.path', references=True) + n = nms[0].goto_assignments()[0] + assert n.name == 'os' + assert n.type == 'module' + n = nms[1].goto_assignments()[0] + assert n.name == 'path' + assert n.type == 'import' + + def test_import_alias(self): + nms = names('import json as foo', references=True) + assert nms[0].name == 'json' + assert nms[0].type == 'import' + n = nms[0].goto_assignments()[0] + assert n.name == 'json' + assert n.type == 'module' + + assert nms[1].name == 'foo' + assert nms[1].type == 'import' + assert [nms[1]] == nms[1].goto_assignments() diff --git a/test/test_evaluate/test_helpers.py b/test/test_evaluate/test_helpers.py index c6ba5c33..056ba98e 100644 --- a/test/test_evaluate/test_helpers.py +++ b/test/test_evaluate/test_helpers.py @@ -1,18 +1,9 @@ from jedi._compatibility import unicode from jedi.evaluate import helpers -from jedi.parser import representation as pr from jedi.parser import Parser -def test_deep_ast_copy(): - name = pr.Name(object, [('hallo', (0, 0))], (0, 0), (0, 0)) - - # fast parent copy should switch parent - new_name = helpers.deep_ast_copy(name) - assert new_name.names[0].parent == new_name - - def test_statement_elements_in_statement(): def get_stmt_els(string): p = Parser(unicode(string)) diff --git a/test/test_evaluate/test_representation.py b/test/test_evaluate/test_representation.py index 2ed4a40d..315ba63e 100644 --- a/test/test_evaluate/test_representation.py +++ b/test/test_evaluate/test_representation.py @@ -5,7 +5,7 @@ from jedi import Script def get_definition_and_evaluator(source): d = Script(dedent(source)).goto_definitions()[0] - return d._name.parent.parent, d._evaluator + return d._name.parent, d._evaluator def test_function_execution(): diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 04a9115f..ab57d4f1 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -26,4 +26,4 @@ asdfasdf""" + "h" def test_tokenizer_with_string_literal_backslash(): import jedi c = jedi.Script("statement = u'foo\\\n'; statement").goto_definitions() - assert c[0]._name.parent.parent.obj == 'foo' + assert c[0]._name.parent.obj == 'foo'