Merge remote-tracking branch 'upstream/dev' into bugfix/performances_degradation

# Conflicts:
#	jedi/evaluate/compiled/fake.py

jedi/evaluate/__init__.py

@@ -85,10 +85,13 @@ class Evaluator(object):
         self.memoize_cache = {}  # for memoize decorators
         # To memorize modules -> equals `sys.modules`.
         self.modules = {}  # like `sys.modules`.
-        self.compiled_cache = {}  # see `compiled.create()`
-        self.recursion_detector = recursion.RecursionDetector()
-        self.execution_recursion_detector = recursion.ExecutionRecursionDetector()
+        self.compiled_cache = {}  # see `evaluate.compiled.create()`
+        self.mixed_cache = {}  # see `evaluate.compiled.mixed.create()`
         self.analysis = []
+        self.predefined_if_name_dict_dict = {}
+        self.dynamic_params_depth = 0
+        self.is_analysis = False
+
         if sys_path is None:
             sys_path = sys.path
         self.sys_path = copy.copy(sys_path)
@@ -97,16 +100,28 @@ class Evaluator(object):
         except ValueError:
             pass
 
+        self.reset_recursion_limitations()
+
+        # Constants
+        self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
+
+    def reset_recursion_limitations(self):
+        self.recursion_detector = recursion.RecursionDetector(self)
+        self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
+
     def wrap(self, element):
-        if isinstance(element, tree.Class):
+        if isinstance(element, (er.Wrapper, er.InstanceElement,
+            er.ModuleWrapper, er.FunctionExecution, er.Instance, compiled.CompiledObject)) or element is None:
+            # TODO this is so ugly, please refactor.
+            return element
+
+        if element.type == 'classdef':
             return er.Class(self, element)
-        elif isinstance(element, tree.Function):
-            if isinstance(element, tree.Lambda):
-                return er.LambdaWrapper(self, element)
-            else:
-                return er.Function(self, element)
-        elif isinstance(element, (tree.Module)) \
-                and not isinstance(element, er.ModuleWrapper):
+        elif element.type == 'funcdef':
+            return er.Function(self, element)
+        elif element.type == 'lambda':
+            return er.LambdaWrapper(self, element)
+        elif element.type == 'file_input':
             return er.ModuleWrapper(self, element)
         else:
             return element
@@ -125,10 +140,10 @@ class Evaluator(object):
         scopes = f.scopes(search_global)
         if is_goto:
             return f.filter_name(scopes)
-        return f.find(scopes, search_global)
+        return f.find(scopes, attribute_lookup=not search_global)
 
-    @memoize_default(default=[], evaluator_is_first_arg=True)
-    @recursion.recursion_decorator
+    #@memoize_default(default=[], evaluator_is_first_arg=True)
+    #@recursion.recursion_decorator
     @debug.increase_indent
     def eval_statement(self, stmt, seek_name=None):
         """
@@ -140,10 +155,11 @@ class Evaluator(object):
         :param stmt: A `tree.ExprStmt`.
         """
         debug.dbg('eval_statement %s (%s)', stmt, seek_name)
-        types = self.eval_element(stmt.get_rhs())
+        rhs = stmt.get_rhs()
+        types = self.eval_element(rhs)
 
         if seek_name:
-            types = finder.check_tuple_assignments(types, seek_name)
+            types = finder.check_tuple_assignments(self, types, seek_name)
 
         first_operation = stmt.first_operation()
         if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement):  # TODO don't check for this.
@@ -153,71 +169,168 @@ class Evaluator(object):
             name = str(stmt.get_defined_names()[0])
             parent = self.wrap(stmt.get_parent_scope())
             left = self.find_types(parent, name, stmt.start_pos, search_global=True)
-            if isinstance(stmt.get_parent_until(tree.ForStmt), tree.ForStmt):
+
+            for_stmt = stmt.get_parent_until(tree.ForStmt)
+            if isinstance(for_stmt, tree.ForStmt) and types \
+                    and for_stmt.defines_one_name():
                 # Iterate through result and add the values, that's possible
                 # only in for loops without clutter, because they are
-                # predictable.
-                for r in types:
-                    left = precedence.calculate(self, left, operator, [r])
+                # predictable. Also only do it, if the variable is not a tuple.
+                node = for_stmt.get_input_node()
+                for_iterables = self.eval_element(node)
+                ordered = list(iterable.py__iter__(self, for_iterables, node))
+
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    self.predefined_if_name_dict_dict[for_stmt] = dct
+                    t = self.eval_element(rhs)
+                    left = precedence.calculate(self, left, operator, t)
                 types = left
+                if ordered:
+                    # If there are no for entries, we cannot iterate and the
+                    # types are defined by += entries. Therefore the for loop
+                    # is never called.
+                    del self.predefined_if_name_dict_dict[for_stmt]
             else:
                 types = precedence.calculate(self, left, operator, types)
         debug.dbg('eval_statement result %s', types)
         return types
 
-    @memoize_default(evaluator_is_first_arg=True)
     def eval_element(self, element):
         if isinstance(element, iterable.AlreadyEvaluated):
-            return list(element)
+            return set(element)
         elif isinstance(element, iterable.MergedNodes):
             return iterable.unite(self.eval_element(e) for e in element)
 
+        if_stmt = element.get_parent_until((tree.IfStmt, tree.ForStmt, tree.IsScope))
+        predefined_if_name_dict = self.predefined_if_name_dict_dict.get(if_stmt)
+        if predefined_if_name_dict is None and isinstance(if_stmt, tree.IfStmt):
+            if_stmt_test = if_stmt.children[1]
+            name_dicts = [{}]
+            # If we already did a check, we don't want to do it again -> If
+            # predefined_if_name_dict_dict is filled, we stop.
+            # We don't want to check the if stmt itself, it's just about
+            # the content.
+            if element.start_pos > if_stmt_test.end_pos:
+                # Now we need to check if the names in the if_stmt match the
+                # names in the suite.
+                if_names = helpers.get_names_of_node(if_stmt_test)
+                element_names = helpers.get_names_of_node(element)
+                str_element_names = [str(e) for e in element_names]
+                if any(str(i) in str_element_names for i in if_names):
+                    for if_name in if_names:
+                        definitions = self.goto_definitions(if_name)
+                        # Every name that has multiple different definitions
+                        # causes the complexity to rise. The complexity should
+                        # never fall below 1.
+                        if len(definitions) > 1:
+                            if len(name_dicts) * len(definitions) > 16:
+                                debug.dbg('Too many options for if branch evaluation %s.', if_stmt)
+                                # There's only a certain amount of branches
+                                # Jedi can evaluate, otherwise it will take to
+                                # long.
+                                name_dicts = [{}]
+                                break
+
+                            original_name_dicts = list(name_dicts)
+                            name_dicts = []
+                            for definition in definitions:
+                                new_name_dicts = list(original_name_dicts)
+                                for i, name_dict in enumerate(new_name_dicts):
+                                    new_name_dicts[i] = name_dict.copy()
+                                    new_name_dicts[i][str(if_name)] = [definition]
+
+                                name_dicts += new_name_dicts
+                        else:
+                            for name_dict in name_dicts:
+                                name_dict[str(if_name)] = definitions
+            if len(name_dicts) > 1:
+                result = set()
+                for name_dict in name_dicts:
+                    self.predefined_if_name_dict_dict[if_stmt] = name_dict
+                    try:
+                        result |= self._eval_element_not_cached(element)
+                    finally:
+                        del self.predefined_if_name_dict_dict[if_stmt]
+                return result
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+        else:
+            if predefined_if_name_dict:
+                return self._eval_element_not_cached(element)
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+
+    def _eval_element_if_evaluated(self, element):
+        """
+        TODO This function is temporary: Merge with eval_element.
+        """
+        parent = element
+        while parent is not None:
+            parent = parent.parent
+            predefined_if_name_dict = self.predefined_if_name_dict_dict.get(parent)
+            if predefined_if_name_dict is not None:
+                return self._eval_element_not_cached(element)
+        return self._eval_element_cached(element)
+
+    @memoize_default(evaluator_is_first_arg=True)
+    def _eval_element_cached(self, element):
+        return self._eval_element_not_cached(element)
+
+    @debug.increase_indent
+    def _eval_element_not_cached(self, element):
         debug.dbg('eval_element %s@%s', element, element.start_pos)
+        types = set()
         if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
-            return self._eval_atom(element)
+            types = self._eval_atom(element)
         elif isinstance(element, tree.Keyword):
             # For False/True/None
             if element.value in ('False', 'True', 'None'):
-                return [compiled.builtin.get_by_name(element.value)]
-            else:
-                return []
+                types.add(compiled.builtin_from_name(self, element.value))
+            # else: print e.g. could be evaluated like this in Python 2.7
         elif element.isinstance(tree.Lambda):
-            return [er.LambdaWrapper(self, element)]
+            types = set([er.LambdaWrapper(self, element)])
         elif element.isinstance(er.LambdaWrapper):
-            return [element]  # TODO this is no real evaluation.
+            types = set([element])  # TODO this is no real evaluation.
         elif element.type == 'expr_stmt':
-            return self.eval_statement(element)
-        elif element.type == 'power':
+            types = self.eval_statement(element)
+        elif element.type in ('power', 'atom_expr'):
             types = self._eval_atom(element.children[0])
             for trailer in element.children[1:]:
                 if trailer == '**':  # has a power operation.
-                    raise NotImplementedError
+                    right = self.eval_element(element.children[2])
+                    types = set(precedence.calculate(self, types, trailer, right))
+                    break
                 types = self.eval_trailer(types, trailer)
-
-            return types
         elif element.type in ('testlist_star_expr', 'testlist',):
             # The implicit tuple in statements.
-            return [iterable.ImplicitTuple(self, element)]
+            types = set([iterable.ImplicitTuple(self, element)])
         elif element.type in ('not_test', 'factor'):
             types = self.eval_element(element.children[-1])
             for operator in element.children[:-1]:
-                types = list(precedence.factor_calculate(self, types, operator))
-            return types
+                types = set(precedence.factor_calculate(self, types, operator))
         elif element.type == 'test':
             # `x if foo else y` case.
-            return (self.eval_element(element.children[0]) +
-                    self.eval_element(element.children[-1]))
+            types = (self.eval_element(element.children[0]) |
+                     self.eval_element(element.children[-1]))
         elif element.type == 'operator':
             # Must be an ellipsis, other operators are not evaluated.
-            return []  # Ignore for now.
+            assert element.value == '...'
+            types = set([compiled.create(self, Ellipsis)])
         elif element.type == 'dotted_name':
             types = self._eval_atom(element.children[0])
             for next_name in element.children[2::2]:
-                types = list(chain.from_iterable(self.find_types(typ, next_name)
-                                                 for typ in types))
-            return types
+                types = set(chain.from_iterable(self.find_types(typ, next_name)
+                                                for typ in types))
+            types = types
+        elif element.type == 'eval_input':
+            types = self._eval_element_not_cached(element.children[0])
         else:
-            return precedence.calculate_children(self, element.children)
+            types = precedence.calculate_children(self, element.children)
+        debug.dbg('eval_element result %s', types)
+        return types
 
     def _eval_atom(self, atom):
         """
@@ -229,6 +342,13 @@ class Evaluator(object):
             # This is the first global lookup.
             stmt = atom.get_definition()
             scope = stmt.get_parent_until(tree.IsScope, include_current=True)
+            if isinstance(scope, (tree.Function, er.FunctionExecution)):
+                # Adjust scope: If the name is not in the suite, it's a param
+                # default or annotation and will be resolved as part of the
+                # parent scope.
+                colon = scope.children.index(':')
+                if atom.start_pos < scope.children[colon + 1].start_pos:
+                    scope = scope.get_parent_scope()
             if isinstance(stmt, tree.CompFor):
                 stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
             if stmt.type != 'expr_stmt':
@@ -237,42 +357,53 @@ class Evaluator(object):
                 stmt = atom
             return self.find_types(scope, atom, stmt.start_pos, search_global=True)
         elif isinstance(atom, tree.Literal):
-            return [compiled.create(self, atom.eval())]
+            return set([compiled.create(self, atom.eval())])
         else:
             c = atom.children
+            if c[0].type == 'string':
+                # Will be one string.
+                types = self._eval_atom(c[0])
+                for string in c[1:]:
+                    right = self._eval_atom(string)
+                    types = precedence.calculate(self, types, '+', right)
+                return types
             # Parentheses without commas are not tuples.
-            if c[0] == '(' and not len(c) == 2 \
+            elif c[0] == '(' and not len(c) == 2 \
                     and not(tree.is_node(c[1], 'testlist_comp')
                             and len(c[1].children) > 1):
                 return self.eval_element(c[1])
+
             try:
                 comp_for = c[1].children[1]
             except (IndexError, AttributeError):
                 pass
             else:
-                if isinstance(comp_for, tree.CompFor):
-                    return [iterable.Comprehension.from_atom(self, atom)]
-            return [iterable.Array(self, atom)]
+                if comp_for == ':':
+                    # Dict comprehensions have a colon at the 3rd index.
+                    try:
+                        comp_for = c[1].children[3]
+                    except IndexError:
+                        pass
+
+                if comp_for.type == 'comp_for':
+                    return set([iterable.Comprehension.from_atom(self, atom)])
+            return set([iterable.Array(self, atom)])
 
     def eval_trailer(self, types, trailer):
         trailer_op, node = trailer.children[:2]
         if node == ')':  # `arglist` is optional.
             node = ()
-        new_types = []
-        for typ in types:
-            debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
-            if trailer_op == '.':
-                new_types += self.find_types(typ, node)
-            elif trailer_op == '(':
-                new_types += self.execute(typ, node, trailer)
-            elif trailer_op == '[':
-                try:
-                    get = typ.get_index_types
-                except AttributeError:
-                    debug.warning("TypeError: '%s' object is not subscriptable"
-                                  % typ)
-                else:
-                    new_types += get(self, node)
+
+        new_types = set()
+        if trailer_op == '[':
+            new_types |= iterable.py__getitem__(self, types, trailer)
+        else:
+            for typ in types:
+                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
+                if trailer_op == '.':
+                    new_types |= self.find_types(typ, node)
+                elif trailer_op == '(':
+                    new_types |= self.execute(typ, node, trailer)
         return new_types
 
     def execute_evaluated(self, obj, *args):
@@ -287,6 +418,9 @@ class Evaluator(object):
         if not isinstance(arguments, param.Arguments):
             arguments = param.Arguments(self, arguments, trailer)
 
+        if self.is_analysis:
+            arguments.eval_all()
+
         if obj.isinstance(er.Function):
             obj = obj.get_decorated_func()
 
@@ -302,17 +436,28 @@ class Evaluator(object):
             func = obj.py__call__
         except AttributeError:
             debug.warning("no execution possible %s", obj)
-            return []
+            return set()
         else:
-            types = func(self, arguments)
+            types = func(arguments)
             debug.dbg('execute result: %s in %s', types, obj)
             return types
 
-    def goto_definition(self, name):
+    def goto_definitions(self, name):
         def_ = name.get_definition()
-        if def_.type == 'expr_stmt' and name in def_.get_defined_names():
-            return self.eval_statement(def_, name)
-        call = helpers.call_of_name(name)
+        is_simple_name = name.parent.type not in ('power', 'trailer')
+        if is_simple_name:
+            if name.parent.type in ('file_input', 'classdef', 'funcdef'):
+                return [self.wrap(name.parent)]
+            if def_.type == 'expr_stmt' and name in def_.get_defined_names():
+                return self.eval_statement(def_, name)
+            elif def_.type == 'for_stmt':
+                container_types = self.eval_element(def_.children[3])
+                for_types = iterable.py__iter__types(self, container_types, def_.children[3])
+                return finder.check_tuple_assignments(self, for_types, name)
+            elif def_.type in ('import_from', 'import_name'):
+                return imports.ImportWrapper(self, name).follow()
+
+        call = helpers.call_of_leaf(name)
         return self.eval_element(call)
 
     def goto(self, name):
@@ -372,8 +517,8 @@ class Evaluator(object):
                 ))
 
         scope = name.get_parent_scope()
-        if tree.is_node(name.parent, 'trailer'):
-            call = helpers.call_of_name(name, cut_own_trailer=True)
+        if tree.is_node(par, 'trailer') and par.children[0] == '.':
+            call = helpers.call_of_leaf(name, cut_own_trailer=True)
             types = self.eval_element(call)
             return resolve_implicit_imports(iterable.unite(
                 self.find_types(typ, name, is_goto=True) for typ in types

jedi/evaluate/analysis.py

@@ -5,19 +5,25 @@ from jedi import debug
 from jedi.parser import tree
 from jedi.evaluate.compiled import CompiledObject
 
+from jedi.common import unite
+
 
 CODES = {
     'attribute-error': (1, AttributeError, 'Potential AttributeError.'),
     'name-error': (2, NameError, 'Potential NameError.'),
     'import-error': (3, ImportError, 'Potential ImportError.'),
-    'type-error-generator': (4, TypeError, "TypeError: 'generator' object is not subscriptable."),
-    'type-error-too-many-arguments': (5, TypeError, None),
-    'type-error-too-few-arguments': (6, TypeError, None),
-    'type-error-keyword-argument': (7, TypeError, None),
-    'type-error-multiple-values': (8, TypeError, None),
-    'type-error-star-star': (9, TypeError, None),
-    'type-error-star': (10, TypeError, None),
-    'type-error-operation': (11, TypeError, None),
+    'type-error-too-many-arguments': (4, TypeError, None),
+    'type-error-too-few-arguments': (5, TypeError, None),
+    'type-error-keyword-argument': (6, TypeError, None),
+    'type-error-multiple-values': (7, TypeError, None),
+    'type-error-star-star': (8, TypeError, None),
+    'type-error-star': (9, TypeError, None),
+    'type-error-operation': (10, TypeError, None),
+    'type-error-not-iterable': (11, TypeError, None),
+    'type-error-isinstance': (12, TypeError, None),
+    'type-error-not-subscriptable': (13, TypeError, None),
+    'value-error-too-many-values': (14, ValueError, None),
+    'value-error-too-few-values': (15, ValueError, None),
 }
 
 
@@ -158,8 +164,8 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
                     from jedi.evaluate import iterable
                     if isinstance(cls, iterable.Array) and cls.type == 'tuple':
                         # multiple exceptions
-                        for c in cls.values():
-                            if check_match(c, exception):
+                        for typ in unite(cls.py__iter__()):
+                            if check_match(typ, exception):
                                 return True
                     else:
                         if check_match(cls, exception):
@@ -168,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
     def check_hasattr(node, suite):
         try:
             assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos
-            assert node.type == 'power'
+            assert node.type in ('power', 'atom_expr')
             base = node.children[0]
             assert base.type == 'name' and base.value == 'hasattr'
             trailer = node.children[1]
@@ -183,7 +189,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
             # Check name
             key, values = args[1]
             assert len(values) == 1
-            names = evaluator.eval_element(values[0])
+            names = list(evaluator.eval_element(values[0]))
             assert len(names) == 1 and isinstance(names[0], CompiledObject)
             assert names[0].obj == str(payload[1])
 
@@ -208,95 +214,3 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
         obj = obj.parent
 
     return False
-
-
-def get_module_statements(module):
-    """
-    Returns the statements used in a module. All these statements should be
-    evaluated to check for potential exceptions.
-    """
-    def check_children(node):
-        try:
-            children = node.children
-        except AttributeError:
-            return []
-        else:
-            nodes = []
-            for child in children:
-                nodes += check_children(child)
-                if child.type == 'trailer':
-                    c = child.children
-                    if c[0] == '(' and c[1] != ')':
-                        if c[1].type != 'arglist':
-                            if c[1].type == 'argument':
-                                nodes.append(c[1].children[-1])
-                            else:
-                                nodes.append(c[1])
-                        else:
-                            for argument in c[1].children:
-                                if argument.type == 'argument':
-                                    nodes.append(argument.children[-1])
-                                elif argument.type != 'operator':
-                                    nodes.append(argument)
-            return nodes
-
-    def add_nodes(nodes):
-        new = set()
-        for node in nodes:
-            if isinstance(node, tree.Flow):
-                children = node.children
-                if node.type == 'for_stmt':
-                    children = children[2:]  # Don't want to include the names.
-                # Pick the suite/simple_stmt.
-                new |= add_nodes(children)
-            elif node.type in ('simple_stmt', 'suite'):
-                new |= add_nodes(node.children)
-            elif node.type in ('return_stmt', 'yield_expr'):
-                try:
-                    new.add(node.children[1])
-                except IndexError:
-                    pass
-            elif node.type not in ('whitespace', 'operator', 'keyword',
-                                   'parameters', 'decorated', 'except_clause') \
-                    and not isinstance(node, (tree.ClassOrFunc, tree.Import)):
-                new.add(node)
-
-                try:
-                    children = node.children
-                except AttributeError:
-                    pass
-                else:
-                    for next_node in children:
-                        new.update(check_children(node))
-                        if next_node.type != 'keyword' and node.type != 'expr_stmt':
-                            new.add(node)
-        return new
-
-    nodes = set()
-    import_names = set()
-    decorated_funcs = []
-    for scope in module.walk():
-        for imp in set(scope.imports):
-            import_names |= set(imp.get_defined_names())
-            if imp.is_nested():
-                import_names |= set(path[-1] for path in imp.paths())
-
-        children = scope.children
-        if isinstance(scope, tree.ClassOrFunc):
-            children = children[2:]  # We don't want to include the class name.
-        nodes |= add_nodes(children)
-
-        for flow in scope.flows:
-            if flow.type == 'for_stmt':
-                nodes.add(flow.children[3])
-            elif flow.type == 'try_stmt':
-                nodes.update(e for e in flow.except_clauses() if e is not None)
-
-        try:
-            decorators = scope.get_decorators()
-        except AttributeError:
-            pass
-        else:
-            if decorators:
-                decorated_funcs.append(scope)
-    return nodes, import_names, decorated_funcs

jedi/evaluate/compiled/__init__.py

@@ -41,34 +41,30 @@ class CompiledObject(Base):
     path = None  # modules have this attribute - set it to None.
     used_names = {}  # To be consistent with modules.
 
-    def __init__(self, obj, parent=None):
+    def __init__(self, evaluator, obj, parent=None):
+        self._evaluator = evaluator
         self.obj = obj
         self.parent = parent
 
-    @property
-    def py__call__(self):
-        def actual(evaluator, params):
-            if inspect.isclass(self.obj):
-                from jedi.evaluate.representation import Instance
-                return [Instance(evaluator, self, params)]
-            else:
-                return list(self._execute_function(evaluator, params))
-
-        # Might raise an AttributeError, which is intentional.
-        self.obj.__call__
-        return actual
+    @CheckAttribute
+    def py__call__(self, params):
+        if inspect.isclass(self.obj):
+            from jedi.evaluate.representation import Instance
+            return set([Instance(self._evaluator, self, params)])
+        else:
+            return set(self._execute_function(params))
 
     @CheckAttribute
-    def py__class__(self, evaluator):
-        return CompiledObject(self.obj.__class__, parent=self.parent)
+    def py__class__(self):
+        return create(self._evaluator, self.obj.__class__)
 
     @CheckAttribute
-    def py__mro__(self, evaluator):
-        return tuple(create(evaluator, cls, self.parent) for cls in self.obj.__mro__)
+    def py__mro__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__)
 
     @CheckAttribute
-    def py__bases__(self, evaluator):
-        return tuple(create(evaluator, cls) for cls in self.obj.__bases__)
+    def py__bases__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__)
 
     def py__bool__(self):
         return bool(self.obj)
@@ -87,7 +83,7 @@ class CompiledObject(Base):
     def params(self):
         params_str, ret = self._parse_function_doc()
         tokens = params_str.split(',')
-        if inspect.ismethoddescriptor(self._cls().obj):
+        if inspect.ismethoddescriptor(self.obj):
             tokens.insert(0, 'self')
         params = []
         for p in tokens:
@@ -108,22 +104,21 @@ class CompiledObject(Base):
         return _parse_function_doc(self.doc)
 
     def api_type(self):
-        if fake.is_class_instance(self.obj):
-            return 'instance'
-
-        cls = self._cls().obj
-        if inspect.isclass(cls):
+        obj = self.obj
+        if inspect.isclass(obj):
             return 'class'
-        elif inspect.ismodule(cls):
+        elif inspect.ismodule(obj):
             return 'module'
-        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \
-                or inspect.ismethoddescriptor(cls):
+        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
             return 'function'
+        # Everything else...
+        return 'instance'
 
     @property
     def type(self):
         """Imitate the tree.Node.type values."""
-        cls = self._cls().obj
+        cls = self._get_class()
         if inspect.isclass(cls):
             return 'classdef'
         elif inspect.ismodule(cls):
@@ -134,17 +129,24 @@ class CompiledObject(Base):
 
     @underscore_memoization
     def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
         # Ensures that a CompiledObject is returned that is not an instance (like list)
-        if fake.is_class_instance(self.obj):
-            try:
-                c = self.obj.__class__
-            except AttributeError:
-                # happens with numpy.core.umath._UFUNC_API (you get it
-                # automatically by doing `import numpy`.
-                c = type(None)
-            return CompiledObject(c, self.parent)
         return self
 
+    def _get_class(self):
+        if not fake.is_class_instance(self.obj):
+            return self.obj
+
+        try:
+            return self.obj.__class__
+        except AttributeError:
+            # happens with numpy.core.umath._UFUNC_API (you get it
+            # automatically by doing `import numpy`.
+            return type
+
     @property
     def names_dict(self):
         # For compatibility with `representation.Class`.
@@ -159,64 +161,55 @@ class CompiledObject(Base):
         search_global shouldn't change the fact that there's one dict, this way
         there's only one `object`.
         """
-        return [LazyNamesDict(self._cls(), is_instance)]
+        return [LazyNamesDict(self._evaluator, self, is_instance)]
 
     def get_subscope_by_name(self, name):
-        if name in dir(self._cls().obj):
-            return CompiledName(self._cls(), name).parent
+        if name in dir(self.obj):
+            return CompiledName(self._evaluator, self, name).parent
         else:
             raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
 
-    def get_index_types(self, evaluator, index_array=()):
-        # If the object doesn't have `__getitem__`, just raise the
-        # AttributeError.
-        if not hasattr(self.obj, '__getitem__'):
-            debug.warning('Tried to call __getitem__ on non-iterable.')
-            return []
+    @CheckAttribute
+    def py__getitem__(self, index):
         if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
             # Get rid of side effects, we won't call custom `__getitem__`s.
-            return []
+            return set()
 
-        result = []
-        from jedi.evaluate.iterable import create_indexes_or_slices
-        for typ in create_indexes_or_slices(evaluator, index_array):
-            index = None
-            try:
-                index = typ.obj
-                new = self.obj[index]
-            except (KeyError, IndexError, TypeError, AttributeError):
-                # Just try, we don't care if it fails, except for slices.
-                if isinstance(index, slice):
-                    result.append(self)
-            else:
-                result.append(CompiledObject(new))
-        if not result:
-            try:
-                for obj in self.obj:
-                    result.append(CompiledObject(obj))
-            except TypeError:
-                pass  # self.obj maynot have an __iter__ method.
-        return result
+        return set([create(self._evaluator, self.obj[index])])
+
+    @CheckAttribute
+    def py__iter__(self):
+        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return
+
+        for part in self.obj:
+            yield set([create(self._evaluator, part)])
 
     @property
     def name(self):
-        # might not exist sometimes (raises AttributeError)
-        return FakeName(self._cls().obj.__name__, self)
+        try:
+            name = self._get_class().__name__
+        except AttributeError:
+            name = repr(self.obj)
+        return FakeName(name, self)
 
-    def _execute_function(self, evaluator, params):
+    def _execute_function(self, params):
         if self.type != 'funcdef':
             return
 
         for name in self._parse_function_doc()[1].split():
             try:
-                bltn_obj = _create_from_name(builtin, builtin, name)
+                bltn_obj = getattr(_builtins, name)
             except AttributeError:
                 continue
             else:
-                if isinstance(bltn_obj, CompiledObject) and bltn_obj.obj is None:
-                    # We want everything except None.
+                if bltn_obj is None:
+                    # We want to evaluate everything except None.
+                    # TODO do we?
                     continue
-                for result in evaluator.execute(bltn_obj, params):
+                bltn_obj = create(self._evaluator, bltn_obj)
+                for result in self._evaluator.execute(bltn_obj, params):
                     yield result
 
     @property
@@ -228,7 +221,7 @@ class CompiledObject(Base):
         """
         module = self.get_parent_until()
         faked_subscopes = []
-        for name in dir(self._cls().obj):
+        for name in dir(self.obj):
             f = fake.get_faked(module.obj, self.obj, name)
             if f:
                 f.parent = self
@@ -245,11 +238,42 @@ class CompiledObject(Base):
         return []  # Builtins don't have imports
 
 
+class CompiledName(FakeName):
+    def __init__(self, evaluator, compiled_obj, name):
+        super(CompiledName, self).__init__(name)
+        self._evaluator = evaluator
+        self._compiled_obj = compiled_obj
+        self.name = name
+
+    def __repr__(self):
+        try:
+            name = self._compiled_obj.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
+
+    def is_definition(self):
+        return True
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        module = self._compiled_obj.get_parent_until()
+        return _create_from_name(self._evaluator, module, self._compiled_obj, self.name)
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
+
+
 class LazyNamesDict(object):
     """
     A names_dict instance for compiled objects, resembles the parser.tree.
     """
-    def __init__(self, compiled_obj, is_instance):
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_obj, is_instance=False):
+        self._evaluator = evaluator
         self._compiled_obj = compiled_obj
         self._is_instance = is_instance
 
@@ -262,7 +286,12 @@ class LazyNamesDict(object):
             getattr(self._compiled_obj.obj, name)
         except AttributeError:
             raise KeyError('%s in %s not found.' % (name, self._compiled_obj))
-        return [CompiledName(self._compiled_obj, name)]
+        except Exception:
+            # This is a bit ugly. We're basically returning this to make
+            # lookups possible without having the actual attribute. However
+            # this makes proper completion possible.
+            return [FakeName(name, create(self._evaluator, None), is_definition=True)]
+        return [self.name_class(self._evaluator, self._compiled_obj, name)]
 
     def values(self):
         obj = self._compiled_obj.obj
@@ -275,39 +304,13 @@ class LazyNamesDict(object):
                 # The dir function can be wrong.
                 pass
 
-        # dir doesn't include the type names.
-        if not inspect.ismodule(obj) and obj != type and not self._is_instance:
-            values += _type_names_dict.values()
+        is_instance = self._is_instance or fake.is_class_instance(obj)
+        # ``dir`` doesn't include the type names.
+        if not inspect.ismodule(obj) and obj != type and not is_instance:
+            values += create(self._evaluator, type).names_dict.values()
         return values
 
 
-class CompiledName(FakeName):
-    def __init__(self, obj, name):
-        super(CompiledName, self).__init__(name)
-        self._obj = obj
-        self.name = name
-
-    def __repr__(self):
-        try:
-            name = self._obj.name  # __name__ is not defined all the time
-        except AttributeError:
-            name = None
-        return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
-
-    def is_definition(self):
-        return True
-
-    @property
-    @underscore_memoization
-    def parent(self):
-        module = self._obj.get_parent_until()
-        return _create_from_name(module, self._obj, self.name)
-
-    @parent.setter
-    def parent(self, value):
-        pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
-
-
 def dotted_from_fs_path(fs_path, sys_path):
     """
     Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
@@ -369,7 +372,7 @@ def load_module(evaluator, path=None, name=None):
     # complicated import structure of Python.
     module = sys.modules[dotted_path]
 
-    return CompiledObject(module)
+    return create(evaluator, module)
 
 
 docstr_defaults = {
@@ -441,19 +444,7 @@ def _parse_function_doc(doc):
     return param_str, ret
 
 
-class Builtin(CompiledObject):
-    @memoize_method
-    def get_by_name(self, name):
-        return self.names_dict[name][0].parent
-
-
-def _a_generator(foo):
-    """Used to have an object to return for generators."""
-    yield 42
-    yield foo
-
-
-def _create_from_name(module, parent, name):
+def _create_from_name(evaluator, module, parent, name):
     faked = fake.get_faked(module.obj, parent.obj, name)
     # only functions are necessary.
     if faked is not None:
@@ -467,61 +458,78 @@ def _create_from_name(module, parent, name):
         # PyQt4.QtGui.QStyleOptionComboBox.currentText
         # -> just set it to None
         obj = None
-    return CompiledObject(obj, parent)
+    return create(evaluator, obj, parent)
 
 
-builtin = Builtin(_builtins)
-magic_function_class = CompiledObject(type(load_module), parent=builtin)
-generator_obj = CompiledObject(_a_generator(1.0))
-_type_names_dict = builtin.get_by_name('type').names_dict
-none_obj = builtin.get_by_name('None')
-false_obj = builtin.get_by_name('False')
-true_obj = builtin.get_by_name('True')
-object_obj = builtin.get_by_name('object')
+def builtin_from_name(evaluator, string):
+    bltn_obj = getattr(_builtins, string)
+    return create(evaluator, bltn_obj)
 
 
-def keyword_from_value(obj):
-    if obj is None:
-        return none_obj
-    elif obj is False:
-        return false_obj
-    elif obj is True:
-        return true_obj
-    else:
-        raise NotImplementedError
+def _a_generator(foo):
+    """Used to have an object to return for generators."""
+    yield 42
+    yield foo
 
 
-def compiled_objects_cache(func):
-    def wrapper(evaluator, obj, parent=builtin, module=None):
-        # Do a very cheap form of caching here.
-        key = id(obj), id(parent), id(module)
-        try:
-            return evaluator.compiled_cache[key][0]
-        except KeyError:
-            result = func(evaluator, obj, parent, module)
-            # Need to cache all of them, otherwise the id could be overwritten.
-            evaluator.compiled_cache[key] = result, obj, parent, module
-            return result
-    return wrapper
+_SPECIAL_OBJECTS = {
+    'FUNCTION_CLASS': type(load_module),
+    'MODULE_CLASS': type(os),
+    'GENERATOR_OBJECT': _a_generator(1.0),
+    'BUILTINS': _builtins,
+}
 
 
-@compiled_objects_cache
-def create(evaluator, obj, parent=builtin, module=None):
+def get_special_object(evaluator, identifier):
+    obj = _SPECIAL_OBJECTS[identifier]
+    return create(evaluator, obj, parent=create(evaluator, _builtins))
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent=None, module=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj), id(parent)
+            try:
+                return cache[key][0]
+            except KeyError:
+                # TODO this whole decorator looks way too ugly and this if
+                # doesn't make it better. Find a more generic solution.
+                if parent or module:
+                    result = func(evaluator, obj, parent, module)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent, module
+                return result
+        return wrapper
+
+    return decorator
+
+
+@compiled_objects_cache('compiled_cache')
+def create(evaluator, obj, parent=None, module=None):
     """
     A very weird interface class to this module. The more options provided the
     more acurate loading compiled objects is.
     """
+    if inspect.ismodule(obj):
+        if parent is not None:
+            # Modules don't have parents, be careful with caching: recurse.
+            return create(evaluator, obj)
+    else:
+        if parent is None and obj != _builtins:
+            return create(evaluator, obj, create(evaluator, _builtins))
 
-    if not inspect.ismodule(obj):
         faked = fake.get_faked(module and module.obj, obj)
         if faked is not None:
             faked.parent = parent
             return faked
 
-    try:
-        if parent == builtin and obj.__module__ in ('builtins', '__builtin__'):
-            return builtin.get_by_name(obj.__name__)
-    except AttributeError:
-        pass
-
-    return CompiledObject(obj, parent)
+    return CompiledObject(evaluator, obj, parent)

jedi/evaluate/compiled/fake.py

@@ -6,16 +6,44 @@ mixing in Python code, the autocompletion should work much better for builtins.
 
 import os
 import inspect
+import types
 
-from jedi._compatibility import is_py3, builtins, unicode
+from jedi._compatibility import is_py3, builtins, unicode, is_py34
 from jedi.cache import memoize_function
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser import tree as pt
 from jedi.evaluate.helpers import FakeName
 
 modules = {}
 
 
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace
+    )
+    if is_py34:
+        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
+
+
 def _load_faked_module(module):
     module_name = module.__name__
     if module_name == '__builtin__' and not is_py3:
@@ -31,8 +59,8 @@ def _load_faked_module(module):
         except IOError:
             modules[module_name] = None
             return
-        grammar = load_grammar('grammar3.4')
-        module = Parser(grammar, unicode(source), module_name).module
+        grammar = load_grammar(version='3.4')
+        module = ParserWithRecovery(grammar, unicode(source), module_name).module
         modules[module_name] = module
 
         if module_name == 'builtins' and not is_py3:
@@ -65,7 +93,15 @@ def get_module(obj):
         # Unfortunately in some cases like `int` there's no __module__
         return builtins
     else:
-        return __import__(imp_plz)
+        if imp_plz is None:
+            # Happens for example in `(_ for _ in []).send.__module__`.
+            return builtins
+        else:
+            try:
+                return __import__(imp_plz)
+            except ImportError:
+                # __module__ can be something arbitrary that doesn't exist.
+                return builtins
 
 
 def _faked(module, obj, name):
@@ -75,7 +111,7 @@ def _faked(module, obj, name):
 
     faked_mod = _load_faked_module(module)
     if faked_mod is None:
-        return
+        return None
 
     # Having the module as a `parser.representation.module`, we need to scan
     # for methods.
@@ -84,23 +120,32 @@ def _faked(module, obj, name):
             return search_scope(faked_mod, obj.__name__)
         elif not inspect.isclass(obj):
             # object is a method or descriptor
-            cls = search_scope(faked_mod, obj.__objclass__.__name__)
-            if cls is None:
-                return
-            return search_scope(cls, obj.__name__)
+            try:
+                objclass = obj.__objclass__
+            except AttributeError:
+                return None
+            else:
+                cls = search_scope(faked_mod, objclass.__name__)
+                if cls is None:
+                    return None
+                return search_scope(cls, obj.__name__)
     else:
         if obj == module:
             return search_scope(faked_mod, name)
         else:
-            cls = search_scope(faked_mod, obj.__name__)
+            try:
+                cls_name = obj.__name__
+            except AttributeError:
+                return None
+            cls = search_scope(faked_mod, cls_name)
             if cls is None:
-                return
+                return None
             return search_scope(cls, name)
 
 
 @memoize_function
 def get_faked(module, obj, name=None):
-    obj = obj.__class__ if is_class_instance(obj) else obj
+    obj = type(obj) if is_class_instance(obj) else obj
     result = _faked(module, obj, name)
     if result is None or isinstance(result, pt.Class):
         # We're not interested in classes. What we want is functions.
@@ -119,7 +164,9 @@ def get_faked(module, obj, name=None):
 
 def is_class_instance(obj):
     """Like inspect.* methods."""
-    return not (inspect.isclass(obj) or inspect.ismodule(obj)
-                or inspect.isbuiltin(obj) or inspect.ismethod(obj)
-                or inspect.ismethoddescriptor(obj) or inspect.iscode(obj)
-                or inspect.isgenerator(obj))
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

jedi/evaluate/compiled/fake/builtins.pym

@@ -124,7 +124,7 @@ class list():
         return self.__iterable[y]
 
     def pop(self):
-        return self.__iterable[-1]
+        return self.__iterable[int()]
 
 
 class tuple():
@@ -202,11 +202,29 @@ class dict():
         except KeyError:
             return d
 
+    def values(self):
+        return self.__elements.values()
+
     def setdefault(self, k, d):
         # TODO maybe also return the content
         return d
 
 
+class enumerate():
+    def __init__(self, sequence, start=0):
+        self.__sequence = sequence
+
+    def __iter__(self):
+        for i in self.__sequence:
+            yield 1, i
+
+    def __next__(self):
+        return next(self.__iter__())
+
+    def next(self):
+        return next(self.__iter__())
+
+
 class reversed():
     def __init__(self, sequence):
         self.__sequence = sequence

jedi/evaluate/compiled/mixed.py

@@ -0,0 +1,158 @@
+"""
+Used only for REPL Completion.
+"""
+
+import inspect
+import os
+
+from jedi import common
+from jedi.parser.fast import FastParser
+from jedi.evaluate import compiled
+from jedi.cache import underscore_memoization
+
+
+class MixedObject(object):
+    """
+    A ``MixedObject`` is used in two ways:
+
+    1. It uses the default logic of ``parser.tree`` objects,
+    2. except for getattr calls. The names dicts are generated in a fashion
+       like ``CompiledObject``.
+
+    This combined logic makes it possible to provide more powerful REPL
+    completion. It allows side effects that are not noticable with the default
+    parser structure to still be completeable.
+
+    The biggest difference from CompiledObject to MixedObject is that we are
+    generally dealing with Python code and not with C code. This will generate
+    fewer special cases, because we in Python you don't have the same freedoms
+    to modify the runtime.
+    """
+    def __init__(self, evaluator, obj, node_name):
+        self._evaluator = evaluator
+        self.obj = obj
+        self.node_name = node_name
+        self.definition = node_name.get_definition()
+
+    @property
+    def names_dict(self):
+        return LazyMixedNamesDict(self._evaluator, self)
+
+    def names_dicts(self, search_global):
+        # TODO is this needed?
+        assert search_global is False
+        return [self.names_dict]
+
+    def api_type(self):
+        mappings = {
+            'expr_stmt': 'statement',
+            'classdef': 'class',
+            'funcdef': 'function',
+            'file_input': 'module',
+        }
+        return mappings[self.definition.type]
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+
+    def __getattr__(self, name):
+        return getattr(self.definition, name)
+
+
+class MixedName(compiled.CompiledName):
+    """
+    The ``CompiledName._compiled_object`` is our MixedObject.
+    """
+    @property
+    @underscore_memoization
+    def parent(self):
+        return create(self._evaluator, getattr(self._compiled_obj.obj, self.name))
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, Name tries to overwrite the parent attribute.
+
+    @property
+    def start_pos(self):
+        if isinstance(self.parent, MixedObject):
+            return self.parent.node_name.start_pos
+
+        # This means a start_pos that doesn't exist (compiled objects).
+        return (0, 0)
+
+
+class LazyMixedNamesDict(compiled.LazyNamesDict):
+    name_class = MixedName
+
+
+def parse(grammar, path):
+    with open(path) as f:
+        source = f.read()
+    source = common.source_to_unicode(source)
+    return FastParser(grammar, source, path)
+
+
+def _load_module(evaluator, path, python_object):
+    module = parse(evaluator.grammar, path).module
+    python_module = inspect.getmodule(python_object)
+
+    evaluator.modules[python_module.__name__] = module
+    return module
+
+
+def find_syntax_node_name(evaluator, python_object):
+    try:
+        path = inspect.getsourcefile(python_object)
+    except TypeError:
+        # The type might not be known (e.g. class_with_dict.__weakref__)
+        return None
+    if path is None or not os.path.exists(path):
+        # The path might not exist or be e.g. <stdin>.
+        return None
+
+    module = _load_module(evaluator, path, python_object)
+
+    if inspect.ismodule(python_object):
+        # We don't need to check names for modules, because there's not really
+        # a way to write a module in a module in Python (and also __name__ can
+        # be something like ``email.utils``).
+        return module
+
+    try:
+        names = module.used_names[python_object.__name__]
+    except NameError:
+        return None
+
+    names = [n for n in names if n.is_definition()]
+
+    try:
+        code = python_object.__code__
+        # By using the line number of a code object we make the lookup in a
+        # file pretty easy. There's still a possibility of people defining
+        # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
+        # do so we just don't care.
+        line_nr = code.co_firstlineno
+    except AttributeError:
+        pass
+    else:
+        line_names = [name for name in names if name.start_pos[0] == line_nr]
+        # There's a chance that the object is not available anymore, because
+        # the code has changed in the background.
+        if line_names:
+            return line_names[-1]
+
+    # It's really hard to actually get the right definition, here as a last
+    # resort we just return the last one. This chance might lead to odd
+    # completions at some points but will lead to mostly correct type
+    # inference, because people tend to define a public name in a module only
+    # once.
+    return names[-1]
+
+
+@compiled.compiled_objects_cache('mixed_cache')
+def create(evaluator, obj):
+    name = find_syntax_node_name(evaluator, obj)
+    if name is None:
+        return compiled.create(evaluator, obj)
+    else:
+        return MixedObject(evaluator, obj, name)

jedi/evaluate/docstrings.py

@@ -20,7 +20,7 @@ from itertools import chain
 from textwrap import dedent
 
 from jedi.evaluate.cache import memoize_default
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.common import indent_block
 from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated
 
@@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module):
     # Take the default grammar here, if we load the Python 2.7 grammar here, it
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
-    p = Parser(load_grammar(), code % indent_block(string))
+    p = ParserWithRecovery(load_grammar(), code % indent_block(string))
     try:
         pseudo_cls = p.module.subscopes[0]
         # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
@@ -164,8 +164,8 @@ def _execute_array_values(evaluator, array):
     """
     if isinstance(array, Array):
         values = []
-        for typ in array.values():
-            objects = _execute_array_values(evaluator, typ)
+        for types in array.py__iter__():
+            objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types))
             values.append(AlreadyEvaluated(objects))
         return [FakeSequence(evaluator, values, array.type)]
     else:
@@ -176,11 +176,11 @@ def _execute_array_values(evaluator, array):
 def follow_param(evaluator, param):
     func = param.parent_function
 
-    return [p
-            for param_str in _search_param_in_docstr(func.raw_doc,
-                                                     str(param.name))
+    return set(
+        [p for param_str in _search_param_in_docstr(func.raw_doc,
+                                                    str(param.name))
             for p in _evaluate_for_statement_string(evaluator, param_str,
-                                                    param.get_parent_until())]
+                                                    param.get_parent_until())])
 
 
 @memoize_default(None, evaluator_is_first_arg=True)

jedi/evaluate/dynamic.py

@@ -26,6 +26,9 @@ from jedi.evaluate.cache import memoize_default
 from jedi.evaluate import imports
 
 
+MAX_PARAM_SEARCHES = 20
+
+
 class ParamListener(object):
     """
     This listener is used to get the params for a function.
@@ -52,17 +55,21 @@ def search_params(evaluator, param):
     is.
     """
     if not settings.dynamic_params:
-        return []
+        return set()
 
-    func = param.get_parent_until(tree.Function)
-    debug.dbg('Dynamic param search for %s in %s.', param, str(func.name))
-    # Compare the param names.
-    names = [n for n in search_function_call(evaluator, func)
-             if n.value == param.name.value]
-    # Evaluate the ExecutedParams to types.
-    result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names))
-    debug.dbg('Dynamic param result %s', result)
-    return result
+    evaluator.dynamic_params_depth += 1
+    try:
+        func = param.get_parent_until(tree.Function)
+        debug.dbg('Dynamic param search for %s in %s.', param, str(func.name), color='MAGENTA')
+        # Compare the param names.
+        names = [n for n in search_function_call(evaluator, func)
+                 if n.value == param.name.value]
+        # Evaluate the ExecutedParams to types.
+        result = set(chain.from_iterable(n.parent.eval(evaluator) for n in names))
+        debug.dbg('Dynamic param result %s', result, color='MAGENTA')
+        return result
+    finally:
+        evaluator.dynamic_params_depth -= 1
 
 
 @memoize_default([], evaluator_is_first_arg=True)
@@ -72,52 +79,29 @@ def search_function_call(evaluator, func):
     """
     from jedi.evaluate import representation as er
 
-    def get_params_for_module(module):
-        """
-        Returns the values of a param, or an empty array.
-        """
-        @memoize_default([], evaluator_is_first_arg=True)
-        def get_posibilities(evaluator, module, func_name):
+    def get_possible_nodes(module, func_name):
             try:
                 names = module.used_names[func_name]
             except KeyError:
-                return []
+                return
 
             for name in names:
-                parent = name.parent
-                if tree.is_node(parent, 'trailer'):
-                    parent = parent.parent
+                bracket = name.get_next_leaf()
+                trailer = bracket.parent
+                if trailer.type == 'trailer' and bracket == '(':
+                    yield name, trailer
 
-                trailer = None
-                if tree.is_node(parent, 'power'):
-                    for t in parent.children[1:]:
-                        if t == '**':
-                            break
-                        if t.start_pos > name.start_pos and t.children[0] == '(':
-                            trailer = t
-                            break
-                if trailer is not None:
-                    types = evaluator.goto_definition(name)
-
-                    # We have to remove decorators, because they are not the
-                    # "original" functions, this way we can easily compare.
-                    # At the same time we also have to remove InstanceElements.
-                    undec = []
-                    for escope in types:
-                        if escope.isinstance(er.Function, er.Instance) \
-                                and escope.decorates is not None:
-                            undec.append(escope.decorates)
-                        elif isinstance(escope, er.InstanceElement):
-                            undec.append(escope.var)
-                        else:
-                            undec.append(escope)
-
-                    if evaluator.wrap(compare) in undec:
-                        # Only if we have the correct function we execute
-                        # it, otherwise just ignore it.
-                        evaluator.eval_trailer(types, trailer)
-            return listener.param_possibilities
-        return get_posibilities(evaluator, module, func_name)
+    def undecorate(typ):
+        # We have to remove decorators, because they are not the
+        # "original" functions, this way we can easily compare.
+        # At the same time we also have to remove InstanceElements.
+        if typ.isinstance(er.Function, er.Instance) \
+                and typ.decorates is not None:
+            return typ.decorates
+        elif isinstance(typ, er.InstanceElement):
+            return typ.var
+        else:
+            return typ
 
     current_module = func.get_parent_until()
     func_name = unicode(func.name)
@@ -134,13 +118,32 @@ def search_function_call(evaluator, func):
 
     try:
         result = []
-        # This is like backtracking: Get the first possible result.
+        i = 0
         for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name):
-            result = get_params_for_module(mod)
+            for name, trailer in get_possible_nodes(mod, func_name):
+                i += 1
+
+                # This is a simple way to stop Jedi's dynamic param recursion
+                # from going wild: The deeper Jedi's in the recursin, the less
+                # code should be evaluated.
+                if i * evaluator.dynamic_params_depth > MAX_PARAM_SEARCHES:
+                    return listener.param_possibilities
+
+                for typ in evaluator.goto_definitions(name):
+                    undecorated = undecorate(typ)
+                    if evaluator.wrap(compare) == undecorated:
+                        # Only if we have the correct function we execute
+                        # it, otherwise just ignore it.
+                        evaluator.eval_trailer([typ], trailer)
+
+            result = listener.param_possibilities
+
+            # If there are results after processing a module, we're probably
+            # good to process.
             if result:
-                break
+                return result
     finally:
         # cleanup: remove the listener; important: should not stick.
         func.listeners.remove(listener)
 
-    return result
+    return set()

jedi/evaluate/finder.py

@@ -3,6 +3,9 @@ Searching for names with given scope and name. This is very central in Jedi and
 Python. The name resolution is quite complicated with descripter,
 ``__getattribute__``, ``__getattr__``, ``global``, etc.
 
+If you want to understand name resolution, please read the first few chapters
+in http://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/.
+
 Flow checks
 +++++++++++
 
@@ -13,15 +16,17 @@ check for -> a is a string). There's big potential in these checks.
 """
 from itertools import chain
 
-from jedi._compatibility import unicode, u
+from jedi._compatibility import unicode
 from jedi.parser import tree
 from jedi import debug
 from jedi import common
+from jedi.common import unite
 from jedi import settings
 from jedi.evaluate import representation as er
 from jedi.evaluate import dynamic
 from jedi.evaluate import compiled
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import iterable
 from jedi.evaluate import imports
 from jedi.evaluate import analysis
@@ -53,12 +58,15 @@ def filter_definition_names(names, origin, position=None):
     Filter names that are actual definitions in a scope. Names that are just
     used will be ignored.
     """
+    if not names:
+        return []
+
     # Just calculate the scope from the first
     stmt = names[0].get_definition()
     scope = stmt.get_parent_scope()
 
-    if not (isinstance(scope, er.FunctionExecution)
-            and isinstance(scope.base, er.LambdaWrapper)):
+    if not (isinstance(scope, er.FunctionExecution) and
+            isinstance(scope.base, er.LambdaWrapper)):
         names = filter_after_position(names, position)
     names = [name for name in names if name.is_definition()]
 
@@ -79,25 +87,34 @@ class NameFinder(object):
         self.scope = evaluator.wrap(scope)
         self.name_str = name_str
         self.position = position
+        self._found_predefined_if_name = None
 
     @debug.increase_indent
-    def find(self, scopes, search_global=False):
+    def find(self, scopes, attribute_lookup):
+        """
+        :params bool attribute_lookup: Tell to logic if we're accessing the
+            attribute or the contents of e.g. a function.
+        """
         # TODO rename scopes to names_dicts
+
         names = self.filter_name(scopes)
-        types = self._names_to_types(names, search_global)
+        if self._found_predefined_if_name is not None:
+            return self._found_predefined_if_name
+
+        types = self._names_to_types(names, attribute_lookup)
 
         if not names and not types \
-                and not (isinstance(self.name_str, tree.Name)
-                         and isinstance(self.name_str.parent.parent, tree.Param)):
+                and not (isinstance(self.name_str, tree.Name) and
+                         isinstance(self.name_str.parent.parent, tree.Param)):
             if not isinstance(self.name_str, (str, unicode)):  # TODO Remove?
-                if search_global:
+                if attribute_lookup:
+                    analysis.add_attribute_error(self._evaluator,
+                                                 self.scope, self.name_str)
+                else:
                     message = ("NameError: name '%s' is not defined."
                                % self.name_str)
                     analysis.add(self._evaluator, 'name-error', self.name_str,
                                  message)
-                else:
-                    analysis.add_attribute_error(self._evaluator,
-                                                 self.scope, self.name_str)
 
         debug.dbg('finder._names_to_types: %s -> %s', names, types)
         return types
@@ -143,6 +160,13 @@ class NameFinder(object):
                 last_names.append(name)
                 continue
 
+            if isinstance(stmt, er.ModuleWrapper):
+                # In case of REPL completion, we can infer modules names that
+                # don't really have a definition (because they are really just
+                # namespaces). In this case we can just add it.
+                last_names.append(name)
+                continue
+
             if isinstance(name, compiled.CompiledName) \
                     or isinstance(name, er.InstanceName) and isinstance(name._origin_name, compiled.CompiledName):
                 last_names.append(name)
@@ -150,8 +174,35 @@ class NameFinder(object):
 
             if isinstance(self.name_str, tree.Name):
                 origin_scope = self.name_str.get_parent_until(tree.Scope, reverse=True)
+                scope = self.name_str
+                check = None
+                while True:
+                    scope = scope.parent
+                    if scope.type in ("if_stmt", "for_stmt", "comp_for"):
+                        try:
+                            name_dict = self._evaluator.predefined_if_name_dict_dict[scope]
+                            types = set(name_dict[str(self.name_str)])
+                        except KeyError:
+                            continue
+                        else:
+                            if self.name_str.start_pos < scope.children[1].end_pos:
+                                # It doesn't make any sense to check if
+                                # statements in the if statement itself, just
+                                # deliver types.
+                                self._found_predefined_if_name = types
+                            else:
+                                check = flow_analysis.break_check(self._evaluator, self.scope,
+                                                                  origin_scope)
+                                if check is flow_analysis.UNREACHABLE:
+                                    self._found_predefined_if_name = set()
+                                else:
+                                    self._found_predefined_if_name = types
+                            break
+                    if isinstance(scope, tree.IsScope) or scope is None:
+                        break
             else:
                 origin_scope = None
+
             if isinstance(stmt.parent, compiled.CompiledObject):
                 # TODO seriously? this is stupid.
                 continue
@@ -159,6 +210,7 @@ class NameFinder(object):
                                               stmt, origin_scope)
             if check is not flow_analysis.UNREACHABLE:
                 last_names.append(name)
+
             if check is flow_analysis.REACHABLE:
                 break
 
@@ -179,7 +231,7 @@ class NameFinder(object):
                 break
 
         debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self.name_str,
-                  self.scope, u(names), self.position)
+                  self.scope, names, self.position)
         return list(self._clean_names(names))
 
     def _clean_names(self, names):
@@ -190,29 +242,37 @@ class NameFinder(object):
         """
         for n in names:
             definition = n.parent
-            if isinstance(definition, (tree.Function, tree.Class, tree.Module)):
+            if isinstance(definition, (compiled.CompiledObject,
+                iterable.BuiltinMethod)):
+                # TODO this if should really be removed by changing the type of
+                #      those classes.
+                yield n
+            elif definition.type in ('funcdef', 'classdef', 'file_input'):
                 yield self._evaluator.wrap(definition).name
             else:
                 yield n
 
     def _check_getattr(self, inst):
         """Checks for both __getattr__ and __getattribute__ methods"""
-        result = []
+        result = set()
         # str is important, because it shouldn't be `Name`!
         name = compiled.create(self._evaluator, str(self.name_str))
         with common.ignored(KeyError):
             result = inst.execute_subscope_by_name('__getattr__', name)
         if not result:
-            # this is a little bit special. `__getattribute__` is executed
-            # before anything else. But: I know no use case, where this
-            # could be practical and the jedi would return wrong types. If
-            # you ever have something, let me know!
+            # This is a little bit special. `__getattribute__` is in Python
+            # executed before `__getattr__`. But: I know no use case, where
+            # this could be practical and where jedi would return wrong types.
+            # If you ever find something, let me know!
+            # We are inversing this, because a hand-crafted `__getattribute__`
+            # could still call another hand-crafted `__getattr__`, but not the
+            # other way around.
             with common.ignored(KeyError):
                 result = inst.execute_subscope_by_name('__getattribute__', name)
         return result
 
-    def _names_to_types(self, names, search_global):
-        types = []
+    def _names_to_types(self, names, attribute_lookup):
+        types = set()
 
         # Add isinstance and other if/assert knowledge.
         if isinstance(self.name_str, tree.Name):
@@ -231,13 +291,13 @@ class NameFinder(object):
 
         for name in names:
             new_types = _name_to_types(self._evaluator, name, self.scope)
-            if isinstance(self.scope, (er.Class, er.Instance)) and not search_global:
-                types += self._resolve_descriptors(name, new_types)
+            if isinstance(self.scope, (er.Class, er.Instance)) and attribute_lookup:
+                types |= set(self._resolve_descriptors(name, new_types))
             else:
-                types += new_types
+                types |= set(new_types)
         if not names and isinstance(self.scope, er.Instance):
             # handling __getattr__ / __getattribute__
-            types = self._check_getattr(self.scope)
+            return self._check_getattr(self.scope)
 
         return types
 
@@ -249,56 +309,67 @@ class NameFinder(object):
         if not isinstance(name_scope, (er.Instance, tree.Class)):
             return types
 
-        result = []
+        result = set()
         for r in types:
             try:
                 desc_return = r.get_descriptor_returns
             except AttributeError:
-                result.append(r)
+                result.add(r)
             else:
-                result += desc_return(self.scope)
+                result |= desc_return(self.scope)
         return result
 
 
-@memoize_default([], evaluator_is_first_arg=True)
+def _get_global_stmt_scopes(evaluator, global_stmt, name):
+    global_stmt_scope = global_stmt.get_parent_scope()
+    module = global_stmt_scope.get_parent_until()
+    for used_name in module.used_names[str(name)]:
+        if used_name.parent.type == 'global_stmt':
+            yield evaluator.wrap(used_name.get_parent_scope())
+
+
+@memoize_default(set(), evaluator_is_first_arg=True)
 def _name_to_types(evaluator, name, scope):
     types = []
     typ = name.get_definition()
     if typ.isinstance(tree.ForStmt):
-        for_types = evaluator.eval_element(typ.children[3])
-        for_types = iterable.get_iterator_types(for_types)
-        types += check_tuple_assignments(for_types, name)
-    elif typ.isinstance(tree.CompFor):
-        for_types = evaluator.eval_element(typ.children[3])
-        for_types = iterable.get_iterator_types(for_types)
-        types += check_tuple_assignments(for_types, name)
+        types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.WithStmt):
+        types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.ForStmt, tree.CompFor):
+        container_types = evaluator.eval_element(typ.children[3])
+        for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3])
+        types = check_tuple_assignments(evaluator, for_types, name)
     elif isinstance(typ, tree.Param):
-        types += _eval_param(evaluator, typ, scope)
+        types = _eval_param(evaluator, typ, scope)
     elif typ.isinstance(tree.ExprStmt):
-        types += _remove_statements(evaluator, typ, name)
+        types = _remove_statements(evaluator, typ, name)
     elif typ.isinstance(tree.WithStmt):
-        types += evaluator.eval_element(typ.node_from_name(name))
+        types = evaluator.eval_element(typ.node_from_name(name))
     elif isinstance(typ, tree.Import):
-        types += imports.ImportWrapper(evaluator, name).follow()
-    elif isinstance(typ, tree.GlobalStmt):
-        # TODO theoretically we shouldn't be using search_global here, it
-        # doesn't make sense, because it's a local search (for that name)!
-        # However, globals are not that important and resolving them doesn't
-        # guarantee correctness in any way, because we don't check for when
-        # something is executed.
-        types += evaluator.find_types(typ.get_parent_scope(), str(name),
-                                      search_global=True)
+        types = imports.ImportWrapper(evaluator, name).follow()
+    elif typ.type == 'global_stmt':
+        for s in _get_global_stmt_scopes(evaluator, typ, name):
+            finder = NameFinder(evaluator, s, str(name))
+            names_dicts = finder.scopes(search_global=True)
+            # For global_stmt lookups, we only need the first possible scope,
+            # which means the function itself.
+            names_dicts = [next(names_dicts)]
+            types += finder.find(names_dicts, attribute_lookup=False)
     elif isinstance(typ, tree.TryStmt):
         # TODO an exception can also be a tuple. Check for those.
         # TODO check for types that are not classes and add it to
         # the static analysis report.
-        exceptions = evaluator.eval_element(name.prev_sibling().prev_sibling())
-        types = list(chain.from_iterable(
-                     evaluator.execute(t) for t in exceptions))
+        exceptions = evaluator.eval_element(name.get_previous_sibling().get_previous_sibling())
+        types = set(chain.from_iterable(evaluator.execute(t) for t in exceptions))
     else:
         if typ.isinstance(er.Function):
             typ = typ.get_decorated_func()
-        types.append(typ)
+        types = set([typ])
     return types
 
 
@@ -309,7 +380,7 @@ def _remove_statements(evaluator, stmt, name):
     Due to lazy evaluation, statements like a = func; b = a; b() have to be
     evaluated.
     """
-    types = []
+    types = set()
     # Remove the statement docstr stuff for now, that has to be
     # implemented with the evaluator class.
     #if stmt.docstr:
@@ -320,18 +391,22 @@ def _remove_statements(evaluator, stmt, name):
         check_instance = stmt.instance
         stmt = stmt.var
 
-    types += evaluator.eval_statement(stmt, seek_name=name)
+    pep0484types = \
+        pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name)
+    if pep0484types:
+        return pep0484types
+    types |= evaluator.eval_statement(stmt, seek_name=name)
 
     if check_instance is not None:
         # class renames
-        types = [er.get_instance_el(evaluator, check_instance, a, True)
-                 if isinstance(a, (er.Function, tree.Function))
-                 else a for a in types]
+        types = set([er.get_instance_el(evaluator, check_instance, a, True)
+                     if isinstance(a, (er.Function, tree.Function))
+                     else a for a in types])
     return types
 
 
 def _eval_param(evaluator, param, scope):
-    res_new = []
+    res_new = set()
     func = param.get_parent_scope()
 
     cls = func.parent.get_parent_until((tree.Class, tree.Function))
@@ -342,11 +417,11 @@ def _eval_param(evaluator, param, scope):
         # This is where we add self - if it has never been
         # instantiated.
         if isinstance(scope, er.InstanceElement):
-            res_new.append(scope.instance)
+            res_new.add(scope.instance)
         else:
             inst = er.Instance(evaluator, evaluator.wrap(cls),
                                Arguments(evaluator, ()), is_generated=True)
-            res_new.append(inst)
+            res_new.add(inst)
         return res_new
 
     # Instances are typically faked, if the instance is not called from
@@ -355,23 +430,24 @@ def _eval_param(evaluator, param, scope):
             and func.instance.is_generated and str(func.name) == '__init__':
         param = func.var.params[param.position_nr]
 
-    # Add docstring knowledge.
+    # Add pep0484 and docstring knowledge.
+    pep0484_hints = pep0484.follow_param(evaluator, param)
     doc_params = docstrings.follow_param(evaluator, param)
-    if doc_params:
-        return doc_params
+    if pep0484_hints or doc_params:
+        return list(set(pep0484_hints) | set(doc_params))
 
     if isinstance(param, ExecutedParam):
-        return res_new + param.eval(evaluator)
+        return res_new | param.eval(evaluator)
     else:
         # Param owns no information itself.
-        res_new += dynamic.search_params(evaluator, param)
+        res_new |= dynamic.search_params(evaluator, param)
         if not res_new:
             if param.stars:
                 t = 'tuple' if param.stars == 1 else 'dict'
-                typ = evaluator.find_types(compiled.builtin, t)[0]
+                typ = list(evaluator.find_types(evaluator.BUILTINS, t))[0]
                 res_new = evaluator.execute(typ)
         if param.default:
-            res_new += evaluator.eval_element(param.default)
+            res_new |= evaluator.eval_element(param.default)
         return res_new
 
 
@@ -387,7 +463,7 @@ def check_flow_information(evaluator, flow, search_name, pos):
     if not settings.dynamic_flow_information:
         return None
 
-    result = []
+    result = set()
     if flow.is_scope():
         # Check for asserts.
         try:
@@ -403,14 +479,16 @@ def check_flow_information(evaluator, flow, search_name, pos):
                     break
 
     if isinstance(flow, (tree.IfStmt, tree.WhileStmt)):
-        element = flow.children[1]
-        result = _check_isinstance_type(evaluator, element, search_name)
+        potential_ifs = [c for c in flow.children[1::4] if c != ':']
+        for if_test in reversed(potential_ifs):
+            if search_name.start_pos > if_test.end_pos:
+                return _check_isinstance_type(evaluator, if_test, search_name)
     return result
 
 
 def _check_isinstance_type(evaluator, element, search_name):
     try:
-        assert element.type == 'power'
+        assert element.type in ('power', 'atom_expr')
         # this might be removed if we analyze and, etc
         assert len(element.children) == 2
         first, trailer = element.children
@@ -428,15 +506,18 @@ def _check_isinstance_type(evaluator, element, search_name):
         # Do a simple get_code comparison. They should just have the same code,
         # and everything will be all right.
         classes = lst[1][1][0]
-        call = helpers.call_of_name(search_name)
-        assert name.get_code() == call.get_code()
+        call = helpers.call_of_leaf(search_name)
+        assert name.get_code(normalized=True) == call.get_code(normalized=True)
     except AssertionError:
-        return []
+        return set()
 
-    result = []
-    for typ in evaluator.eval_element(classes):
-        for typ in (typ.values() if isinstance(typ, iterable.Array) else [typ]):
-            result += evaluator.execute(typ)
+    result = set()
+    for cls_or_tup in evaluator.eval_element(classes):
+        if isinstance(cls_or_tup, iterable.Array) and cls_or_tup.type == 'tuple':
+            for typ in unite(cls_or_tup.py__iter__()):
+                result |= evaluator.execute(typ)
+        else:
+            result |= evaluator.execute(cls_or_tup)
     return result
 
 
@@ -449,8 +530,8 @@ def global_names_dict_generator(evaluator, scope, position):
     the current scope is function:
 
     >>> from jedi._compatibility import u, no_unicode_pprint
-    >>> from jedi.parser import Parser, load_grammar
-    >>> parser = Parser(load_grammar(), u('''
+    >>> from jedi.parser import ParserWithRecovery, load_grammar
+    >>> parser = ParserWithRecovery(load_grammar(), u('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None
@@ -499,6 +580,12 @@ def global_names_dict_generator(evaluator, scope, position):
 
             for names_dict in scope.names_dicts(True):
                 yield names_dict, position
+                if hasattr(scope, 'resets_positions'):
+                    # TODO This is so ugly, seriously. However there's
+                    #      currently no good way of influencing
+                    #      global_names_dict_generator when it comes to certain
+                    #      objects.
+                    position = None
             if scope.type == 'funcdef':
                 # The position should be reset if the current scope is a function.
                 in_func = True
@@ -506,28 +593,26 @@ def global_names_dict_generator(evaluator, scope, position):
         scope = evaluator.wrap(scope.get_parent_scope())
 
     # Add builtins to the global scope.
-    for names_dict in compiled.builtin.names_dicts(True):
+    for names_dict in evaluator.BUILTINS.names_dicts(True):
         yield names_dict, None
 
 
-def check_tuple_assignments(types, name):
+def check_tuple_assignments(evaluator, types, name):
     """
     Checks if tuples are assigned.
     """
-    for index in name.assignment_indexes():
-        new_types = []
-        for r in types:
+    for index, node in name.assignment_indexes():
+        iterated = iterable.py__iter__(evaluator, types, node)
+        for _ in range(index + 1):
             try:
-                func = r.get_exact_index_types
-            except AttributeError:
-                debug.warning("Invalid tuple lookup #%s of result %s in %s",
-                              index, types, name)
-            else:
-                try:
-                    new_types += func(index)
-                except IndexError:
-                    pass
-        types = new_types
+                types = next(iterated)
+            except StopIteration:
+                # We could do this with the default param in next. But this
+                # would allow this loop to run for a very long time if the
+                # index number is high. Therefore break if the loop is
+                # finished.
+                types = set()
+                break
     return types
 
 

jedi/evaluate/flow_analysis.py

@@ -45,7 +45,8 @@ def break_check(evaluator, base_scope, stmt, origin_scope=None):
         if element_scope == origin_scope:
             return REACHABLE
         origin_scope = origin_scope.parent
-    return _break_check(evaluator, stmt, base_scope, element_scope)
+    x = _break_check(evaluator, stmt, base_scope, element_scope)
+    return x
 
 
 def _break_check(evaluator, stmt, base_scope, element_scope):
@@ -62,7 +63,8 @@ def _break_check(evaluator, stmt, base_scope, element_scope):
             reachable = reachable.invert()
         else:
             node = element_scope.node_in_which_check_node(stmt)
-            reachable = _check_if(evaluator, node)
+            if node is not None:
+                reachable = _check_if(evaluator, node)
     elif isinstance(element_scope, (tree.TryStmt, tree.WhileStmt)):
         return UNSURE
 
@@ -70,9 +72,14 @@ def _break_check(evaluator, stmt, base_scope, element_scope):
     if reachable in (UNREACHABLE, UNSURE):
         return reachable
 
+    if element_scope.type == 'file_input':
+        # The definition is in another module and therefore just return what we
+        # have generated.
+        return reachable
     if base_scope != element_scope and base_scope != element_scope.parent:
         return reachable & _break_check(evaluator, stmt, base_scope, element_scope.parent)
-    return reachable
+    else:
+        return reachable
 
 
 def _check_if(evaluator, node):

jedi/evaluate/helpers.py

@@ -26,7 +26,8 @@ def deep_ast_copy(obj, parent=None, new_elements=None):
         new_children = []
         for child in obj.children:
             typ = child.type
-            if typ in ('whitespace', 'operator', 'keyword', 'number', 'string'):
+            if typ in ('whitespace', 'operator', 'keyword', 'number', 'string',
+                       'indent', 'dedent', 'error_leaf'):
                 # At the moment we're not actually copying those primitive
                 # elements, because there's really no need to. The parents are
                 # obviously wrong, but that's not an issue.
@@ -55,23 +56,18 @@ def deep_ast_copy(obj, parent=None, new_elements=None):
                     new_names_dict[string] = [new_elements[n] for n in names]
         return new_obj
 
-    if obj.type == 'name':
+    if isinstance(obj, tree.BaseNode):
+        new_obj = copy_node(obj)
+    else:
         # Special case of a Name object.
         new_elements[obj] = new_obj = copy.copy(obj)
-        if parent is not None:
-            new_obj.parent = parent
-    elif isinstance(obj, tree.BaseNode):
-        new_obj = copy_node(obj)
-        if parent is not None:
-            for child in new_obj.children:
-                if isinstance(child, (tree.Name, tree.BaseNode)):
-                    child.parent = parent
-    else:  # String literals and so on.
-        new_obj = obj  # Good enough, don't need to copy anything.
+
+    if parent is not None:
+        new_obj.parent = parent
     return new_obj
 
 
-def call_of_name(name, cut_own_trailer=False):
+def call_of_leaf(leaf, cut_own_trailer=False):
     """
     Creates a "call" node that consist of all ``trailer`` and ``power``
     objects.  E.g. if you call it with ``append``::
@@ -81,28 +77,53 @@ def call_of_name(name, cut_own_trailer=False):
     You would get a node with the content ``list([]).append`` back.
 
     This generates a copy of the original ast node.
+
+    If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
+
+    # TODO remove cut_own_trailer option, since its always used with it. Just
+    #      ignore it, It's not what we want anyway. Or document it better?
     """
-    par = name
-    if tree.is_node(par.parent, 'trailer'):
-        par = par.parent
+    trailer = leaf.parent
+    # The leaf may not be the last or first child, because there exist three
+    # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
+    # we should not match anything more than x.
+    if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]):
+        if trailer.type == 'atom':
+            return trailer
+        return leaf
 
-    power = par.parent
-    if tree.is_node(power, 'power') and power.children[0] != name \
-            and not (power.children[-2] == '**' and
-                     name.start_pos > power.children[-1].start_pos):
-        par = power
-        # Now the name must be part of a trailer
-        index = par.children.index(name.parent)
-        if index != len(par.children) - 1 or cut_own_trailer:
-            # Now we have to cut the other trailers away.
-            par = deep_ast_copy(par)
-            if not cut_own_trailer:
-                # Normally we would remove just the stuff after the index, but
-                # if the option is set remove the index as well. (for goto)
-                index = index + 1
-            par.children[index:] = []
+    power = trailer.parent
+    index = power.children.index(trailer)
+    power = deep_ast_copy(power)
+    if cut_own_trailer:
+        cut = index
+    else:
+        cut = index + 1
+    power.children[cut:] = []
 
-    return par
+    if power.type == 'error_node':
+        start = index
+        while True:
+            start -= 1
+            if power.children[start].type != 'trailer':
+                break
+        transformed = tree.Node('power', power.children[start:])
+        transformed.parent = power.parent
+        return transformed
+
+    return power
+
+
+def get_names_of_node(node):
+    try:
+        children = node.children
+    except AttributeError:
+        if node.type == 'name':
+            return [node]
+        else:
+            return []
+    else:
+        return list(chain.from_iterable(get_names_of_node(c) for c in children))
 
 
 def get_module_names(module, all_scopes):

jedi/evaluate/imports.py

@@ -20,9 +20,9 @@ from itertools import chain
 from jedi._compatibility import find_module, unicode
 from jedi import common
 from jedi import debug
-from jedi import cache
 from jedi.parser import fast
 from jedi.parser import tree
+from jedi.parser.utils import save_parser, load_parser, parser_cache
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi import settings
@@ -70,7 +70,7 @@ class ImportWrapper(tree.Base):
     def follow(self, is_goto=False):
         if self._evaluator.recursion_detector.push_stmt(self._import):
             # check recursion
-            return []
+            return set()
 
         try:
             module = self._evaluator.wrap(self._import.get_parent_until())
@@ -97,7 +97,7 @@ class ImportWrapper(tree.Base):
             #    scopes = [NestedImportModule(module, self._import)]
 
             if from_import_name is not None:
-                types = list(chain.from_iterable(
+                types = set(chain.from_iterable(
                     self._evaluator.find_types(t, unicode(from_import_name),
                                                is_goto=is_goto)
                     for t in types))
@@ -109,11 +109,11 @@ class ImportWrapper(tree.Base):
                     types = importer.follow()
                     # goto only accepts `Name`
                     if is_goto:
-                        types = [s.name for s in types]
+                        types = set(s.name for s in types)
             else:
                 # goto only accepts `Name`
                 if is_goto:
-                    types = [s.name for s in types]
+                    types = set(s.name for s in types)
 
             debug.dbg('after import: %s', types)
         finally:
@@ -201,23 +201,24 @@ class Importer(object):
                 base = []
             if level > len(base):
                 path = module.py__file__()
-                import_path = list(import_path)
-                for i in range(level):
-                    path = os.path.dirname(path)
-                dir_name = os.path.basename(path)
-                # This is not the proper way to do relative imports. However, since
-                # Jedi cannot be sure about the entry point, we just calculate an
-                # absolute path here.
-                if dir_name:
-                    import_path.insert(0, dir_name)
-                else:
-                    _add_error(self._evaluator, import_path[-1])
-                    import_path = []
-                    # TODO add import error.
-                    debug.warning('Attempted relative import beyond top-level package.')
+                if path is not None:
+                    import_path = list(import_path)
+                    for i in range(level):
+                        path = os.path.dirname(path)
+                    dir_name = os.path.basename(path)
+                    # This is not the proper way to do relative imports. However, since
+                    # Jedi cannot be sure about the entry point, we just calculate an
+                    # absolute path here.
+                    if dir_name:
+                        import_path.insert(0, dir_name)
+                    else:
+                        _add_error(self._evaluator, import_path[-1])
+                        import_path = []
+                        # TODO add import error.
+                        debug.warning('Attempted relative import beyond top-level package.')
             else:
                 # Here we basically rewrite the level to 0.
-                import_path = tuple(base) + import_path
+                import_path = tuple(base) + tuple(import_path)
         self.import_path = import_path
 
     @property
@@ -248,7 +249,7 @@ class Importer(object):
     @memoize_default(NO_DEFAULT)
     def follow(self):
         if not self.import_path:
-            return []
+            return set()
         return self._do_import(self.import_path, self.sys_path_with_modifications())
 
     def _do_import(self, import_path, sys_path):
@@ -271,7 +272,7 @@ class Importer(object):
 
         module_name = '.'.join(import_parts)
         try:
-            return [self._evaluator.modules[module_name]]
+            return set([self._evaluator.modules[module_name]])
         except KeyError:
             pass
 
@@ -280,11 +281,11 @@ class Importer(object):
             # the module cache.
             bases = self._do_import(import_path[:-1], sys_path)
             if not bases:
-                return []
+                return set()
             # We can take the first element, because only the os special
             # case yields multiple modules, which is not important for
             # further imports.
-            base = bases[0]
+            base = list(bases)[0]
 
             # This is a huge exception, we follow a nested import
             # ``os.path``, because it's a very important one in Python
@@ -301,7 +302,7 @@ class Importer(object):
             except AttributeError:
                 # The module is not a package.
                 _add_error(self._evaluator, import_path[-1])
-                return []
+                return set()
             else:
                 debug.dbg('search_module %s in paths %s', module_name, paths)
                 for path in paths:
@@ -315,7 +316,7 @@ class Importer(object):
                         module_path = None
                 if module_path is None:
                     _add_error(self._evaluator, import_path[-1])
-                    return []
+                    return set()
         else:
             try:
                 debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
@@ -330,7 +331,7 @@ class Importer(object):
             except ImportError:
                 # The module is not a package.
                 _add_error(self._evaluator, import_path[-1])
-                return []
+                return set()
 
         source = None
         if is_pkg:
@@ -346,11 +347,20 @@ class Importer(object):
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path)
 
+        if module is None:
+            # The file might raise an ImportError e.g. and therefore not be
+            # importable.
+            return set()
+
         self._evaluator.modules[module_name] = module
-        return [module]
+        return set([module])
 
     def _generate_name(self, name):
-        return helpers.FakeName(name, parent=self.module)
+        # Create a pseudo import to be able to follow them.
+        name = helpers.FakeName(name)
+        imp = helpers.FakeImport(name, parent=self.module)
+        name.parent = imp
+        return name
 
     def _get_module_names(self, search_path=None):
         """
@@ -435,7 +445,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
     def load(source):
         dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
         if path is not None and path.endswith('.py') \
-                and not dotted_path in settings.auto_import_modules:
+                and dotted_path not in settings.auto_import_modules:
             if source is None:
                 with open(path, 'rb') as f:
                     source = f.read()
@@ -443,13 +453,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
             return compiled.load_module(evaluator, path)
         p = path
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
-        cache.save_parser(path, p)
+        save_parser(path, p)
         return p.module
 
     if sys_path is None:
         sys_path = evaluator.sys_path
 
-    cached = cache.load_parser(path)
+    cached = load_parser(path)
     module = load(source) if cached is None else cached.module
     module = evaluator.wrap(module)
     return module
@@ -470,7 +480,7 @@ def get_modules_containing_name(evaluator, mods, name):
     """
     def check_python_file(path):
         try:
-            return cache.parser_cache[path].parser.module
+            return parser_cache[path].parser.module
         except KeyError:
             try:
                 return check_fs(path)
@@ -497,7 +507,9 @@ def get_modules_containing_name(evaluator, mods, name):
         paths = set(settings.additional_dynamic_modules)
         for p in mod_paths:
             if p is not None:
-                d = os.path.dirname(p)
+                # We need abspath, because the seetings paths might not already
+                # have been converted to absolute paths.
+                d = os.path.dirname(os.path.abspath(p))
                 for entry in os.listdir(d):
                     if entry not in mod_paths:
                         if entry.endswith('.py'):

jedi/evaluate/iterable.py

@@ -20,74 +20,117 @@ It is important to note that:
 1. Array modfications work only in the current module.
 2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
 """
-from itertools import chain
-
-from jedi import common
+from jedi.common import unite, safe_property
 from jedi import debug
 from jedi import settings
-from jedi._compatibility import use_metaclass, is_py3, unicode
+from jedi._compatibility import use_metaclass, unicode, zip_longest
 from jedi.parser import tree
 from jedi.evaluate import compiled
 from jedi.evaluate import helpers
 from jedi.evaluate.cache import CachedMetaClass, memoize_default
 from jedi.evaluate import analysis
-
-
-def unite(iterable):
-    """Turns a two dimensional array into a one dimensional."""
-    return list(chain.from_iterable(iterable))
+from jedi.evaluate import pep0484
 
 
 class IterableWrapper(tree.Base):
     def is_class(self):
         return False
 
+    @memoize_default()
+    def _get_names_dict(self, names_dict):
+        builtin_methods = {}
+        for cls in reversed(type(self).mro()):
+            try:
+                builtin_methods.update(cls.builtin_methods)
+            except AttributeError:
+                pass
 
+        if not builtin_methods:
+            return names_dict
+
+        dct = {}
+        for names in names_dict.values():
+            for name in names:
+                name_str = name.value
+                try:
+                    method = builtin_methods[name_str, self.type]
+                except KeyError:
+                    dct[name_str] = [name]
+                else:
+                    parent = BuiltinMethod(self, method, name.parent)
+                    dct[name_str] = [helpers.FakeName(name_str, parent, is_definition=True)]
+        return dct
+
+
+class BuiltinMethod(IterableWrapper):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    def __init__(self, builtin, method, builtin_func):
+        self._builtin = builtin
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, params):
+        return self._method(self._builtin)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+def has_builtin_methods(cls):
+    cls.builtin_methods = {}
+    for func in cls.__dict__.values():
+        try:
+            cls.builtin_methods.update(func.registered_builtin_methods)
+        except AttributeError:
+            pass
+    return cls
+
+
+def register_builtin_method(method_name, type=None):
+    def wrapper(func):
+        dct = func.__dict__.setdefault('registered_builtin_methods', {})
+        dct[method_name, type] = func
+        return func
+    return wrapper
+
+
+@has_builtin_methods
 class GeneratorMixin(object):
+    type = None
+
+    @register_builtin_method('send')
+    @register_builtin_method('next')
+    @register_builtin_method('__next__')
+    def py__next__(self):
+        # TODO add TypeError if params are given.
+        return unite(self.py__iter__())
+
     @memoize_default()
     def names_dicts(self, search_global=False):  # is always False
-        dct = {}
-        executes_generator = '__next__', 'send', 'next'
-        for names in compiled.generator_obj.names_dict.values():
-            for name in names:
-                if name.value in executes_generator:
-                    parent = GeneratorMethod(self, name.parent)
-                    dct[name.value] = [helpers.FakeName(name.name, parent, is_definition=True)]
-                else:
-                    dct[name.value] = [name]
-        yield dct
-
-    def get_index_types(self, evaluator, index_array):
-        #debug.warning('Tried to get array access on a generator: %s', self)
-        analysis.add(self._evaluator, 'type-error-generator', index_array)
-        return []
-
-    def get_exact_index_types(self, index):
-        """
-        Exact lookups are used for tuple lookups, which are perfectly fine if
-        used with generators.
-        """
-        return [self.iter_content()[index]]
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        yield self._get_names_dict(gen_obj.names_dict)
 
     def py__bool__(self):
         return True
 
+    def py__class__(self):
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        return gen_obj.py__class__()
+
 
 class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)):
     """Handling of `yield` functions."""
+
     def __init__(self, evaluator, func, var_args):
         super(Generator, self).__init__()
         self._evaluator = evaluator
         self.func = func
         self.var_args = var_args
 
-    def iter_content(self):
-        """ returns the content of __iter__ """
-        # Directly execute it, because with a normal call to py__call__ a
-        # Generator will be returned.
+    def py__iter__(self):
         from jedi.evaluate.representation import FunctionExecution
         f = FunctionExecution(self._evaluator, self.func, self.var_args)
-        return f.get_return_types(check_yields=True)
+        return f.get_yield_types()
 
     def __getattr__(self, name):
         if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
@@ -101,90 +144,172 @@ class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin))
         return "<%s of %s>" % (type(self).__name__, self.func)
 
 
-class GeneratorMethod(IterableWrapper):
-    """``__next__`` and ``send`` methods."""
-    def __init__(self, generator, builtin_func):
-        self._builtin_func = builtin_func
-        self._generator = generator
-
-    def py__call__(self, evaluator, params):
-        # TODO add TypeError if params are given.
-        return self._generator.iter_content()
-
-    def __getattr__(self, name):
-        return getattr(self._builtin_func, name)
-
-
 class Comprehension(IterableWrapper):
     @staticmethod
     def from_atom(evaluator, atom):
-        mapping = {
-            '(': GeneratorComprehension,
-            '[': ListComprehension
-        }
-        return mapping[atom.children[0]](evaluator, atom)
+        bracket = atom.children[0]
+        if bracket == '{':
+            if atom.children[1].children[1] == ':':
+                cls = DictComprehension
+            else:
+                cls = SetComprehension
+        elif bracket == '(':
+            cls = GeneratorComprehension
+        elif bracket == '[':
+            cls = ListComprehension
+        return cls(evaluator, atom)
 
     def __init__(self, evaluator, atom):
         self._evaluator = evaluator
         self._atom = atom
 
+    def _get_comprehension(self):
+        # The atom contains a testlist_comp
+        return self._atom.children[1]
+
+    def _get_comp_for(self):
+        # The atom contains a testlist_comp
+        return self._get_comprehension().children[1]
+
     @memoize_default()
-    def eval_node(self):
+    def _eval_node(self, index=0):
         """
         The first part `x + 1` of the list comprehension:
 
             [x + 1 for x in foo]
         """
-        comprehension = self._atom.children[1]
+        comp_for = self._get_comp_for()
         # For nested comprehensions we need to search the last one.
-        last = comprehension.children[-1]
-        last_comp = comprehension.children[1]
-        while True:
-            if isinstance(last, tree.CompFor):
-                last_comp = last
-            elif not tree.is_node(last, 'comp_if'):
-                break
-            last = last.children[-1]
+        last_comp = list(comp_for.get_comp_fors())[-1]
+        return helpers.deep_ast_copy(self._get_comprehension().children[index], parent=last_comp)
 
-        return helpers.deep_ast_copy(comprehension.children[0], parent=last_comp)
+    @memoize_default()
+    def _iterate(self):
+        def nested(comp_fors):
+            comp_for = comp_fors[0]
+            input_node = comp_for.children[3]
+            input_types = evaluator.eval_element(input_node)
 
-    def get_exact_index_types(self, index):
-        return [self._evaluator.eval_element(self.eval_node())[index]]
+            iterated = py__iter__(evaluator, input_types, input_node)
+            exprlist = comp_for.children[1]
+            for types in iterated:
+                evaluator.predefined_if_name_dict_dict[comp_for] = \
+                    unpack_tuple_to_dict(evaluator, types, exprlist)
+                try:
+                    for result in nested(comp_fors[1:]):
+                        yield result
+                except IndexError:
+                    iterated = evaluator.eval_element(self._eval_node())
+                    if self.type == 'dict':
+                        yield iterated, evaluator.eval_element(self._eval_node(2))
+                    else:
+                        yield iterated
+                finally:
+                    del evaluator.predefined_if_name_dict_dict[comp_for]
+
+        evaluator = self._evaluator
+        comp_fors = list(self._get_comp_for().get_comp_fors())
+        for result in nested(comp_fors):
+            yield result
+
+    def py__iter__(self):
+        return self._iterate()
 
     def __repr__(self):
-        return "<e%s of %s>" % (type(self).__name__, self._atom)
+        return "<%s of %s>" % (type(self).__name__, self._atom)
 
 
+@has_builtin_methods
 class ArrayMixin(object):
     @memoize_default()
     def names_dicts(self, search_global=False):  # Always False.
         # `array.type` is a string with the type, e.g. 'list'.
-        scope = self._evaluator.find_types(compiled.builtin, self.type)[0]
+        scope = compiled.builtin_from_name(self._evaluator, self.type)
         # builtins only have one class -> [0]
-        scope = self._evaluator.execute(scope, (AlreadyEvaluated((self,)),))[0]
-        return scope.names_dicts(search_global)
+        scopes = self._evaluator.execute_evaluated(scope, self)
+        names_dicts = list(scopes)[0].names_dicts(search_global)
+        #yield names_dicts[0]
+        yield self._get_names_dict(names_dicts[1])
 
     def py__bool__(self):
         return None  # We don't know the length, because of appends.
 
+    def py__class__(self):
+        return compiled.builtin_from_name(self._evaluator, self.type)
+
+    @safe_property
+    def parent(self):
+        return self._evaluator.BUILTINS
+
+    @property
+    def name(self):
+        return FakeSequence(self._evaluator, [], self.type).name
+
+    @memoize_default()
+    def dict_values(self):
+        return unite(self._evaluator.eval_element(v) for k, v in self._items())
+
+    @register_builtin_method('values', type='dict')
+    def _imitate_values(self):
+        items = self.dict_values()
+        return create_evaluated_sequence_set(self._evaluator, items, type='list')
+        #return set([FakeSequence(self._evaluator, [AlreadyEvaluated(items)], 'tuple')])
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = [set([FakeSequence(self._evaluator, (k, v), 'tuple')])
+                 for k, v in self._items()]
+
+        return create_evaluated_sequence_set(self._evaluator, *items, type='list')
+
 
 class ListComprehension(Comprehension, ArrayMixin):
     type = 'list'
 
-    def get_index_types(self, evaluator, index):
-        return self.iter_content()
+    def py__getitem__(self, index):
+        all_types = list(self.py__iter__())
+        return all_types[index]
+
+
+class SetComprehension(Comprehension, ArrayMixin):
+    type = 'set'
+
+
+@has_builtin_methods
+class DictComprehension(Comprehension, ArrayMixin):
+    type = 'dict'
+
+    def _get_comp_for(self):
+        return self._get_comprehension().children[3]
+
+    def py__iter__(self):
+        for keys, values in self._iterate():
+            yield keys
+
+    def py__getitem__(self, index):
+        for keys, values in self._iterate():
+            for k in keys:
+                if isinstance(k, compiled.CompiledObject):
+                    if k.obj == index:
+                        return values
+        return self.dict_values()
+
+    def dict_values(self):
+        return unite(values for keys, values in self._iterate())
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = set(FakeSequence(self._evaluator,
+                    (AlreadyEvaluated(keys), AlreadyEvaluated(values)), 'tuple')
+                    for keys, values in self._iterate())
+
+        return create_evaluated_sequence_set(self._evaluator, items, type='list')
 
-    def iter_content(self):
-        return self._evaluator.eval_element(self.eval_node())
 
-    @property
-    def name(self):
-        return FakeSequence(self._evaluator, [], 'list').name
 
 
 class GeneratorComprehension(Comprehension, GeneratorMixin):
-    def iter_content(self):
-        return self._evaluator.eval_element(self.eval_node())
+    pass
 
 
 class Array(IterableWrapper, ArrayMixin):
@@ -209,60 +334,21 @@ class Array(IterableWrapper, ArrayMixin):
     def name(self):
         return helpers.FakeName(self.type, parent=self)
 
-    @memoize_default()
-    def get_index_types(self, evaluator, index=()):
-        """
-        Get the types of a specific index or all, if not given.
-
-        :param index: A subscriptlist node (or subnode).
-        """
-        indexes = create_indexes_or_slices(evaluator, index)
-        lookup_done = False
-        types = []
-        for index in indexes:
-            if isinstance(index, Slice):
-                types += [self]
-                lookup_done = True
-            elif isinstance(index, compiled.CompiledObject) \
-                    and isinstance(index.obj, (int, str, unicode)):
-                with common.ignored(KeyError, IndexError, TypeError):
-                    types += self.get_exact_index_types(index.obj)
-                    lookup_done = True
-
-        return types if lookup_done else self.values()
-
-    @memoize_default()
-    def values(self):
-        result = unite(self._evaluator.eval_element(v) for v in self._values())
-        result += check_array_additions(self._evaluator, self)
-        return result
-
-    def get_exact_index_types(self, mixed_index):
-        """ Here the index is an int/str. Raises IndexError/KeyError """
+    def py__getitem__(self, index):
+        """Here the index is an int/str. Raises IndexError/KeyError."""
         if self.type == 'dict':
-            for key, values in self._items():
-                # Because we only want the key to be a string.
-                keys = self._evaluator.eval_element(key)
-
-                for k in keys:
+            for key, value in self._items():
+                for k in self._evaluator.eval_element(key):
                     if isinstance(k, compiled.CompiledObject) \
-                            and mixed_index == k.obj:
-                        for value in values:
-                            return self._evaluator.eval_element(value)
+                            and index == k.obj:
+                        return self._evaluator.eval_element(value)
             raise KeyError('No key found in dictionary %s.' % self)
 
         # Can raise an IndexError
-        return self._evaluator.eval_element(self._items()[mixed_index])
-
-    def iter_content(self):
-        return self.values()
-
-    @common.safe_property
-    def parent(self):
-        return compiled.builtin
-
-    def get_parent_until(self):
-        return compiled.builtin
+        if isinstance(index, slice):
+            return set([self])
+        else:
+            return self._evaluator.eval_element(self._items()[index])
 
     def __getattr__(self, name):
         if name not in ['start_pos', 'get_only_subelement', 'parent',
@@ -270,10 +356,33 @@ class Array(IterableWrapper, ArrayMixin):
             raise AttributeError('Strange access on %s: %s.' % (self, name))
         return getattr(self.atom, name)
 
+    # @memoize_default()
+    def py__iter__(self):
+        """
+        While values returns the possible values for any array field, this
+        function returns the value for a certain index.
+        """
+        if self.type == 'dict':
+            # Get keys.
+            types = set()
+            for k, _ in self._items():
+                types |= self._evaluator.eval_element(k)
+            # We don't know which dict index comes first, therefore always
+            # yield all the types.
+            for _ in types:
+                yield types
+        else:
+            for value in self._items():
+                yield self._evaluator.eval_element(value)
+
+            additions = check_array_additions(self._evaluator, self)
+            if additions:
+                yield additions
+
     def _values(self):
         """Returns a list of a list of node."""
         if self.type == 'dict':
-            return list(chain.from_iterable(v for k, v in self._items()))
+            return unite(v for k, v in self._items())
         else:
             return self._items()
 
@@ -292,18 +401,14 @@ class Array(IterableWrapper, ArrayMixin):
                 op = next(iterator, None)
                 if op is None or op == ',':
                     kv.append(key)  # A set.
-                elif op == ':':  # A dict.
-                    kv.append((key, [next(iterator)]))
-                    next(iterator, None)  # Possible comma.
                 else:
-                    raise NotImplementedError('dict/set comprehensions')
+                    assert op == ':'  # A dict.
+                    kv.append((key, next(iterator)))
+                    next(iterator, None)  # Possible comma.
             return kv
         else:
             return [array_node]
 
-    def __iter__(self):
-        return iter(self._items())
-
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.atom)
 
@@ -326,20 +431,29 @@ class ImplicitTuple(_FakeArray):
 
 class FakeSequence(_FakeArray):
     def __init__(self, evaluator, sequence_values, type):
+        """
+        type should be one of "tuple", "list"
+        """
         super(FakeSequence, self).__init__(evaluator, sequence_values, type)
         self._sequence_values = sequence_values
 
     def _items(self):
         return self._sequence_values
 
-    def get_exact_index_types(self, index):
-        value = self._sequence_values[index]
-        return self._evaluator.eval_element(value)
+
+def create_evaluated_sequence_set(evaluator, *types_order, **kwargs):
+    """
+    ``sequence_type`` is a named argument, that doesn't work in Python2. For backwards
+    compatibility reasons, we're now using kwargs.
+    """
+    sequence_type = kwargs.get('sequence_type')
+    sets = tuple(AlreadyEvaluated(types) for types in types_order)
+    return set([FakeSequence(evaluator, sets, sequence_type)])
 
 
 class AlreadyEvaluated(frozenset):
     """A simple container to add already evaluated objects to an array."""
-    def get_code(self):
+    def get_code(self, normalized=False):
         # For debugging purposes.
         return str(self)
 
@@ -353,12 +467,16 @@ class FakeDict(_FakeArray):
         super(FakeDict, self).__init__(evaluator, dct, 'dict')
         self._dct = dct
 
-    def get_exact_index_types(self, index):
-        return list(chain.from_iterable(self._evaluator.eval_element(v)
-                                        for v in self._dct[index]))
+    def py__iter__(self):
+        yield set(compiled.create(self._evaluator, key) for key in self._dct)
+
+    def py__getitem__(self, index):
+        return unite(self._evaluator.eval_element(v) for v in self._dct[index])
 
     def _items(self):
-        return self._dct.items()
+        for key, values in self._dct.items():
+            # TODO this is not proper. The values could be multiple values?!
+            yield key, values[0]
 
 
 class MergedArray(_FakeArray):
@@ -366,56 +484,138 @@ class MergedArray(_FakeArray):
         super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].type)
         self._arrays = arrays
 
-    def get_exact_index_types(self, mixed_index):
-        raise IndexError
-
-    def values(self):
-        return list(chain(*(a.values() for a in self._arrays)))
-
-    def __iter__(self):
+    def py__iter__(self):
         for array in self._arrays:
-            for a in array:
+            for types in array.py__iter__():
+                yield types
+
+    def py__getitem__(self, index):
+        return unite(self.py__iter__())
+
+    def _items(self):
+        for array in self._arrays:
+            for a in array._items():
                 yield a
 
     def __len__(self):
         return sum(len(a) for a in self._arrays)
 
 
-def get_iterator_types(inputs):
-    """Returns the types of any iterator (arrays, yields, __iter__, etc)."""
-    iterators = []
-    # Take the first statement (for has always only
-    # one, remember `in`). And follow it.
-    for it in inputs:
-        if isinstance(it, (Generator, Array, ArrayInstance, Comprehension)):
-            iterators.append(it)
-        else:
-            if not hasattr(it, 'execute_subscope_by_name'):
-                debug.warning('iterator/for loop input wrong: %s', it)
-                continue
+def unpack_tuple_to_dict(evaluator, types, exprlist):
+    """
+    Unpacking tuple assignments in for statements and expr_stmts.
+    """
+    if exprlist.type == 'name':
+        return {exprlist.value: types}
+    elif exprlist.type == 'atom' and exprlist.children[0] in '([':
+        return unpack_tuple_to_dict(evaluator, types, exprlist.children[1])
+    elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
+                           'testlist_star_expr'):
+        dct = {}
+        parts = iter(exprlist.children[::2])
+        n = 0
+        for iter_types in py__iter__(evaluator, types, exprlist):
+            n += 1
             try:
-                iterators += it.execute_subscope_by_name('__iter__')
-            except KeyError:
-                debug.warning('iterators: No __iter__ method found.')
+                part = next(parts)
+            except StopIteration:
+                analysis.add(evaluator, 'value-error-too-many-values', part,
+                             message="ValueError: too many values to unpack (expected %s)" % n)
+            else:
+                dct.update(unpack_tuple_to_dict(evaluator, iter_types, part))
+        has_parts = next(parts, None)
+        if types and has_parts is not None:
+            analysis.add(evaluator, 'value-error-too-few-values', has_parts,
+                         message="ValueError: need more than %s values to unpack" % n)
+        return dct
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
+        # Something like ``arr[x], var = ...``.
+        # This is something that is not yet supported, would also be difficult
+        # to write into a dict.
+        return {}
+    elif exprlist.type == 'star_expr':  # `a, *b, c = x` type unpackings
+        # Currently we're not supporting them.
+        return {}
+    raise NotImplementedError
 
-    result = []
-    from jedi.evaluate.representation import Instance
-    for it in iterators:
-        if isinstance(it, Array):
-            # Array is a little bit special, since this is an internal array,
-            # but there's also the list builtin, which is another thing.
-            result += it.values()
-        elif isinstance(it, Instance):
-            # __iter__ returned an instance.
-            name = '__next__' if is_py3 else 'next'
-            try:
-                result += it.execute_subscope_by_name(name)
-            except KeyError:
-                debug.warning('Instance has no __next__ function in %s.', it)
+
+def py__iter__(evaluator, types, node=None):
+    debug.dbg('py__iter__')
+    type_iters = []
+    for typ in types:
+        try:
+            iter_method = typ.py__iter__
+        except AttributeError:
+            if node is not None:
+                analysis.add(evaluator, 'type-error-not-iterable', node,
+                             message="TypeError: '%s' object is not iterable" % typ)
         else:
-            # TODO this is not correct, __iter__ can return arbitrary input!
-            # Is a generator.
-            result += it.iter_content()
+            type_iters.append(iter_method())
+            #for result in iter_method():
+                #yield result
+
+    for t in zip_longest(*type_iters, fillvalue=set()):
+        yield unite(t)
+
+
+def py__iter__types(evaluator, types, node=None):
+    """
+    Calls `py__iter__`, but ignores the ordering in the end and just returns
+    all types that it contains.
+    """
+    return unite(py__iter__(evaluator, types, node))
+
+
+def py__getitem__(evaluator, types, trailer):
+    from jedi.evaluate.representation import Class
+    result = set()
+
+    trailer_op, node, trailer_cl = trailer.children
+    assert trailer_op == "["
+    assert trailer_cl == "]"
+
+    # special case: PEP0484 typing module, see
+    # https://github.com/davidhalter/jedi/issues/663
+    for typ in list(types):
+        if isinstance(typ, Class):
+            typing_module_types = \
+                pep0484.get_types_for_typing_module(evaluator, typ, node)
+            if typing_module_types is not None:
+                types.remove(typ)
+                result |= typing_module_types
+
+    if not types:
+        # all consumed by special cases
+        return result
+
+    for index in create_index_types(evaluator, node):
+        if isinstance(index, (compiled.CompiledObject, Slice)):
+            index = index.obj
+
+        if type(index) not in (float, int, str, unicode, slice):
+            # If the index is not clearly defined, we have to get all the
+            # possiblities.
+            for typ in list(types):
+                if isinstance(typ, Array) and typ.type == 'dict':
+                    types.remove(typ)
+                    result |= typ.dict_values()
+            return result | py__iter__types(evaluator, types)
+
+        for typ in types:
+            # The actual getitem call.
+            try:
+                getitem = typ.py__getitem__
+            except AttributeError:
+                analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op,
+                             message="TypeError: '%s' object is not subscriptable" % typ)
+            else:
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= py__iter__types(evaluator, set([typ]))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= typ.dict_values()
     return result
 
 
@@ -423,7 +623,7 @@ def check_array_additions(evaluator, array):
     """ Just a mapper function for the internal _check_array_additions """
     if array.type not in ('list', 'set'):
         # TODO also check for dict updates
-        return []
+        return set()
 
     is_list = array.type == 'list'
     try:
@@ -432,11 +632,12 @@ def check_array_additions(evaluator, array):
         # If there's no get_parent_until, it's a FakeSequence or another Fake
         # type. Those fake types are used inside Jedi's engine. No values may
         # be added to those after their creation.
-        return []
+        return set()
     return _check_array_additions(evaluator, array, current_module, is_list)
 
 
-@memoize_default([], evaluator_is_first_arg=True)
+@memoize_default(default=set(), evaluator_is_first_arg=True)
+@debug.increase_indent
 def _check_array_additions(evaluator, compare_array, module, is_list):
     """
     Checks if a `Array` has "add" (append, insert, extend) statements:
@@ -444,21 +645,24 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
     >>> a = [""]
     >>> a.append(1)
     """
+    debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA')
     if not settings.dynamic_array_additions or isinstance(module, compiled.CompiledObject):
-        return []
+        debug.dbg('Dynamic array search aborted.', color='MAGENTA')
+        return set()
 
     def check_additions(arglist, add_name):
         params = list(param.Arguments(evaluator, arglist).unpack())
-        result = []
+        result = set()
         if add_name in ['insert']:
             params = params[1:]
         if add_name in ['append', 'add', 'insert']:
             for key, nodes in params:
-                result += unite(evaluator.eval_element(node) for node in nodes)
+                result |= unite(evaluator.eval_element(node) for node in nodes)
         elif add_name in ['extend', 'update']:
             for key, nodes in params:
-                iterators = unite(evaluator.eval_element(node) for node in nodes)
-                result += get_iterator_types(iterators)
+                for node in nodes:
+                    types = evaluator.eval_element(node)
+                    result |= py__iter__types(evaluator, types, node)
         return result
 
     from jedi.evaluate import representation as er, param
@@ -469,7 +673,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
             node = element.atom
         else:
             # Is an Instance with an
-            # Arguments([AlreadyEvaluated([ArrayInstance])]) inside
+            # Arguments([AlreadyEvaluated([_ArrayInstance])]) inside
             # Yeah... I know... It's complicated ;-)
             node = list(element.var_args.argument_node[0])[0].var_args.trailer
         if isinstance(node, er.InstanceElement):
@@ -482,7 +686,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
     search_names = ['append', 'extend', 'insert'] if is_list else ['add', 'update']
     comp_arr_parent = get_execution_parent(compare_array)
 
-    added_types = []
+    added_types = set()
     for add_name in search_names:
         try:
             possible_names = module.used_names[add_name]
@@ -515,7 +719,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
                             or execution_trailer.children[0] != '(' \
                             or execution_trailer.children[1] == ')':
                         continue
-                power = helpers.call_of_name(name, cut_own_trailer=True)
+                power = helpers.call_of_leaf(name, cut_own_trailer=True)
                 # InstanceElements are special, because they don't get copied,
                 # but have this wrapper around them.
                 if isinstance(comp_arr_parent, er.InstanceElement):
@@ -527,11 +731,12 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
                     continue
                 if compare_array in evaluator.eval_element(power):
                     # The arrays match. Now add the results
-                    added_types += check_additions(execution_trailer.children[1], add_name)
+                    added_types |= check_additions(execution_trailer.children[1], add_name)
 
                 evaluator.recursion_detector.pop_stmt()
     # reset settings
     settings.dynamic_params_for_other_modules = temp_param_add
+    debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
     return added_types
 
 
@@ -540,12 +745,12 @@ def check_array_instances(evaluator, instance):
     if not settings.dynamic_array_additions:
         return instance.var_args
 
-    ai = ArrayInstance(evaluator, instance)
+    ai = _ArrayInstance(evaluator, instance)
     from jedi.evaluate import param
     return param.Arguments(evaluator, [AlreadyEvaluated([ai])])
 
 
-class ArrayInstance(IterableWrapper):
+class _ArrayInstance(IterableWrapper):
     """
     Used for the usage of set() and list().
     This is definitely a hack, but a good one :-)
@@ -561,21 +766,21 @@ class ArrayInstance(IterableWrapper):
         self.instance = instance
         self.var_args = instance.var_args
 
-    def iter_content(self):
-        """
-        The index is here just ignored, because of all the appends, etc.
-        lists/sets are too complicated too handle that.
-        """
-        items = []
-        for key, nodes in self.var_args.unpack():
-            for node in nodes:
-                for typ in self._evaluator.eval_element(node):
-                    items += get_iterator_types([typ])
+    def py__iter__(self):
+        try:
+            _, first_nodes = next(self.var_args.unpack())
+        except StopIteration:
+            types = set()
+        else:
+            types = unite(self._evaluator.eval_element(node) for node in first_nodes)
+            for types in py__iter__(self._evaluator, types, first_nodes[0]):
+                yield types
 
         module = self.var_args.get_parent_until()
         is_list = str(self.instance.name) == 'list'
-        items += _check_array_additions(self._evaluator, self.instance, module, is_list)
-        return items
+        additions = _check_array_additions(self._evaluator, self.instance, module, is_list)
+        if additions:
+            yield additions
 
 
 class Slice(object):
@@ -598,11 +803,11 @@ class Slice(object):
 
             result = self._evaluator.eval_element(element)
             if len(result) != 1:
-                # We want slices to be clear defined with just one type.
-                # Otherwise we will return an empty slice object.
+                # For simplicity, we want slices to be clear defined with just
+                # one type.  Otherwise we will return an empty slice object.
                 raise IndexError
             try:
-                return result[0].obj
+                return list(result)[0].obj
             except AttributeError:
                 return None
 
@@ -612,9 +817,15 @@ class Slice(object):
             return slice(None, None, None)
 
 
-def create_indexes_or_slices(evaluator, index):
-    if tree.is_node(index, 'subscript'):  # subscript is a slice operation.
-        start, stop, step = None, None, None
+def create_index_types(evaluator, index):
+    """
+    Handles slices in subscript nodes.
+    """
+    if index == ':':
+        # Like array[:]
+        return set([Slice(evaluator, None, None, None)])
+    elif tree.is_node(index, 'subscript'):  # subscript is a slice operation.
+        # Like array[:3]
         result = []
         for el in index.children:
             if el == ':':
@@ -627,5 +838,7 @@ def create_indexes_or_slices(evaluator, index):
                 result.append(el)
         result += [None] * (3 - len(result))
 
-        return (Slice(evaluator, *result),)
+        return set([Slice(evaluator, *result)])
+
+    # No slices
     return evaluator.eval_element(index)

jedi/evaluate/jedi_typing.py

@@ -0,0 +1,100 @@
+"""
+This module is not intended to be used in jedi, rather it will be fed to the
+jedi-parser to replace classes in the typing module
+"""
+
+try:
+    from collections import abc
+except ImportError:
+    # python 2
+    import collections as abc
+
+
+def factory(typing_name, indextypes):
+    class Iterable(abc.Iterable):
+        def __iter__(self):
+            while True:
+                yield indextypes[0]()
+
+    class Iterator(Iterable, abc.Iterator):
+        def next(self):
+            """ needed for python 2 """
+            return self.__next__()
+
+        def __next__(self):
+            return indextypes[0]()
+
+    class Sequence(abc.Sequence):
+        def __getitem__(self, index):
+            return indextypes[0]()
+
+    class MutableSequence(Sequence, abc.MutableSequence):
+        pass
+
+    class List(MutableSequence, list):
+        pass
+
+    class Tuple(Sequence, tuple):
+        def __getitem__(self, index):
+            if indextypes[1] == Ellipsis:
+                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
+                # Tuple[int, ...] means a tuple of ints of indetermined length
+                return indextypes[0]()
+            else:
+                return indextypes[index]()
+
+    class AbstractSet(Iterable, abc.Set):
+        pass
+
+    class MutableSet(AbstractSet, abc.MutableSet):
+        pass
+
+    class KeysView(Iterable, abc.KeysView):
+        pass
+
+    class ValuesView(abc.ValuesView):
+        def __iter__(self):
+            while True:
+                yield indextypes[1]()
+
+    class ItemsView(abc.ItemsView):
+        def __iter__(self):
+            while True:
+                yield (indextypes[0](), indextypes[1]())
+
+    class Mapping(Iterable, abc.Mapping):
+        def __getitem__(self, item):
+            return indextypes[1]()
+
+        def keys(self):
+            return KeysView()
+
+        def values(self):
+            return ValuesView()
+
+        def items(self):
+            return ItemsView()
+
+    class MutableMapping(Mapping, abc.MutableMapping):
+        pass
+
+    class Dict(MutableMapping, dict):
+        pass
+
+    dct = {
+        "Sequence": Sequence,
+        "MutableSequence": MutableSequence,
+        "List": List,
+        "Iterable": Iterable,
+        "Iterator": Iterator,
+        "AbstractSet": AbstractSet,
+        "MutableSet": MutableSet,
+        "Mapping": Mapping,
+        "MutableMapping": MutableMapping,
+        "Tuple": Tuple,
+        "KeysView": KeysView,
+        "ItemsView": ItemsView,
+        "ValuesView": ValuesView,
+        "Dict": Dict,
+    }
+    return dct[typing_name]

jedi/evaluate/param.py

@@ -12,6 +12,23 @@ from jedi.evaluate.helpers import FakeName
 from jedi.cache import underscore_memoization
 
 
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for iter_types in f():
+                try_iter_content(iter_types, depth + 1)
+
+
 class Arguments(tree.Base):
     def __init__(self, evaluator, argument_node, trailer=None):
         """
@@ -30,7 +47,10 @@ class Arguments(tree.Base):
             for el in self.argument_node:
                 yield 0, el
         else:
-            if not tree.is_node(self.argument_node, 'arglist'):
+            if not (tree.is_node(self.argument_node, 'arglist') or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (tree.is_node(self.argument_node, 'argument') and
+                     self.argument_node.children[0] in ('*', '**')))):
                 yield 0, self.argument_node
                 return
 
@@ -40,6 +60,10 @@ class Arguments(tree.Base):
                     continue
                 elif child in ('*', '**'):
                     yield len(child.value), next(iterator)
+                elif tree.is_node(child, 'argument') and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
                 else:
                     yield 0, child
 
@@ -49,7 +73,7 @@ class Arguments(tree.Base):
                 element = self.argument_node[0]
                 from jedi.evaluate.iterable import AlreadyEvaluated
                 if isinstance(element, AlreadyEvaluated):
-                    element = self._evaluator.eval_element(element)[0]
+                    element = list(self._evaluator.eval_element(element))[0]
             except IndexError:
                 return None
             else:
@@ -131,8 +155,8 @@ class Arguments(tree.Base):
                 debug.warning('TypeError: %s expected at least %s arguments, got %s',
                               name, len(arguments), i)
                 raise ValueError
-            values = list(chain.from_iterable(self._evaluator.eval_element(el)
-                                              for el in va_values))
+            values = set(chain.from_iterable(self._evaluator.eval_element(el)
+                                             for el in va_values))
             if not values and not optional:
                 # For the stdlib we always want values. If we don't get them,
                 # that's ok, maybe something is too hard to resolve, however,
@@ -160,6 +184,16 @@ class Arguments(tree.Base):
         else:
             return None
 
+    def eval_all(self, func=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, element_values in self.unpack():
+            for element in element_values:
+                types = self._evaluator.eval_element(element)
+                try_iter_content(types)
+
 
 class ExecutedParam(tree.Param):
     """Fake a param and give it values."""
@@ -169,9 +203,9 @@ class ExecutedParam(tree.Param):
         self._values = values
 
     def eval(self, evaluator):
-        types = []
+        types = set()
         for v in self._values:
-            types += evaluator.eval_element(v)
+            types |= evaluator.eval_element(v)
         return types
 
     @property
@@ -242,7 +276,7 @@ def get_params(evaluator, func, var_args):
             try:
                 key_param = param_dict[unicode(key)]
             except KeyError:
-                non_matching_keys[key] += va_values
+                non_matching_keys[key] = va_values
             else:
                 param_names.append(ExecutedParam(key_param, var_args, va_values).name)
 
@@ -353,11 +387,12 @@ def get_params(evaluator, func, var_args):
 def _iterate_star_args(evaluator, array, input_node, func=None):
     from jedi.evaluate.representation import Instance
     if isinstance(array, iterable.Array):
-        for field_stmt in array:  # yield from plz!
-            yield field_stmt
+        # TODO ._items is not the call we want here. Replace in the future.
+        for node in array._items():
+            yield node
     elif isinstance(array, iterable.Generator):
-        for field_stmt in array.iter_content():
-            yield iterable.AlreadyEvaluated([field_stmt])
+        for types in array.py__iter__():
+            yield iterable.AlreadyEvaluated(types)
     elif isinstance(array, Instance) and array.name.get_code() == 'tuple':
         debug.warning('Ignored a tuple *args input %s' % array)
     else:
@@ -379,10 +414,10 @@ def _star_star_dict(evaluator, array, input_node, func):
         return array._dct
     elif isinstance(array, iterable.Array) and array.type == 'dict':
         # TODO bad call to non-public API
-        for key_node, values in array._items():
+        for key_node, value in array._items():
             for key in evaluator.eval_element(key_node):
                 if precedence.is_string(key):
-                    dct[key.obj] += values
+                    dct[key.obj].append(value)
 
     else:
         if func is not None:

jedi/evaluate/pep0484.py

@@ -0,0 +1,195 @@
+"""
+PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
+through function annotations. There is a strong suggestion in this document
+that only the type of type hinting defined in PEP0484 should be allowed
+as annotations in future python versions.
+
+The (initial / probably incomplete) implementation todo list for pep-0484:
+v Function parameter annotations with builtin/custom type classes
+v Function returntype annotations with builtin/custom type classes
+v Function parameter annotations with strings (forward reference)
+v Function return type annotations with strings (forward reference)
+v Local variable type hints
+v Assigned types: `Url = str\ndef get(url:Url) -> str:`
+v Type hints in `with` statements
+x Stub files support
+x support `@no_type_check` and `@no_type_check_decorator`
+x support for typing.cast() operator
+x support for type hint comments for functions, `# type: (int, str) -> int`.
+    See comment from Guido https://github.com/davidhalter/jedi/issues/662
+"""
+
+import itertools
+
+import os
+from jedi.parser import \
+    Parser, load_grammar, ParseError, ParserWithRecovery, tree
+from jedi.evaluate.cache import memoize_default
+from jedi.common import unite
+from jedi.evaluate import compiled
+from jedi import debug
+from jedi import _compatibility
+import re
+
+
+def _evaluate_for_annotation(evaluator, annotation, index=None):
+    """
+    Evaluates a string-node, looking for an annotation
+    If index is not None, the annotation is expected to be a tuple
+    and we're interested in that index
+    """
+    if annotation is not None:
+        definitions = evaluator.eval_element(
+            _fix_forward_reference(evaluator, annotation))
+        if index is not None:
+            definitions = list(itertools.chain.from_iterable(
+                definition.py__getitem__(index) for definition in definitions
+                if definition.type == 'tuple' and
+                len(list(definition.py__iter__())) >= index))
+        return list(itertools.chain.from_iterable(
+            evaluator.execute(d) for d in definitions))
+    else:
+        return []
+
+
+def _fix_forward_reference(evaluator, node):
+    evaled_nodes = evaluator.eval_element(node)
+    if len(evaled_nodes) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (node, evaled_nodes))
+        return node
+    evaled_node = list(evaled_nodes)[0]
+    if isinstance(evaled_node, compiled.CompiledObject) and \
+            isinstance(evaled_node.obj, str):
+        try:
+            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
+                       start_symbol='eval_input')
+            newnode = p.get_parsed_node()
+        except ParseError:
+            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
+            return node
+        else:
+            module = node.get_parent_until()
+            p.position_modifier.line = module.end_pos[0]
+            newnode.parent = module
+            return newnode
+    else:
+        return node
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def follow_param(evaluator, param):
+    annotation = param.annotation()
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def find_return_types(evaluator, func):
+    annotation = func.py__annotations__().get("return", None)
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+_typing_module = None
+
+
+def _get_typing_replacement_module():
+    """
+    The idea is to return our jedi replacement for the PEP-0484 typing module
+    as discussed at https://github.com/davidhalter/jedi/issues/663
+    """
+    global _typing_module
+    if _typing_module is None:
+        typing_path = \
+            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
+        with open(typing_path) as f:
+            code = _compatibility.unicode(f.read())
+        p = ParserWithRecovery(load_grammar(), code)
+        _typing_module = p.module
+    return _typing_module
+
+
+def get_types_for_typing_module(evaluator, typ, node):
+    from jedi.evaluate.iterable import FakeSequence
+    if not typ.base.get_parent_until().name.value == "typing":
+        return None
+    # we assume that any class using [] in a module called
+    # "typing" with a name for which we have a replacement
+    # should be replaced by that class. This is not 100%
+    # airtight but I don't have a better idea to check that it's
+    # actually the PEP-0484 typing module and not some other
+    if tree.is_node(node, "subscriptlist"):
+        nodes = node.children[::2]  # skip the commas
+    else:
+        nodes = [node]
+    del node
+
+    nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
+
+    # hacked in Union and Optional, since it's hard to do nicely in parsed code
+    if typ.name.value == "Union":
+        return unite(evaluator.eval_element(node) for node in nodes)
+    if typ.name.value == "Optional":
+        return evaluator.eval_element(nodes[0])
+
+    typing = _get_typing_replacement_module()
+    factories = evaluator.find_types(typing, "factory")
+    assert len(factories) == 1
+    factory = list(factories)[0]
+    assert factory
+    function_body_nodes = factory.children[4].children
+    valid_classnames = set(child.name.value
+                           for child in function_body_nodes
+                           if isinstance(child, tree.Class))
+    if typ.name.value not in valid_classnames:
+        return None
+    compiled_classname = compiled.create(evaluator, typ.name.value)
+
+    args = FakeSequence(evaluator, nodes, "tuple")
+
+    result = evaluator.execute_evaluated(factory, compiled_classname, args)
+    return result
+
+
+def find_type_from_comment_hint_for(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[1], name)
+
+
+def find_type_from_comment_hint_with(evaluator, node, name):
+    assert len(node.children[1].children) == 3, \
+        "Can only be here when children[1] is 'foo() as f'"
+    return _find_type_from_comment_hint(
+        evaluator, node, node.children[1].children[2], name)
+
+
+def find_type_from_comment_hint_assign(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[0], name)
+
+
+def _find_type_from_comment_hint(evaluator, node, varlist, name):
+    index = None
+    if varlist.type in ("testlist_star_expr", "exprlist"):
+        # something like "a, b = 1, 2"
+        index = 0
+        for child in varlist.children:
+            if child == name:
+                break
+            if child.type == "operator":
+                continue
+            index += 1
+        else:
+            return []
+
+    comment = node.get_following_comment_same_line()
+    if comment is None:
+        return []
+    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
+    if not match:
+        return []
+    annotation = tree.String(
+        tree.zero_position_modifier,
+        repr(str(match.group(1).strip())),
+        node.start_pos)
+    annotation.parent = node.parent
+    return _evaluate_for_annotation(evaluator, annotation, index)

jedi/evaluate/precedence.py

@@ -6,8 +6,7 @@ import operator
 from jedi._compatibility import unicode
 from jedi.parser import tree
 from jedi import debug
-from jedi.evaluate.compiled import (CompiledObject, create, builtin,
-                                    keyword_from_value, true_obj, false_obj)
+from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
 from jedi.evaluate import analysis
 
 # Maps Python syntax to the operator module.
@@ -23,16 +22,19 @@ COMPARISON_OPERATORS = {
 }
 
 
-def _literals_to_types(evaluator, result):
+def literals_to_types(evaluator, result):
     # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
     # int(), float(), etc).
-    for i, r in enumerate(result):
-        if is_literal(r):
+    new_result = set()
+    for typ in result:
+        if is_literal(typ):
             # Literals are only valid as long as the operations are
             # correct. Otherwise add a value-free instance.
-            cls = builtin.get_by_name(r.name.get_code())
-            result[i] = evaluator.execute(cls)[0]
-    return list(set(result))
+            cls = builtin_from_name(evaluator, typ.name.value)
+            new_result |= evaluator.execute(cls)
+        else:
+            new_result.add(typ)
+    return new_result
 
 
 def calculate_children(evaluator, children):
@@ -64,21 +66,21 @@ def calculate_children(evaluator, children):
 
 
 def calculate(evaluator, left_result, operator, right_result):
-    result = []
+    result = set()
     if not left_result or not right_result:
         # illegal slices e.g. cause left/right_result to be None
-        result = (left_result or []) + (right_result or [])
-        result = _literals_to_types(evaluator, result)
+        result = (left_result or set()) | (right_result or set())
+        result = literals_to_types(evaluator, result)
     else:
         # I don't think there's a reasonable chance that a string
         # operation is still correct, once we pass something like six
         # objects.
         if len(left_result) * len(right_result) > 6:
-            result = _literals_to_types(evaluator, left_result + right_result)
+            result = literals_to_types(evaluator, left_result | right_result)
         else:
             for left in left_result:
                 for right in right_result:
-                    result += _element_calculate(evaluator, left, operator, right)
+                    result |= _element_calculate(evaluator, left, operator, right)
     return result
 
 
@@ -94,7 +96,7 @@ def factor_calculate(evaluator, types, operator):
             value = typ.py__bool__()
             if value is None:  # Uncertainty.
                 return
-            yield keyword_from_value(not value)
+            yield create(evaluator, not value)
         else:
             yield typ
 
@@ -130,21 +132,21 @@ def _element_calculate(evaluator, left, operator, right):
     if operator == '*':
         # for iterables, ignore * operations
         if isinstance(left, iterable.Array) or is_string(left):
-            return [left]
+            return set([left])
         elif isinstance(right, iterable.Array) or is_string(right):
-            return [right]
+            return set([right])
     elif operator == '+':
         if l_is_num and r_is_num or is_string(left) and is_string(right):
-            return [create(evaluator, left.obj + right.obj)]
+            return set([create(evaluator, left.obj + right.obj)])
         elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
-            return [iterable.MergedArray(evaluator, (left, right))]
+            return set([iterable.MergedArray(evaluator, (left, right))])
     elif operator == '-':
         if l_is_num and r_is_num:
-            return [create(evaluator, left.obj - right.obj)]
+            return set([create(evaluator, left.obj - right.obj)])
     elif operator == '%':
         # With strings and numbers the left type typically remains. Except for
         # `int() % float()`.
-        return [left]
+        return set([left])
     elif operator in COMPARISON_OPERATORS:
         operation = COMPARISON_OPERATORS[operator]
         if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
@@ -153,12 +155,14 @@ def _element_calculate(evaluator, left, operator, right):
             right = right.obj
 
         try:
-            return [keyword_from_value(operation(left, right))]
+            result = operation(left, right)
         except TypeError:
             # Could be True or False.
-            return [true_obj, false_obj]
+            return set([create(evaluator, True), create(evaluator, False)])
+        else:
+            return set([create(evaluator, result)])
     elif operator == 'in':
-        return []
+        return set()
 
     def check(obj):
         """Checks if a Jedi object is either a float or an int."""
@@ -171,4 +175,4 @@ def _element_calculate(evaluator, left, operator, right):
         analysis.add(evaluator, 'type-error-operation', operator,
                      message % (left, right))
 
-    return [left, right]
+    return set([left, right])

jedi/evaluate/recursion.py

@@ -9,7 +9,6 @@ count the function calls.
 """
 from jedi import debug
 from jedi import settings
-from jedi.evaluate import compiled
 from jedi.evaluate import iterable
 
 
@@ -18,7 +17,7 @@ def recursion_decorator(func):
         rec_detect = evaluator.recursion_detector
         # print stmt, len(self.node_statements())
         if rec_detect.push_stmt(stmt):
-            return []
+            return set()
         else:
             result = func(evaluator, stmt, *args, **kwargs)
             rec_detect.pop_stmt()
@@ -31,12 +30,13 @@ class RecursionDetector(object):
     A decorator to detect recursions in statements. In a recursion a statement
     at the same place, in the same module may not be executed two times.
     """
-    def __init__(self):
+    def __init__(self, evaluator):
         self.top = None
         self.current = None
+        self._evaluator = evaluator
 
     def push_stmt(self, stmt):
-        self.current = _RecursionNode(stmt, self.current)
+        self.current = _RecursionNode(self._evaluator, stmt, self.current)
         check = self._check_recursion()
         if check:
             debug.warning('catched stmt recursion: %s against %s @%s', stmt,
@@ -71,7 +71,8 @@ class RecursionDetector(object):
 
 class _RecursionNode(object):
     """ A node of the RecursionDecorator. """
-    def __init__(self, stmt, parent):
+    def __init__(self, evaluator, stmt, parent):
+        self._evaluator = evaluator
         self.script = stmt.get_parent_until()
         self.position = stmt.start_pos
         self.parent = parent
@@ -80,7 +81,7 @@ class _RecursionNode(object):
         # Don't check param instances, they are not causing recursions
         # The same's true for the builtins, because the builtins are really
         # simple.
-        self.is_ignored = self.script == compiled.builtin
+        self.is_ignored = self.script == self._evaluator.BUILTINS
 
     def __eq__(self, other):
         if not other:
@@ -95,7 +96,7 @@ def execution_recursion_decorator(func):
     def run(execution, **kwargs):
         detector = execution._evaluator.execution_recursion_detector
         if detector.push_execution(execution):
-            result = []
+            result = set()
         else:
             result = func(execution, **kwargs)
         detector.pop_execution()
@@ -107,51 +108,51 @@ def execution_recursion_decorator(func):
 class ExecutionRecursionDetector(object):
     """
     Catches recursions of executions.
-    It is designed like a Singelton. Only one instance should exist.
     """
-    def __init__(self):
+    def __init__(self, evaluator):
         self.recursion_level = 0
         self.parent_execution_funcs = []
         self.execution_funcs = set()
         self.execution_count = 0
+        self._evaluator = evaluator
 
     def __call__(self, execution):
         debug.dbg('Execution recursions: %s', execution, self.recursion_level,
                   self.execution_count, len(self.execution_funcs))
         if self.check_recursion(execution):
-            result = []
+            result = set()
         else:
             result = self.func(execution)
         self.pop_execution()
         return result
 
-    def pop_execution(cls):
-        cls.parent_execution_funcs.pop()
-        cls.recursion_level -= 1
+    def pop_execution(self):
+        self.parent_execution_funcs.pop()
+        self.recursion_level -= 1
 
-    def push_execution(cls, execution):
-        in_par_execution_funcs = execution.base in cls.parent_execution_funcs
-        in_execution_funcs = execution.base in cls.execution_funcs
-        cls.recursion_level += 1
-        cls.execution_count += 1
-        cls.execution_funcs.add(execution.base)
-        cls.parent_execution_funcs.append(execution.base)
+    def push_execution(self, execution):
+        in_par_execution_funcs = execution.base in self.parent_execution_funcs
+        in_execution_funcs = execution.base in self.execution_funcs
+        self.recursion_level += 1
+        self.execution_count += 1
+        self.execution_funcs.add(execution.base)
+        self.parent_execution_funcs.append(execution.base)
 
-        if cls.execution_count > settings.max_executions:
+        if self.execution_count > settings.max_executions:
             return True
 
         if isinstance(execution.base, (iterable.Array, iterable.Generator)):
             return False
         module = execution.get_parent_until()
-        if module == compiled.builtin:
+        if module == self._evaluator.BUILTINS:
             return False
 
         if in_par_execution_funcs:
-            if cls.recursion_level > settings.max_function_recursion_level:
+            if self.recursion_level > settings.max_function_recursion_level:
                 return True
         if in_execution_funcs and \
-                len(cls.execution_funcs) > settings.max_until_execution_unique:
+                len(self.execution_funcs) > settings.max_until_execution_unique:
             return True
-        if cls.execution_count > settings.max_executions_without_builtins:
+        if self.execution_count > settings.max_executions_without_builtins:
             return True
         return False

jedi/evaluate/representation.py

@@ -17,13 +17,18 @@ and others. Here's a list:
 ====================================== ========================================
 **Method**                             **Description**
 -------------------------------------- ----------------------------------------
-py__call__(evaluator, params: Array)   On callable objects, returns types.
+py__call__(params: Array)              On callable objects, returns types.
 py__bool__()                           Returns True/False/None; None means that
                                        there's no certainty.
-py__bases__(evaluator)                 Returns a list of base classes.
-py__mro__(evaluator)                   Returns a list of classes (the mro).
-py__getattribute__(evaluator, name)    Returns a list of attribute values. The
-                                       name can be str or Name.
+py__bases__()                          Returns a list of base classes.
+py__mro__()                            Returns a list of classes (the mro).
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__getitem__(index: int/str)          Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__file__()                           Only on modules.
+py__package__()                        Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
 ====================================== ========================================
 
 __
@@ -34,16 +39,18 @@ import imp
 import re
 from itertools import chain
 
-from jedi._compatibility import use_metaclass, unicode, Python3Method
+from jedi._compatibility import use_metaclass, unicode, Python3Method, is_py3
 from jedi.parser import tree
 from jedi import debug
 from jedi import common
 from jedi.cache import underscore_memoization, cache_star_import
 from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT
 from jedi.evaluate import compiled
+from jedi.evaluate.compiled import mixed
 from jedi.evaluate import recursion
 from jedi.evaluate import iterable
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import helpers
 from jedi.evaluate import param
 from jedi.evaluate import flow_analysis
@@ -83,7 +90,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         self.is_generated = is_generated
 
         if base.name.get_code() in ['list', 'set'] \
-                and compiled.builtin == base.get_parent_until():
+                and evaluator.BUILTINS == base.get_parent_until():
             # compare the module path with the builtin name.
             self.var_args = iterable.check_array_instances(evaluator, self)
         elif not is_generated:
@@ -96,10 +103,13 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
             else:
                 evaluator.execute(method, self.var_args)
 
+    def is_class(self):
+        return False
+
     @property
     def py__call__(self):
-        def actual(evaluator, params):
-            return evaluator.execute(method, params)
+        def actual(params):
+            return self._evaluator.execute(method, params)
 
         try:
             method = self.get_subscope_by_name('__call__')
@@ -109,7 +119,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
 
         return actual
 
-    def py__class__(self, evaluator):
+    def py__class__(self):
         return self.base
 
     def py__bool__(self):
@@ -153,8 +163,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
                     sub = self._get_method_execution(sub)
             for name_list in sub.names_dict.values():
                 for name in name_list:
-                    if name.value == self_name and name.prev_sibling() is None:
-                        trailer = name.next_sibling()
+                    if name.value == self_name and name.get_previous_sibling() is None:
+                        trailer = name.get_next_sibling()
                         if tree.is_node(trailer, 'trailer') \
                                 and len(trailer.children) == 2 \
                                 and trailer.children[0] == '.':
@@ -176,17 +186,18 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         """ Throws a KeyError if there's no method. """
         # Arguments in __get__ descriptors are obj, class.
         # `method` is the new parent of the array, don't know if that's good.
-        args = [obj, obj.base] if isinstance(obj, Instance) else [compiled.none_obj, obj]
+        none_obj = compiled.create(self._evaluator, None)
+        args = [obj, obj.base] if isinstance(obj, Instance) else [none_obj, obj]
         try:
             return self.execute_subscope_by_name('__get__', *args)
         except KeyError:
-            return [self]
+            return set([self])
 
     @memoize_default()
     def names_dicts(self, search_global):
         yield self._self_names_dict()
 
-        for s in self.base.py__mro__(self._evaluator)[1:]:
+        for s in self.base.py__mro__()[1:]:
             if not isinstance(s, compiled.CompiledObject):
                 # Compiled objects don't have `self.` names.
                 for inst in self._evaluator.execute(s):
@@ -195,21 +206,35 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         for names_dict in self.base.names_dicts(search_global=False, is_instance=True):
             yield LazyInstanceDict(self._evaluator, self, names_dict)
 
-    def get_index_types(self, evaluator, index_array):
-        indexes = iterable.create_indexes_or_slices(self._evaluator, index_array)
-        if any([isinstance(i, iterable.Slice) for i in indexes]):
-            # Slice support in Jedi is very marginal, at the moment, so just
-            # ignore them in case of __getitem__.
-            # TODO support slices in a more general way.
-            indexes = []
-
+    def py__getitem__(self, index):
         try:
             method = self.get_subscope_by_name('__getitem__')
         except KeyError:
             debug.warning('No __getitem__, cannot access the array.')
-            return []
+            return set()
         else:
-            return self._evaluator.execute(method, [iterable.AlreadyEvaluated(indexes)])
+            index_obj = compiled.create(self._evaluator, index)
+            return self._evaluator.execute_evaluated(method, index_obj)
+
+    def py__iter__(self):
+        try:
+            method = self.get_subscope_by_name('__iter__')
+        except KeyError:
+            debug.warning('No __iter__ on %s.' % self)
+            return
+        else:
+            iters = self._evaluator.execute(method)
+            for generator in iters:
+                if isinstance(generator, Instance):
+                    # `__next__` logic.
+                    name = '__next__' if is_py3 else 'next'
+                    try:
+                        yield generator.execute_subscope_by_name(name)
+                    except KeyError:
+                        debug.warning('Instance has no __next__ function in %s.', generator)
+                else:
+                    for typ in generator.py__iter__():
+                        yield typ
 
     @property
     @underscore_memoization
@@ -228,8 +253,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         dec = ''
         if self.decorates is not None:
             dec = " decorates " + repr(self.decorates)
-        return "<e%s of %s(%s)%s>" % (type(self).__name__, self.base,
-                                      self.var_args, dec)
+        return "<%s of %s(%s)%s>" % (type(self).__name__, self.base,
+                                     self.var_args, dec)
 
 
 class LazyInstanceDict(object):
@@ -355,13 +380,13 @@ class InstanceElement(use_metaclass(CachedMetaClass, tree.Base)):
         """
         return self.var.is_scope()
 
-    def py__call__(self, evaluator, params):
+    def py__call__(self, params):
         if isinstance(self.var, compiled.CompiledObject):
             # This check is a bit strange, but CompiledObject itself is a bit
             # more complicated than we would it actually like to be.
-            return self.var.py__call__(evaluator, params)
+            return self.var.py__call__(params)
         else:
-            return Function.py__call__(self, evaluator, params)
+            return Function.py__call__(self, params)
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.var)
@@ -398,7 +423,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         self.base = base
 
     @memoize_default(default=())
-    def py__mro__(self, evaluator):
+    def py__mro__(self):
         def add(cls):
             if cls not in mro:
                 mro.append(cls)
@@ -406,7 +431,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         mro = [self]
         # TODO Do a proper mro resolution. Currently we are just listing
         # classes. However, it's a complicated algorithm.
-        for cls in self.py__bases__(self._evaluator):
+        for cls in self.py__bases__():
             # TODO detect for TypeError: duplicate base class str,
             # e.g.  `class X(str, str): pass`
             try:
@@ -426,24 +451,24 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
                 pass
             else:
                 add(cls)
-                for cls_new in mro_method(evaluator):
+                for cls_new in mro_method():
                     add(cls_new)
         return tuple(mro)
 
     @memoize_default(default=())
-    def py__bases__(self, evaluator):
+    def py__bases__(self):
         arglist = self.base.get_super_arglist()
         if arglist:
             args = param.Arguments(self._evaluator, arglist)
             return list(chain.from_iterable(args.eval_args()))
         else:
-            return [compiled.object_obj]
+            return [compiled.create(self._evaluator, object)]
 
-    def py__call__(self, evaluator, params):
-        return [Instance(evaluator, self, params)]
+    def py__call__(self, params):
+        return set([Instance(self._evaluator, self, params)])
 
-    def py__getattribute__(self, name):
-        return self._evaluator.find_types(self, name)
+    def py__class__(self):
+        return compiled.create(self._evaluator, type)
 
     @property
     def params(self):
@@ -453,7 +478,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         if search_global:
             yield self.names_dict
         else:
-            for scope in self.py__mro__(self._evaluator):
+            for scope in self.py__mro__():
                 if isinstance(scope, compiled.CompiledObject):
                     yield scope.names_dicts(False, is_instance)[0]
                 else:
@@ -463,7 +488,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         return True
 
     def get_subscope_by_name(self, name):
-        for s in self.py__mro__(self._evaluator):
+        for s in self.py__mro__():
             for sub in reversed(s.subscopes):
                 if sub.name.value == name:
                         return sub
@@ -527,8 +552,10 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
                 # Create param array.
                 if isinstance(f, Function):
                     old_func = f  # TODO this is just hacky. change.
-                else:
+                elif f.type == 'funcdef':
                     old_func = Function(self._evaluator, f, is_decorated=True)
+                else:
+                    old_func = f
 
                 wrappers = self._evaluator.execute_evaluated(decorator, old_func)
                 if not len(wrappers):
@@ -538,7 +565,7 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
                     # TODO resolve issue with multiple wrappers -> multiple types
                     debug.warning('multiple wrappers found %s %s',
                                   self.base_func, wrappers)
-                f = wrappers[0]
+                f = list(wrappers)[0]
                 if isinstance(f, (Instance, Function)):
                     f.decorates = self
 
@@ -549,15 +576,33 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
         if search_global:
             yield self.names_dict
         else:
-            for names_dict in compiled.magic_function_class.names_dicts(False):
+            scope = compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS')
+            for names_dict in scope.names_dicts(False):
                 yield names_dict
 
     @Python3Method
-    def py__call__(self, evaluator, params):
+    def py__call__(self, params):
         if self.base.is_generator():
-            return [iterable.Generator(evaluator, self, params)]
+            return set([iterable.Generator(self._evaluator, self, params)])
         else:
-            return FunctionExecution(evaluator, self, params).get_return_types()
+            return FunctionExecution(self._evaluator, self, params).get_return_types()
+
+    @memoize_default()
+    def py__annotations__(self):
+        parser_func = self.base
+        return_annotation = parser_func.annotation()
+        if return_annotation:
+            dct = {'return': return_annotation}
+        else:
+            dct = {}
+        for function_param in parser_func.params:
+            param_annotation = function_param.annotation()
+            if param_annotation is not None:
+                dct[function_param.name.value] = param_annotation
+        return dct
+
+    def py__class__(self):
+        return compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS')
 
     def __getattr__(self, name):
         return getattr(self.base_func, name)
@@ -588,11 +633,22 @@ class FunctionExecution(Executed):
     def __init__(self, evaluator, base, *args, **kwargs):
         super(FunctionExecution, self).__init__(evaluator, base, *args, **kwargs)
         self._copy_dict = {}
-        new_func = helpers.deep_ast_copy(base.base_func, self, self._copy_dict)
-        self.children = new_func.children
-        self.names_dict = new_func.names_dict
+        funcdef = base.base_func
+        if isinstance(funcdef, mixed.MixedObject):
+            # The extra information in mixed is not needed anymore. We can just
+            # unpack it and give it the tree object.
+            funcdef = funcdef.definition
 
-    @memoize_default(default=())
+        # Just overwrite the old version. We don't need it anymore.
+        funcdef = helpers.deep_ast_copy(funcdef, new_elements=self._copy_dict)
+        for child in funcdef.children:
+            if child.type not in ('operator', 'keyword'):
+                # Not all nodes are properly copied by deep_ast_copy.
+                child.parent = self
+        self.children = funcdef.children
+        self.names_dict = funcdef.names_dict
+
+    @memoize_default(default=set())
     @recursion.execution_recursion_decorator
     def get_return_types(self, check_yields=False):
         func = self.base
@@ -607,26 +663,87 @@ class FunctionExecution(Executed):
             # If we do have listeners, that means that there's not a regular
             # execution ongoing. In this case Jedi is interested in the
             # inserted params, not in the actual execution of the function.
-            return []
+            return set()
 
         if check_yields:
-            types = []
+            types = set()
             returns = self.yields
         else:
             returns = self.returns
-            types = list(docstrings.find_return_types(self._evaluator, func))
+            types = set(docstrings.find_return_types(self._evaluator, func))
+            types |= set(pep0484.find_return_types(self._evaluator, func))
 
         for r in returns:
             check = flow_analysis.break_check(self._evaluator, self, r)
             if check is flow_analysis.UNREACHABLE:
                 debug.dbg('Return unreachable: %s', r)
             else:
-                types += self._evaluator.eval_element(r.children[1])
+                if check_yields:
+                    types |= iterable.unite(self._eval_yield(r))
+                else:
+                    types |= self._evaluator.eval_element(r.children[1])
             if check is flow_analysis.REACHABLE:
                 debug.dbg('Return reachable: %s', r)
                 break
         return types
 
+    def _eval_yield(self, yield_expr):
+        element = yield_expr.children[1]
+        if element.type == 'yield_arg':
+            # It must be a yield from.
+            yield_from_types = self._evaluator.eval_element(element.children[1])
+            for result in iterable.py__iter__(self._evaluator, yield_from_types, element):
+                yield result
+        else:
+            yield self._evaluator.eval_element(element)
+
+    @recursion.execution_recursion_decorator
+    def get_yield_types(self):
+        yields = self.yields
+        stopAt = tree.ForStmt, tree.WhileStmt, FunctionExecution, tree.IfStmt
+        for_parents = [(x, x.get_parent_until((stopAt))) for x in yields]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self \
+                    and for_stmt.defines_one_name():  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self:
+                yields_order.append((None, [yield_]))
+            else:
+                yield self.get_return_types(check_yields=True)
+                return
+            last_for_stmt = for_stmt
+
+        evaluator = self._evaluator
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._eval_yield(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_input_node()
+                for_types = evaluator.eval_element(input_node)
+                ordered = iterable.py__iter__(evaluator, for_types, input_node)
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    evaluator.predefined_if_name_dict_dict[for_stmt] = dct
+                    for yield_in_same_for_stmt in yields:
+                        for result in self._eval_yield(yield_in_same_for_stmt):
+                            yield result
+                    del evaluator.predefined_if_name_dict_dict[for_stmt]
+
     def names_dicts(self, search_global):
         yield self.names_dict
 
@@ -646,50 +763,28 @@ class FunctionExecution(Executed):
     def name_for_position(self, position):
         return tree.Function.name_for_position(self, position)
 
-    def _copy_list(self, lst):
-        """
-        Copies a list attribute of a parser Function. Copying is very
-        expensive, because it is something like `copy.deepcopy`. However, these
-        copied objects can be used for the executions, as if they were in the
-        execution.
-        """
-        objects = []
-        for element in lst:
-            self._scope_copy(element.parent)
-            copied = helpers.deep_ast_copy(element, self._copy_dict)
-            objects.append(copied)
-        return objects
-
     def __getattr__(self, name):
         if name not in ['start_pos', 'end_pos', 'imports', 'name', 'type']:
             raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
         return getattr(self.base, name)
 
-    def _scope_copy(self, scope):
-        raise NotImplementedError
-        """ Copies a scope (e.g. `if foo:`) in an execution """
-        if scope != self.base.base_func:
-            # Just make sure the parents been copied.
-            self._scope_copy(scope.parent)
-            helpers.deep_ast_copy(scope, self._copy_dict)
-
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def returns(self):
         return tree.Scope._search_in_scope(self, tree.ReturnStmt)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def yields(self):
         return tree.Scope._search_in_scope(self, tree.YieldExpr)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def statements(self):
         return tree.Scope._search_in_scope(self, tree.ExprStmt)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def subscopes(self):
         return tree.Scope._search_in_scope(self, tree.Scope)
 
@@ -742,7 +837,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
     @memoize_default()
     def _module_attributes_dict(self):
         def parent_callback():
-            return self._evaluator.execute(compiled.create(self._evaluator, str))[0]
+            # Create a string type object (without a defined string in it):
+            return list(self._evaluator.execute(compiled.create(self._evaluator, str)))[0]
 
         names = ['__file__', '__package__', '__doc__', '__name__']
         # All the additional module attributes are strings.
@@ -757,7 +853,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
     def _get_init_directory(self):
         for suffix, _, _ in imp.get_suffixes():
             ending = '__init__' + suffix
-            if self.py__file__().endswith(ending):
+            py__file__ = self.py__file__()
+            if py__file__ is not None and py__file__.endswith(ending):
                 # Remove the ending, including the separator.
                 return self.py__file__()[:-len(ending) - 1]
         return None
@@ -850,6 +947,9 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
 
         return names
 
+    def py__class__(self):
+        return compiled.get_special_object(self._evaluator, 'MODULE_CLASS')
+
     def __getattr__(self, name):
         return getattr(self._module, name)
 

jedi/evaluate/stdlib.py

@@ -5,32 +5,37 @@ understand them with Jedi.
 To add a new implementation, create a function and add it to the
 ``_implemented`` dict at the bottom of this module.
 
+Note that this module exists only to implement very specific functionality in
+the standard library. The usual way to understand the standard library is the
+compiled module that returns the types for C-builtins.
 """
 import collections
 import re
 
 from jedi._compatibility import unicode
+from jedi.common import unite
 from jedi.evaluate import compiled
 from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
 from jedi import debug
 from jedi.evaluate import precedence
 from jedi.evaluate import param
+from jedi.evaluate import analysis
 
 
 class NotInStdLib(LookupError):
     pass
 
 
-def execute(evaluator, obj, params):
+def execute(evaluator, obj, arguments):
     try:
         obj_name = str(obj.name)
     except AttributeError:
         pass
     else:
-        if obj.parent == compiled.builtin:
+        if obj.parent == evaluator.BUILTINS:
             module_name = 'builtins'
         elif isinstance(obj.parent, tree.Module):
             module_name = str(obj.parent.name)
@@ -39,22 +44,24 @@ def execute(evaluator, obj, params):
 
         # for now we just support builtin functions.
         try:
-            return _implemented[module_name][obj_name](evaluator, obj, params)
+            func = _implemented[module_name][obj_name]
         except KeyError:
             pass
+        else:
+            return func(evaluator, obj, arguments)
     raise NotInStdLib()
 
 
-def _follow_param(evaluator, params, index):
+def _follow_param(evaluator, arguments, index):
     try:
-        key, values = list(params.unpack())[index]
+        key, values = list(arguments.unpack())[index]
     except IndexError:
-        return []
+        return set()
     else:
-        return iterable.unite(evaluator.eval_element(v) for v in values)
+        return unite(evaluator.eval_element(v) for v in values)
 
 
-def argument_clinic(string, want_obj=False, want_scope=False):
+def argument_clinic(string, want_obj=False, want_scope=False, want_arguments=False):
     """
     Works like Argument Clinic (PEP 436), to validate function params.
     """
@@ -77,17 +84,22 @@ def argument_clinic(string, want_obj=False, want_scope=False):
 
     def f(func):
         def wrapper(evaluator, obj, arguments):
+            debug.dbg('builtin start %s' % obj, color='MAGENTA')
             try:
                 lst = list(arguments.eval_argument_clinic(clinic_args))
             except ValueError:
-                return []
+                return set()
             else:
                 kwargs = {}
                 if want_scope:
                     kwargs['scope'] = arguments.scope()
                 if want_obj:
                     kwargs['obj'] = obj
+                if want_arguments:
+                    kwargs['arguments'] = arguments
                 return func(evaluator, *lst, **kwargs)
+            finally:
+                debug.dbg('builtin end', color='MAGENTA')
 
         return wrapper
     return f
@@ -95,7 +107,6 @@ def argument_clinic(string, want_obj=False, want_scope=False):
 
 @argument_clinic('object, name[, default], /')
 def builtins_getattr(evaluator, objects, names, defaults=None):
-    types = []
     # follow the first param
     for obj in objects:
         if not isinstance(obj, (er.Instance, er.Class, tree.Module, compiled.CompiledObject)):
@@ -108,16 +119,16 @@ def builtins_getattr(evaluator, objects, names, defaults=None):
             else:
                 debug.warning('getattr called without str')
                 continue
-    return types
+    return set()
 
 
 @argument_clinic('object[, bases, dict], /')
 def builtins_type(evaluator, objects, bases, dicts):
     if bases or dicts:
-        # metaclass... maybe someday...
-        return []
+        # It's a type creation... maybe someday...
+        return set()
     else:
-        return [o.base for o in objects if isinstance(o, er.Instance)]
+        return set([o.py__class__() for o in objects])
 
 
 class SuperInstance(er.Instance):
@@ -140,17 +151,21 @@ def builtins_super(evaluator, types, objects, scope):
                 cls = er.Class(evaluator, cls)
             elif isinstance(cls, er.Instance):
                 cls = cls.base
-            su = cls.py__bases__(evaluator)
+            su = cls.py__bases__()
             if su:
                 return evaluator.execute(su[0])
-    return []
+    return set()
 
 
-@argument_clinic('sequence, /', want_obj=True)
-def builtins_reversed(evaluator, sequences, obj):
-    # Unpack the iterator values
-    objects = tuple(iterable.get_iterator_types(sequences))
-    rev = [iterable.AlreadyEvaluated([o]) for o in reversed(objects)]
+@argument_clinic('sequence, /', want_obj=True, want_arguments=True)
+def builtins_reversed(evaluator, sequences, obj, arguments):
+    # While we could do without this variable (just by using sequences), we
+    # want static analysis to work well. Therefore we need to generated the
+    # values again.
+    first_arg = next(arguments.as_tuple())[0]
+    ordered = list(iterable.py__iter__(evaluator, sequences, first_arg))
+
+    rev = [iterable.AlreadyEvaluated(o) for o in reversed(ordered)]
     # Repack iterator values and then run it the normal way. This is
     # necessary, because `reversed` is a function and autocompletion
     # would fail in certain cases like `reversed(x).__iter__` if we
@@ -158,35 +173,43 @@ def builtins_reversed(evaluator, sequences, obj):
     rev = iterable.AlreadyEvaluated(
         [iterable.FakeSequence(evaluator, rev, 'list')]
     )
-    return [er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))]
+    return set([er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))])
 
 
-@argument_clinic('obj, type, /')
-def builtins_isinstance(evaluator, objects, types):
+@argument_clinic('obj, type, /', want_arguments=True)
+def builtins_isinstance(evaluator, objects, types, arguments):
     bool_results = set([])
     for o in objects:
         try:
-            mro_func = o.py__class__(evaluator).py__mro__
+            mro_func = o.py__class__().py__mro__
         except AttributeError:
             # This is temporary. Everything should have a class attribute in
             # Python?! Maybe we'll leave it here, because some numpy objects or
             # whatever might not.
-            return [compiled.true_obj, compiled.false_obj]
+            return set([compiled.create(True), compiled.create(False)])
 
-        mro = mro_func(evaluator)
+        mro = mro_func()
 
         for cls_or_tup in types:
             if cls_or_tup.is_class():
                 bool_results.add(cls_or_tup in mro)
-            else:
+            elif str(cls_or_tup.name) == 'tuple' \
+                    and cls_or_tup.get_parent_scope() == evaluator.BUILTINS:
                 # Check for tuples.
-                classes = iterable.get_iterator_types([cls_or_tup])
+                classes = unite(cls_or_tup.py__iter__())
                 bool_results.add(any(cls in mro for cls in classes))
+            else:
+                _, nodes = list(arguments.unpack())[1]
+                for node in nodes:
+                    message = 'TypeError: isinstance() arg 2 must be a ' \
+                              'class, type, or tuple of classes and types, ' \
+                              'not %s.' % cls_or_tup
+                    analysis.add(evaluator, 'type-error-isinstance', node, message)
 
-    return [compiled.keyword_from_value(x) for x in bool_results]
+    return set(compiled.create(evaluator, x) for x in bool_results)
 
 
-def collections_namedtuple(evaluator, obj, params):
+def collections_namedtuple(evaluator, obj, arguments):
     """
     Implementation of the namedtuple function.
 
@@ -198,20 +221,21 @@ def collections_namedtuple(evaluator, obj, params):
     """
     # Namedtuples are not supported on Python 2.6
     if not hasattr(collections, '_class_template'):
-        return []
+        return set()
 
     # Process arguments
-    name = _follow_param(evaluator, params, 0)[0].obj
-    _fields = _follow_param(evaluator, params, 1)[0]
+    # TODO here we only use one of the types, we should use all.
+    name = list(_follow_param(evaluator, arguments, 0))[0].obj
+    _fields = list(_follow_param(evaluator, arguments, 1))[0]
     if isinstance(_fields, compiled.CompiledObject):
         fields = _fields.obj.replace(',', ' ').split()
     elif isinstance(_fields, iterable.Array):
         try:
-            fields = [v.obj for v in _fields.values()]
+            fields = [v.obj for v in unite(_fields.py__iter__())]
         except AttributeError:
-            return []
+            return set()
     else:
-        return []
+        return set()
 
     # Build source
     source = collections._class_template.format(
@@ -225,8 +249,8 @@ def collections_namedtuple(evaluator, obj, params):
     )
 
     # Parse source
-    generated_class = Parser(evaluator.grammar, unicode(source)).module.subscopes[0]
-    return [er.Class(evaluator, generated_class)]
+    generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0]
+    return set([er.Class(evaluator, generated_class)])
 
 
 @argument_clinic('first, /')
@@ -247,8 +271,8 @@ _implemented = {
         'deepcopy': _return_first_param,
     },
     'json': {
-        'load': lambda *args: [],
-        'loads': lambda *args: [],
+        'load': lambda *args: set(),
+        'loads': lambda *args: set(),
     },
     'collections': {
         'namedtuple': collections_namedtuple,

jedi/evaluate/sys_path.py

@@ -5,11 +5,11 @@ from jedi.evaluate.site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.evaluate.cache import memoize_default
 from jedi import debug
 from jedi import common
-from jedi import cache
+from jedi.parser.utils import load_parser, save_parser
 
 
 def get_venv_path(venv):
@@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt):
     for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
         try:
             assert operator in ['=', '+=']
-            assert tree.is_node(assignee, 'power') and len(assignee.children) > 1
+            assert tree.is_node(assignee, 'power', 'atom_expr') and \
+                len(assignee.children) > 1
             c = assignee.children
             assert c[0].type == 'name' and c[0].value == 'sys'
             trailer = c[1]
@@ -118,11 +119,13 @@ def _paths_from_assignment(evaluator, expr_stmt):
         except AssertionError:
             continue
 
-        from jedi.evaluate.iterable import get_iterator_types
+        from jedi.evaluate.iterable import py__iter__
         from jedi.evaluate.precedence import is_string
-        for val in get_iterator_types(evaluator.eval_statement(expr_stmt)):
-            if is_string(val):
-                yield val.obj
+        types = evaluator.eval_element(expr_stmt)
+        for types in py__iter__(evaluator, types, expr_stmt):
+            for typ in types:
+                if is_string(typ):
+                    yield typ.obj
 
 
 def _paths_from_list_modifications(module_path, trailer1, trailer2):
@@ -150,7 +153,7 @@ def _check_module(evaluator, module):
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
-            if tree.is_node(power, 'power'):
+            if tree.is_node(power, 'power', 'atom_expr'):
                 c = power.children
                 if isinstance(c[0], tree.Name) and c[0].value == 'sys' \
                         and tree.is_node(c[1], 'trailer'):
@@ -183,6 +186,7 @@ def sys_path_with_modifications(evaluator, module):
         return list(evaluator.sys_path)
 
     curdir = os.path.abspath(os.curdir)
+    #TODO why do we need a chdir?
     with common.ignored(OSError):
         os.chdir(os.path.dirname(module.path))
 
@@ -207,11 +211,11 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
             debug.dbg('Error trying to read buildout_script: %s', buildout_script)
             return
 
-        p = Parser(evaluator.grammar, source, buildout_script)
-        cache.save_parser(buildout_script, p)
+        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
+        save_parser(buildout_script, p)
         return p.module
 
-    cached = cache.load_parser(buildout_script)
+    cached = load_parser(buildout_script)
     module = cached and cached.module or load(buildout_script)
     if not module:
         return
@@ -271,8 +275,8 @@ def _get_buildout_scripts(module_path):
                 firstline = f.readline()
                 if firstline.startswith('#!') and 'python' in firstline:
                     extra_module_paths.append(filepath)
-        except IOError as e:
-            # either permission error or race cond. because file got deleted
+        except (UnicodeDecodeError, IOError) as e:
+            # Probably a binary file; permission error or race cond. because file got deleted
             # ignore
             debug.warning(unicode(e))
             continue