Add the executable bit to deploy-master.sh

Python 2.6 seems to be harder and harder to run in tox if setuptools is not properly configured for it.
It's still possible to run it and it still runs on travis.

.travis.yml

@@ -8,11 +8,14 @@ python:
   - 3.5
   - 3.6
   - pypy
+  - "3.7-dev"
+
 matrix:
   allow_failures:
     - python: pypy
     - env: TOXENV=cov
     - env: TOXENV=sith
+    - python: 3.7-dev
   include:
     - python: 3.5
       env: TOXENV=cov
@@ -27,4 +30,3 @@ after_script:
       pip install --quiet coveralls;
       coveralls;
       fi
-

AUTHORS.txt

@@ -44,6 +44,7 @@ Mathias Rav (@Mortal) <rav@cs.au.dk>
 Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
-
+Robin Roth (@robinro)
+Malte Plath (@langsamer)
 
 Note: (@user) means a github user name.

deploy-master.sh

@@ -48,5 +48,5 @@ python setup.py sdist bdist_wheel
 twine upload dist/*
 
 cd $BASE_DIR
-# Back in the development directory fetch tags.
-git fetch --tags
+# The tags have been pushed to this repo. Push the tags to github, now.
+git push --tags

docs/docs/features.rst

@@ -57,8 +57,8 @@ Supported Python Features
 - Django / Flask / Buildout support
 
 
-Unsupported Features
---------------------
+Not Supported
+-------------
 
 Not yet implemented:
 
@@ -77,21 +77,6 @@ Will probably never be implemented:
 Caveats
 -------
 
-**Malformed Syntax**
-
-Syntax errors and other strange stuff may lead to undefined behaviour of the
-completion. |jedi| is **NOT** a Python compiler, that tries to correct you. It
-is a tool that wants to help you. But **YOU** have to know Python, not |jedi|.
-
-**Legacy Python 2 Features**
-
-This framework should work for both Python 2/3. However, some things were just
-not as *pythonic* in Python 2 as things should be. To keep things simple, some
-older Python 2 features have been left out:
-
-- Classes: Always Python 3 like, therefore all classes inherit from ``object``.
-- Generators: No ``next()`` method. The ``__next__()`` method is used instead.
-
 **Slow Performance**
 
 Importing ``numpy`` can be quite slow sometimes, as well as loading the
@@ -237,7 +222,7 @@ A little history
 
 The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
 of the precognition the Jedi have. There's even an awesome `scene
-<http://www.youtube.com/watch?v=5BDO3pyavOY>`_ of Monty Python Jedis :-).
+<https://youtu.be/yHRJLIf7wMU>`_ of Monty Python Jedis :-).
 
 But actually the name hasn't so much to do with Star Wars. It's part of my
 second name.

jedi/__init__.py

@@ -36,7 +36,7 @@ As you see Jedi is pretty simple and allows you to concentrate on writing a
 good text editor, while still having very good IDE features for Python.
 """
 
-__version__ = '0.11.0'
+__version__ = '0.11.1'
 
 from jedi.api import Script, Interpreter, set_debug_function, \
     preload_module, names

jedi/_compatibility.py

@@ -174,14 +174,6 @@ except NameError:
     unicode = str
 
 
-# exec function
-if is_py3:
-    def exec_function(source, global_map):
-        exec(source, global_map)
-else:
-    eval(compile("""def exec_function(source, global_map):
-                        exec source in global_map """, 'blub', 'exec'))
-
 # re-raise function
 if is_py3:
     def reraise(exception, traceback):

jedi/api/__init__.py

@@ -22,17 +22,20 @@ from jedi import settings
 from jedi import cache
 from jedi.api import classes
 from jedi.api import interpreter
-from jedi.api import usages
 from jedi.api import helpers
 from jedi.api.completion import Completion
 from jedi.evaluate import Evaluator
-from jedi.evaluate import representation as er
 from jedi.evaluate import imports
-from jedi.evaluate.param import try_iter_content
+from jedi.evaluate import usages
+from jedi.evaluate.project import Project
+from jedi.evaluate.arguments import try_iter_content
 from jedi.evaluate.helpers import get_module_names, evaluate_call_of_leaf
-from jedi.evaluate.sys_path import get_venv_path, dotted_path_in_sys_path
-from jedi.evaluate.iterable import unpack_tuple_to_dict
+from jedi.evaluate.sys_path import dotted_path_in_sys_path
 from jedi.evaluate.filters import TreeNameDefinition
+from jedi.evaluate.syntax_tree import tree_name_to_contexts
+from jedi.evaluate.context import ModuleContext
+from jedi.evaluate.context.module import ModuleName
+from jedi.evaluate.context.iterable import unpack_tuple_to_dict
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -108,11 +111,9 @@ class Script(object):
 
         # Load the Python grammar of the current interpreter.
         self._grammar = parso.load_grammar()
-        if sys_path is None:
-            venv = os.getenv('VIRTUAL_ENV')
-            if venv:
-                sys_path = list(get_venv_path(venv))
-        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
+        project = Project(sys_path=sys_path)
+        self._evaluator = Evaluator(self._grammar, project)
+        project.add_script_path(self.path)
         debug.speed('init')
 
     @cache.memoize_method
@@ -127,13 +128,13 @@ class Script(object):
 
     @cache.memoize_method
     def _get_module(self):
-        module = er.ModuleContext(
+        module = ModuleContext(
             self._evaluator,
             self._get_module_node(),
             self.path
         )
         if self.path is not None:
-            name = dotted_path_in_sys_path(self._evaluator.sys_path, self.path)
+            name = dotted_path_in_sys_path(self._evaluator.project.sys_path, self.path)
             if name is not None:
                 imports.add_module(self._evaluator, name, module)
         return module
@@ -204,10 +205,15 @@ class Script(object):
                 else:
                     yield name
 
-        names = self._goto()
+        tree_name = self._get_module_node().get_name_of_position(self._pos)
+        if tree_name is None:
+            return []
+        context = self._evaluator.create_context(self._get_module(), tree_name)
+        names = list(self._evaluator.goto(context, tree_name))
+
         if follow_imports:
             def check(name):
-                if isinstance(name, er.ModuleName):
+                if isinstance(name, ModuleName):
                     return False
                 return name.api_type == 'module'
         else:
@@ -219,16 +225,6 @@ class Script(object):
         defs = [classes.Definition(self._evaluator, d) for d in set(names)]
         return helpers.sorted_definitions(defs)
 
-    def _goto(self):
-        """
-        Used for goto_assignments and usages.
-        """
-        name = self._get_module_node().get_name_of_position(self._pos)
-        if name is None:
-            return []
-        context = self._evaluator.create_context(self._get_module(), name)
-        return list(self._evaluator.goto(context, name))
-
     def usages(self, additional_module_paths=()):
         """
         Return :class:`classes.Definition` objects, which contain all
@@ -240,36 +236,15 @@ class Script(object):
 
         :rtype: list of :class:`classes.Definition`
         """
-        temp, settings.dynamic_flow_information = \
-            settings.dynamic_flow_information, False
-        try:
-            module_node = self._get_module_node()
-            user_stmt = get_statement_of_position(module_node, self._pos)
-            definition_names = self._goto()
-            if not definition_names and isinstance(user_stmt, tree.Import):
-                # For not defined imports (goto doesn't find something, we take
-                # the name as a definition. This is enough, because every name
-                # points to it.
-                name = user_stmt.get_name_of_position(self._pos)
-                if name is None:
-                    # Must be syntax
-                    return []
-                definition_names = [TreeNameDefinition(self._get_module(), name)]
+        tree_name = self._get_module_node().get_name_of_position(self._pos)
+        if tree_name is None:
+            # Must be syntax
+            return []
 
-            if not definition_names:
-                # Without a definition for a name we cannot find references.
-                return []
+        names = usages.usages(self._get_module(), tree_name)
 
-            definition_names = usages.resolve_potential_imports(self._evaluator,
-                                                                definition_names)
-
-            modules = set([d.get_root_context() for d in definition_names])
-            modules.add(self._get_module())
-            definitions = usages.usages(self._evaluator, definition_names, modules)
-        finally:
-            settings.dynamic_flow_information = temp
-
-        return helpers.sorted_definitions(set(definitions))
+        definitions = [classes.Definition(self._evaluator, n) for n in names]
+        return helpers.sorted_definitions(definitions)
 
     def call_signatures(self):
         """
@@ -319,10 +294,8 @@ class Script(object):
             for node in get_executable_nodes(module_node):
                 context = self._get_module().create_context(node)
                 if node.type in ('funcdef', 'classdef'):
-                    # TODO This is stupid, should be private
-                    from jedi.evaluate.finder import _name_to_types
                     # Resolve the decorators.
-                    _name_to_types(self._evaluator, context, node.children[1])
+                    tree_name_to_contexts(self._evaluator, context, node.children[1])
                 elif isinstance(node, tree.Import):
                     import_names = set(node.get_defined_names())
                     if node.is_nested():

jedi/api/classes.py

@@ -10,14 +10,14 @@ from parso.python.tree import search_ancestor
 
 from jedi._compatibility import u
 from jedi import settings
-from jedi import common
+from jedi.evaluate.utils import ignored, unite
 from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate import instance
 from jedi.evaluate import imports
 from jedi.evaluate import compiled
 from jedi.evaluate.filters import ParamName
 from jedi.evaluate.imports import ImportName
+from jedi.evaluate.context import instance
+from jedi.evaluate.context import ClassContext, FunctionContext, FunctionExecutionContext
 from jedi.api.keywords import KeywordName
 
 
@@ -290,7 +290,7 @@ class BaseDefinition(object):
         if not path:
             return None  # for keywords the path is empty
 
-        with common.ignored(KeyError):
+        with ignored(KeyError):
             path[0] = self._mapping[path[0]]
         for key, repl in self._tuple_mapping.items():
             if tuple(path[:len(key)]) == key:
@@ -322,8 +322,8 @@ class BaseDefinition(object):
                 param_names = list(context.get_param_names())
                 if isinstance(context, instance.BoundMethod):
                     param_names = param_names[1:]
-            elif isinstance(context, (instance.AbstractInstanceContext, er.ClassContext)):
-                if isinstance(context, er.ClassContext):
+            elif isinstance(context, (instance.AbstractInstanceContext, ClassContext)):
+                if isinstance(context, ClassContext):
                     search = '__init__'
                 else:
                     search = '__call__'
@@ -335,7 +335,7 @@ class BaseDefinition(object):
                 # there's no better solution.
                 inferred = names[0].infer()
                 param_names = get_param_names(next(iter(inferred)))
-                if isinstance(context, er.ClassContext):
+                if isinstance(context, ClassContext):
                     param_names = param_names[1:]
                 return param_names
             elif isinstance(context, compiled.CompiledObject):
@@ -354,10 +354,10 @@ class BaseDefinition(object):
         if context is None:
             return None
 
-        if isinstance(context, er.FunctionExecutionContext):
+        if isinstance(context, FunctionExecutionContext):
             # TODO the function context should be a part of the function
             # execution context.
-            context = er.FunctionContext(
+            context = FunctionContext(
                 self._evaluator, context.parent_context, context.tree_node)
         return Definition(self._evaluator, context.name)
 
@@ -567,7 +567,7 @@ class Definition(BaseDefinition):
         """
         defs = self._name.infer()
         return sorted(
-            common.unite(defined_names(self._evaluator, d) for d in defs),
+            unite(defined_names(self._evaluator, d) for d in defs),
             key=lambda s: s._name.start_pos or (0, 0)
         )
 

jedi/api/helpers.py

@@ -10,6 +10,7 @@ from parso.python import tree
 from parso import split_lines
 
 from jedi._compatibility import u
+from jedi.evaluate.syntax_tree import eval_atom
 from jedi.evaluate.helpers import evaluate_call_of_leaf
 from jedi.cache import time_cache
 
@@ -206,7 +207,7 @@ def evaluate_goto_definition(evaluator, context, leaf):
     elif parent.type == 'trailer':
         return evaluate_call_of_leaf(context, leaf)
     elif isinstance(leaf, tree.Literal):
-        return context.evaluator.eval_atom(context, leaf)
+        return eval_atom(context, leaf)
     return []
 
 

jedi/api/interpreter.py

@@ -2,10 +2,10 @@
 TODO Some parts of this module are still not well documented.
 """
 
-from jedi.evaluate.representation import ModuleContext
+from jedi.evaluate.context import ModuleContext
 from jedi.evaluate import compiled
 from jedi.evaluate.compiled import mixed
-from jedi.evaluate.context import Context
+from jedi.evaluate.base_context import Context
 
 
 class NamespaceObject(object):

jedi/api/keywords.py

@@ -2,7 +2,7 @@ import pydoc
 import keyword
 
 from jedi._compatibility import is_py3, is_py35
-from jedi import common
+from jedi.evaluate.utils import ignored
 from jedi.evaluate.filters import AbstractNameDefinition
 from parso.python.tree import Leaf
 
@@ -123,7 +123,7 @@ def imitate_pydoc(string):
     # with unicode strings)
     string = str(string)
     h = pydoc.help
-    with common.ignored(KeyError):
+    with ignored(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')

jedi/api/usages.py

@@ -1,75 +0,0 @@
-from jedi.api import classes
-from parso.python import tree
-from jedi.evaluate import imports
-from jedi.evaluate.filters import TreeNameDefinition
-from jedi.evaluate.representation import ModuleContext
-
-
-def compare_contexts(c1, c2):
-    return c1 == c2 or (c1[1] == c2[1] and c1[0].tree_node == c2[0].tree_node)
-
-
-def usages(evaluator, definition_names, mods):
-    """
-    :param definitions: list of Name
-    """
-    def resolve_names(definition_names):
-        for name in definition_names:
-            if name.api_type == 'module':
-                found = False
-                for context in name.infer():
-                    if isinstance(context, ModuleContext):
-                        found = True
-                        yield context.name
-                if not found:
-                    yield name
-            else:
-                yield name
-
-    def compare_array(definition_names):
-        """ `definitions` are being compared by module/start_pos, because
-        sometimes the id's of the objects change (e.g. executions).
-        """
-        return [
-            (name.get_root_context(), name.start_pos)
-            for name in resolve_names(definition_names)
-        ]
-
-    search_name = list(definition_names)[0].string_name
-    compare_definitions = compare_array(definition_names)
-    mods = mods | set([d.get_root_context() for d in definition_names])
-    definition_names = set(resolve_names(definition_names))
-    for m in imports.get_modules_containing_name(evaluator, mods, search_name):
-        if isinstance(m, ModuleContext):
-            for name_node in m.tree_node.get_used_names().get(search_name, []):
-                context = evaluator.create_context(m, name_node)
-                result = evaluator.goto(context, name_node)
-                if any(compare_contexts(c1, c2)
-                       for c1 in compare_array(result)
-                       for c2 in compare_definitions):
-                    name = TreeNameDefinition(context, name_node)
-                    definition_names.add(name)
-                    # Previous definitions might be imports, so include them
-                    # (because goto might return that import name).
-                    compare_definitions += compare_array([name])
-        else:
-            # compiled objects
-            definition_names.add(m.name)
-
-    return [classes.Definition(evaluator, n) for n in definition_names]
-
-
-def resolve_potential_imports(evaluator, definitions):
-    """ Adds the modules of the imports """
-    new = set()
-    for d in definitions:
-        if isinstance(d, TreeNameDefinition):
-            imp_or_stmt = d.tree_name.get_definition()
-            if isinstance(imp_or_stmt, tree.Import):
-                new |= resolve_potential_imports(
-                    evaluator,
-                    set(imports.infer_import(
-                        d.parent_context, d.tree_name, is_goto=True
-                    ))
-                )
-    return set(definitions) | new

jedi/cache.py

@@ -12,7 +12,6 @@ there are global variables, which are holding the cache information. Some of
 these variables are being cleaned after every API usage.
 """
 import time
-import inspect
 
 from jedi import settings
 from parso.cache import parser_cache
@@ -46,8 +45,6 @@ def underscore_memoization(func):
             return getattr(self, name)
         except AttributeError:
             result = func(self)
-            if inspect.isgenerator(result):
-                result = list(result)
             setattr(self, name, result)
             return result
 

jedi/common/__init__.py

@@ -0,0 +1 @@
+from jedi.common.context import BaseContextSet, BaseContext

jedi/common/context.py

@@ -0,0 +1,67 @@
+class BaseContext(object):
+    def __init__(self, evaluator, parent_context=None):
+        self.evaluator = evaluator
+        self.parent_context = parent_context
+
+    def get_root_context(self):
+        context = self
+        while True:
+            if context.parent_context is None:
+                return context
+            context = context.parent_context
+
+
+class BaseContextSet(object):
+    def __init__(self, *args):
+        self._set = set(args)
+
+    @classmethod
+    def from_iterable(cls, iterable):
+        return cls.from_set(set(iterable))
+
+    @classmethod
+    def from_set(cls, set_):
+        self = cls()
+        self._set = set_
+        return self
+
+    @classmethod
+    def from_sets(cls, sets):
+        """
+        Used to work with an iterable of set.
+        """
+        aggregated = set()
+        sets = list(sets)
+        for set_ in sets:
+            if isinstance(set_, BaseContextSet):
+                aggregated |= set_._set
+            else:
+                aggregated |= set_
+        return cls.from_set(aggregated)
+
+    def __or__(self, other):
+        return type(self).from_set(self._set | other._set)
+
+    def __iter__(self):
+        for element in self._set:
+            yield element
+
+    def __bool__(self):
+        return bool(self._set)
+
+    def __len__(self):
+        return len(self._set)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set))
+
+    def filter(self, filter_func):
+        return type(self).from_iterable(filter(filter_func, self._set))
+
+    def __getattr__(self, name):
+        def mapper(*args, **kwargs):
+            return type(self).from_sets(
+                getattr(context, name)(*args, **kwargs)
+                for context in self._set
+            )
+        return mapper

jedi/evaluate/__init__.py

@@ -12,29 +12,31 @@ Evaluation of Python code in |jedi| is based on three assumptions:
 * The programmer is not a total dick, e.g. like `this
   <https://github.com/davidhalter/jedi/issues/24>`_ :-)
 
-The actual algorithm is based on a principle called lazy evaluation. If you
-don't know about it, google it.  That said, the typical entry point for static
-analysis is calling ``eval_statement``. There's separate logic for
-autocompletion in the API, the evaluator is all about evaluating an expression.
+The actual algorithm is based on a principle called lazy evaluation.  That
+said, the typical entry point for static analysis is calling
+``eval_expr_stmt``. There's separate logic for autocompletion in the API, the
+evaluator is all about evaluating an expression.
 
-Now you need to understand what follows after ``eval_statement``. Let's
+TODO this paragraph is not what jedi does anymore.
+
+Now you need to understand what follows after ``eval_expr_stmt``. Let's
 make an example::
 
     import datetime
     datetime.date.toda# <-- cursor here
 
 First of all, this module doesn't care about completion. It really just cares
-about ``datetime.date``. At the end of the procedure ``eval_statement`` will
+about ``datetime.date``. At the end of the procedure ``eval_expr_stmt`` will
 return the ``date`` class.
 
 To *visualize* this (simplified):
 
-- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment.
-- ``Evaluator.eval_element`` cares for resolving the dotted path
+- ``Evaluator.eval_expr_stmt`` doesn't do much, because there's no assignment.
+- ``Context.eval_node`` cares for resolving the dotted path
 - ``Evaluator.find_types`` searches for global definitions of datetime, which
   it finds in the definition of an import, by scanning the syntax tree.
 - Using the import logic, the datetime module is found.
-- Now ``find_types`` is called again by ``eval_element`` to find ``date``
+- Now ``find_types`` is called again by ``eval_node`` to find ``date``
   inside the datetime module.
 
 Now what would happen if we wanted ``datetime.date.foo.bar``? Two more
@@ -46,7 +48,7 @@ What if the import would contain another ``ExprStmt`` like this::
     from foo import bar
     Date = bar.baz
 
-Well... You get it. Just another ``eval_statement`` recursion. It's really
+Well... You get it. Just another ``eval_expr_stmt`` recursion. It's really
 easy. Python can obviously get way more complicated then this. To understand
 tuple assignments, list comprehensions and everything else, a lot more code had
 to be written.
@@ -60,57 +62,31 @@ only *evaluates* what needs to be *evaluated*. All the statements and modules
 that are not used are just being ignored.
 """
 
-import copy
 import sys
 
 from parso.python import tree
 import parso
 
 from jedi import debug
-from jedi.common import unite
-from jedi.evaluate import representation as er
+from jedi import parser_utils
+from jedi.evaluate.utils import unite
 from jedi.evaluate import imports
 from jedi.evaluate import recursion
-from jedi.evaluate import iterable
 from jedi.evaluate.cache import evaluator_function_cache
-from jedi.evaluate import stdlib
-from jedi.evaluate import finder
 from jedi.evaluate import compiled
-from jedi.evaluate import precedence
-from jedi.evaluate import param
 from jedi.evaluate import helpers
-from jedi.evaluate import pep0484
 from jedi.evaluate.filters import TreeNameDefinition, ParamName
-from jedi.evaluate.instance import AnonymousInstance, BoundMethod
-from jedi.evaluate.context import ContextualizedName, ContextualizedNode
-from jedi import parser_utils
-
-
-def _limit_context_infers(func):
-    """
-    This is for now the way how we limit type inference going wild. There are
-    other ways to ensure recursion limits as well. This is mostly necessary
-    because of instance (self) access that can be quite tricky to limit.
-
-    I'm still not sure this is the way to go, but it looks okay for now and we
-    can still go anther way in the future. Tests are there. ~ dave
-    """
-    def wrapper(evaluator, context, *args, **kwargs):
-        n = context.tree_node
-        try:
-            evaluator.inferred_element_counts[n] += 1
-            if evaluator.inferred_element_counts[n] > 300:
-                debug.warning('In context %s there were too many inferences.', n)
-                return set()
-        except KeyError:
-            evaluator.inferred_element_counts[n] = 1
-        return func(evaluator, context, *args, **kwargs)
-
-    return wrapper
+from jedi.evaluate.base_context import ContextualizedName, ContextualizedNode, \
+    ContextSet, NO_CONTEXTS, iterate_contexts
+from jedi.evaluate.context import ClassContext, FunctionContext, \
+    AnonymousInstance, BoundMethod
+from jedi.evaluate.context.iterable import CompForContext
+from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \
+    eval_node, check_tuple_assignments
 
 
 class Evaluator(object):
-    def __init__(self, grammar, sys_path=None):
+    def __init__(self, grammar, project):
         self.grammar = grammar
         self.latest_grammar = parso.load_grammar(version='3.6')
         self.memoize_cache = {}  # for memoize decorators
@@ -123,14 +99,8 @@ class Evaluator(object):
         self.dynamic_params_depth = 0
         self.is_analysis = False
         self.python_version = sys.version_info[:2]
-
-        if sys_path is None:
-            sys_path = sys.path
-        self.sys_path = copy.copy(sys_path)
-        try:
-            self.sys_path.remove('')
-        except ValueError:
-            pass
+        self.project = project
+        project.add_evaluator(self)
 
         self.reset_recursion_limitations()
 
@@ -141,82 +111,9 @@ class Evaluator(object):
         self.recursion_detector = recursion.RecursionDetector()
         self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
 
-    def find_types(self, context, name_or_str, name_context, position=None,
-                   search_global=False, is_goto=False, analysis_errors=True):
-        """
-        This is the search function. The most important part to debug.
-        `remove_statements` and `filter_statements` really are the core part of
-        this completion.
-
-        :param position: Position of the last statement -> tuple of line, column
-        :return: List of Names. Their parents are the types.
-        """
-        f = finder.NameFinder(self, context, name_context, name_or_str,
-                              position, analysis_errors=analysis_errors)
-        filters = f.get_filters(search_global)
-        if is_goto:
-            return f.filter_name(filters)
-        return f.find(filters, attribute_lookup=not search_global)
-
-    @_limit_context_infers
-    def eval_statement(self, context, stmt, seek_name=None):
-        with recursion.execution_allowed(self, stmt) as allowed:
-            if allowed or context.get_root_context() == self.BUILTINS:
-                return self._eval_stmt(context, stmt, seek_name)
-        return set()
-
-    #@evaluator_function_cache(default=[])
-    @debug.increase_indent
-    def _eval_stmt(self, context, stmt, seek_name=None):
-        """
-        The starting point of the completion. A statement always owns a call
-        list, which are the calls, that a statement does. In case multiple
-        names are defined in the statement, `seek_name` returns the result for
-        this name.
-
-        :param stmt: A `tree.ExprStmt`.
-        """
-        debug.dbg('eval_statement %s (%s)', stmt, seek_name)
-        rhs = stmt.get_rhs()
-        types = self.eval_element(context, rhs)
-
-        if seek_name:
-            c_node = ContextualizedName(context, seek_name)
-            types = finder.check_tuple_assignments(self, c_node, types)
-
-        first_operator = next(stmt.yield_operators(), None)
-        if first_operator not in ('=', None) and first_operator.type == 'operator':
-            # `=` is always the last character in aug assignments -> -1
-            operator = copy.copy(first_operator)
-            operator.value = operator.value[:-1]
-            name = stmt.get_defined_names()[0].value
-            left = context.py__getattribute__(
-                name, position=stmt.start_pos, search_global=True)
-
-            for_stmt = tree.search_ancestor(stmt, 'for_stmt')
-            if for_stmt is not None and for_stmt.type == 'for_stmt' and types \
-                    and parser_utils.for_stmt_defines_one_name(for_stmt):
-                # Iterate through result and add the values, that's possible
-                # only in for loops without clutter, because they are
-                # predictable. Also only do it, if the variable is not a tuple.
-                node = for_stmt.get_testlist()
-                cn = ContextualizedNode(context, node)
-                ordered = list(iterable.py__iter__(self, cn.infer(), cn))
-
-                for lazy_context in ordered:
-                    dct = {for_stmt.children[1].value: lazy_context.infer()}
-                    with helpers.predefine_names(context, for_stmt, dct):
-                        t = self.eval_element(context, rhs)
-                        left = precedence.calculate(self, context, left, operator, t)
-                types = left
-            else:
-                types = precedence.calculate(self, context, left, operator, types)
-        debug.dbg('eval_statement result %s', types)
-        return types
-
     def eval_element(self, context, element):
-        if isinstance(context, iterable.CompForContext):
-            return self._eval_element_not_cached(context, element)
+        if isinstance(context, CompForContext):
+            return eval_node(context, element)
 
         if_stmt = element
         while if_stmt is not None:
@@ -261,23 +158,23 @@ class Evaluator(object):
                                 new_name_dicts = list(original_name_dicts)
                                 for i, name_dict in enumerate(new_name_dicts):
                                     new_name_dicts[i] = name_dict.copy()
-                                    new_name_dicts[i][if_name.value] = set([definition])
+                                    new_name_dicts[i][if_name.value] = ContextSet(definition)
 
                                 name_dicts += new_name_dicts
                         else:
                             for name_dict in name_dicts:
                                 name_dict[if_name.value] = definitions
             if len(name_dicts) > 1:
-                result = set()
+                result = ContextSet()
                 for name_dict in name_dicts:
                     with helpers.predefine_names(context, if_stmt, name_dict):
-                        result |= self._eval_element_not_cached(context, element)
+                        result |= eval_node(context, element)
                 return result
             else:
                 return self._eval_element_if_evaluated(context, element)
         else:
             if predefined_if_name_dict:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
             else:
                 return self._eval_element_if_evaluated(context, element)
 
@@ -290,201 +187,32 @@ class Evaluator(object):
             parent = parent.parent
             predefined_if_name_dict = context.predefined_names.get(parent)
             if predefined_if_name_dict is not None:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
         return self._eval_element_cached(context, element)
 
-    @evaluator_function_cache(default=set())
+    @evaluator_function_cache(default=NO_CONTEXTS)
     def _eval_element_cached(self, context, element):
-        return self._eval_element_not_cached(context, element)
-
-    @debug.increase_indent
-    @_limit_context_infers
-    def _eval_element_not_cached(self, context, element):
-        debug.dbg('eval_element %s@%s', element, element.start_pos)
-        types = set()
-        typ = element.type
-        if typ in ('name', 'number', 'string', 'atom'):
-            types = self.eval_atom(context, element)
-        elif typ == 'keyword':
-            # For False/True/None
-            if element.value in ('False', 'True', 'None'):
-                types.add(compiled.builtin_from_name(self, element.value))
-            # else: print e.g. could be evaluated like this in Python 2.7
-        elif typ == 'lambdef':
-            types = set([er.FunctionContext(self, context, element)])
-        elif typ == 'expr_stmt':
-            types = self.eval_statement(context, element)
-        elif typ in ('power', 'atom_expr'):
-            first_child = element.children[0]
-            if not (first_child.type == 'keyword' and first_child.value == 'await'):
-                types = self.eval_atom(context, first_child)
-                for trailer in element.children[1:]:
-                    if trailer == '**':  # has a power operation.
-                        right = self.eval_element(context, element.children[2])
-                        types = set(precedence.calculate(self, context, types, trailer, right))
-                        break
-                    types = self.eval_trailer(context, types, trailer)
-        elif typ in ('testlist_star_expr', 'testlist',):
-            # The implicit tuple in statements.
-            types = set([iterable.SequenceLiteralContext(self, context, element)])
-        elif typ in ('not_test', 'factor'):
-            types = self.eval_element(context, element.children[-1])
-            for operator in element.children[:-1]:
-                types = set(precedence.factor_calculate(self, types, operator))
-        elif typ == 'test':
-            # `x if foo else y` case.
-            types = (self.eval_element(context, element.children[0]) |
-                     self.eval_element(context, element.children[-1]))
-        elif typ == 'operator':
-            # Must be an ellipsis, other operators are not evaluated.
-            # In Python 2 ellipsis is coded as three single dot tokens, not
-            # as one token 3 dot token.
-            assert element.value in ('.', '...')
-            types = set([compiled.create(self, Ellipsis)])
-        elif typ == 'dotted_name':
-            types = self.eval_atom(context, element.children[0])
-            for next_name in element.children[2::2]:
-                # TODO add search_global=True?
-                types = unite(
-                    typ.py__getattribute__(next_name, name_context=context)
-                    for typ in types
-                )
-            types = types
-        elif typ == 'eval_input':
-            types = self._eval_element_not_cached(context, element.children[0])
-        elif typ == 'annassign':
-            types = pep0484._evaluate_for_annotation(context, element.children[1])
-        else:
-            types = precedence.calculate_children(self, context, element.children)
-        debug.dbg('eval_element result %s', types)
-        return types
-
-    def eval_atom(self, context, atom):
-        """
-        Basically to process ``atom`` nodes. The parser sometimes doesn't
-        generate the node (because it has just one child). In that case an atom
-        might be a name or a literal as well.
-        """
-        if atom.type == 'name':
-            # This is the first global lookup.
-            stmt = tree.search_ancestor(
-                atom, 'expr_stmt', 'lambdef'
-            ) or atom
-            if stmt.type == 'lambdef':
-                stmt = atom
-            return context.py__getattribute__(
-                name_or_str=atom,
-                position=stmt.start_pos,
-                search_global=True
-            )
-        elif isinstance(atom, tree.Literal):
-            string = parser_utils.safe_literal_eval(atom.value)
-            return set([compiled.create(self, string)])
-        else:
-            c = atom.children
-            if c[0].type == 'string':
-                # Will be one string.
-                types = self.eval_atom(context, c[0])
-                for string in c[1:]:
-                    right = self.eval_atom(context, string)
-                    types = precedence.calculate(self, context, types, '+', right)
-                return types
-            # Parentheses without commas are not tuples.
-            elif c[0] == '(' and not len(c) == 2 \
-                    and not(c[1].type == 'testlist_comp' and
-                            len(c[1].children) > 1):
-                return self.eval_element(context, c[1])
-
-            try:
-                comp_for = c[1].children[1]
-            except (IndexError, AttributeError):
-                pass
-            else:
-                if comp_for == ':':
-                    # Dict comprehensions have a colon at the 3rd index.
-                    try:
-                        comp_for = c[1].children[3]
-                    except IndexError:
-                        pass
-
-                if comp_for.type == 'comp_for':
-                    return set([iterable.Comprehension.from_atom(self, context, atom)])
-
-            # It's a dict/list/tuple literal.
-            array_node = c[1]
-            try:
-                array_node_c = array_node.children
-            except AttributeError:
-                array_node_c = []
-            if c[0] == '{' and (array_node == '}' or ':' in array_node_c):
-                context = iterable.DictLiteralContext(self, context, atom)
-            else:
-                context = iterable.SequenceLiteralContext(self, context, atom)
-            return set([context])
-
-    def eval_trailer(self, context, types, trailer):
-        trailer_op, node = trailer.children[:2]
-        if node == ')':  # `arglist` is optional.
-            node = ()
-
-        new_types = set()
-        if trailer_op == '[':
-            new_types |= iterable.py__getitem__(self, context, types, trailer)
-        else:
-            for typ in types:
-                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
-                if trailer_op == '.':
-                    new_types |= typ.py__getattribute__(
-                        name_context=context,
-                        name_or_str=node
-                    )
-                elif trailer_op == '(':
-                    arguments = param.TreeArguments(self, context, node, trailer)
-                    new_types |= self.execute(typ, arguments)
-        return new_types
-
-    @debug.increase_indent
-    def execute(self, obj, arguments):
-        if self.is_analysis:
-            arguments.eval_all()
-
-        debug.dbg('execute: %s %s', obj, arguments)
-        try:
-            # Some stdlib functions like super(), namedtuple(), etc. have been
-            # hard-coded in Jedi to support them.
-            return stdlib.execute(self, obj, arguments)
-        except stdlib.NotInStdLib:
-            pass
-
-        try:
-            func = obj.py__call__
-        except AttributeError:
-            debug.warning("no execution possible %s", obj)
-            return set()
-        else:
-            types = func(arguments)
-            debug.dbg('execute result: %s in %s', types, obj)
-            return types
+        return eval_node(context, element)
 
     def goto_definitions(self, context, name):
         def_ = name.get_definition(import_name_always=True)
         if def_ is not None:
             type_ = def_.type
             if type_ == 'classdef':
-                return [er.ClassContext(self, name.parent, context)]
+                return [ClassContext(self, context, name.parent)]
             elif type_ == 'funcdef':
-                return [er.FunctionContext(self, context, name.parent)]
+                return [FunctionContext(self, context, name.parent)]
 
             if type_ == 'expr_stmt':
                 is_simple_name = name.parent.type not in ('power', 'trailer')
                 if is_simple_name:
-                    return self.eval_statement(context, def_, name)
+                    return eval_expr_stmt(context, def_, name)
             if type_ == 'for_stmt':
-                container_types = self.eval_element(context, def_.children[3])
+                container_types = context.eval_node(def_.children[3])
                 cn = ContextualizedNode(context, def_.children[3])
-                for_types = iterable.py__iter__types(self, container_types, cn)
+                for_types = iterate_contexts(container_types, cn)
                 c_node = ContextualizedName(context, name)
-                return finder.check_tuple_assignments(self, c_node, for_types)
+                return check_tuple_assignments(self, c_node, for_types)
             if type_ in ('import_from', 'import_name'):
                 return imports.infer_import(context, name)
 
@@ -509,25 +237,27 @@ class Evaluator(object):
                 return module_names
 
         par = name.parent
-        typ = par.type
-        if typ == 'argument' and par.children[1] == '=' and par.children[0] == name:
+        node_type = par.type
+        if node_type == 'argument' and par.children[1] == '=' and par.children[0] == name:
             # Named param goto.
             trailer = par.parent
             if trailer.type == 'arglist':
                 trailer = trailer.parent
             if trailer.type != 'classdef':
                 if trailer.type == 'decorator':
-                    types = self.eval_element(context, trailer.children[1])
+                    context_set = context.eval_node(trailer.children[1])
                 else:
                     i = trailer.parent.children.index(trailer)
                     to_evaluate = trailer.parent.children[:i]
-                    types = self.eval_element(context, to_evaluate[0])
+                    if to_evaluate[0] == 'await':
+                        to_evaluate.pop(0)
+                    context_set = context.eval_node(to_evaluate[0])
                     for trailer in to_evaluate[1:]:
-                        types = self.eval_trailer(context, types, trailer)
+                        context_set = eval_trailer(context, context_set, trailer)
                 param_names = []
-                for typ in types:
+                for context in context_set:
                     try:
-                        get_param_names = typ.get_param_names
+                        get_param_names = context.get_param_names
                     except AttributeError:
                         pass
                     else:
@@ -535,18 +265,18 @@ class Evaluator(object):
                             if param_name.string_name == name.value:
                                 param_names.append(param_name)
                 return param_names
-        elif typ == 'dotted_name':  # Is a decorator.
+        elif node_type == 'dotted_name':  # Is a decorator.
             index = par.children.index(name)
             if index > 0:
                 new_dotted = helpers.deep_ast_copy(par)
                 new_dotted.children[index - 1:] = []
-                values = self.eval_element(context, new_dotted)
+                values = context.eval_node(new_dotted)
                 return unite(
                     value.py__getattribute__(name, name_context=context, is_goto=True)
                     for value in values
                 )
 
-        if typ == 'trailer' and par.children[0] == '.':
+        if node_type == 'trailer' and par.children[0] == '.':
             values = helpers.evaluate_call_of_leaf(context, name, cut_own_trailer=True)
             return unite(
                 value.py__getattribute__(name, name_context=context, is_goto=True)
@@ -595,7 +325,7 @@ class Evaluator(object):
                         parent_context.parent_context, scope_node
                     )
                 else:
-                    func = er.FunctionContext(
+                    func = FunctionContext(
                         self,
                         parent_context,
                         scope_node
@@ -604,7 +334,7 @@ class Evaluator(object):
                     return func.get_function_execution()
                 return func
             elif scope_node.type == 'classdef':
-                class_context = er.ClassContext(self, scope_node, parent_context)
+                class_context = ClassContext(self, parent_context, scope_node)
                 if child_is_funcdef:
                     # anonymous instance
                     return AnonymousInstance(self, parent_context, class_context)
@@ -613,7 +343,7 @@ class Evaluator(object):
             elif scope_node.type == 'comp_for':
                 if node.start_pos >= scope_node.children[-1].start_pos:
                     return parent_context
-                return iterable.CompForContext.from_comp_for(parent_context, scope_node)
+                return CompForContext.from_comp_for(parent_context, scope_node)
             raise Exception("There's a scope that was not managed.")
 
         base_node = base_context.tree_node

jedi/evaluate/analysis.py

@@ -92,7 +92,7 @@ def _check_for_setattr(instance):
     """
     Check if there's any setattr method inside an instance. If so, return True.
     """
-    from jedi.evaluate.representation import ModuleContext
+    from jedi.evaluate.context import ModuleContext
     module = instance.get_root_context()
     if not isinstance(module, ModuleContext):
         return False
@@ -109,7 +109,7 @@ def _check_for_setattr(instance):
 
 def add_attribute_error(name_context, lookup_context, name):
     message = ('AttributeError: %s has no attribute %s.' % (lookup_context, name))
-    from jedi.evaluate.instance import AbstractInstanceContext, CompiledInstanceName
+    from jedi.evaluate.context.instance import AbstractInstanceContext, CompiledInstanceName
     # Check for __getattr__/__getattribute__ existance and issue a warning
     # instead of an error, if that happens.
     typ = Error
@@ -159,8 +159,8 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             else:
                 except_classes = node_context.eval_node(node)
                 for cls in except_classes:
-                    from jedi.evaluate import iterable
-                    if isinstance(cls, iterable.AbstractSequence) and \
+                    from jedi.evaluate.context import iterable
+                    if isinstance(cls, iterable.AbstractIterable) and \
                             cls.array_type == 'tuple':
                         # multiple exceptions
                         for lazy_context in cls.py__iter__():
@@ -181,7 +181,7 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             assert trailer.type == 'trailer'
             arglist = trailer.children[1]
             assert arglist.type == 'arglist'
-            from jedi.evaluate.param import TreeArguments
+            from jedi.evaluate.arguments import TreeArguments
             args = list(TreeArguments(node_context.evaluator, node_context, arglist).unpack())
             # Arguments should be very simple
             assert len(args) == 2

jedi/evaluate/arguments.py

@@ -0,0 +1,245 @@
+from parso.python import tree
+
+from jedi._compatibility import zip_longest
+from jedi import debug
+from jedi.evaluate import analysis
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext, get_merged_lazy_context
+from jedi.evaluate.filters import ParamName
+from jedi.evaluate.base_context import NO_CONTEXTS
+from jedi.evaluate.context import iterable
+from jedi.evaluate.param import get_params, ExecutedParam
+
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for lazy_context in f():
+                try_iter_content(lazy_context.infer(), depth + 1)
+
+
+class AbstractArguments(object):
+    context = None
+
+    def eval_argument_clinic(self, parameters):
+        """Uses a list with argument clinic information (see PEP 436)."""
+        iterator = self.unpack()
+        for i, (name, optional, allow_kwargs) in enumerate(parameters):
+            key, argument = next(iterator, (None, None))
+            if key is not None:
+                raise NotImplementedError
+            if argument is None and not optional:
+                debug.warning('TypeError: %s expected at least %s arguments, got %s',
+                              name, len(parameters), i)
+                raise ValueError
+            values = NO_CONTEXTS if argument is None else argument.infer()
+
+            if not values and not optional:
+                # For the stdlib we always want values. If we don't get them,
+                # that's ok, maybe something is too hard to resolve, however,
+                # we will not proceed with the evaluation of that function.
+                debug.warning('argument_clinic "%s" not resolvable.', name)
+                raise ValueError
+            yield values
+
+    def eval_all(self, funcdef=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, lazy_context in self.unpack():
+            types = lazy_context.infer()
+            try_iter_content(types)
+
+    def get_calling_nodes(self):
+        raise NotImplementedError
+
+    def unpack(self, funcdef=None):
+        raise NotImplementedError
+
+    def get_params(self, execution_context):
+        return get_params(execution_context, self)
+
+
+class AnonymousArguments(AbstractArguments):
+    def get_params(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        return search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        )
+
+
+class TreeArguments(AbstractArguments):
+    def __init__(self, evaluator, context, argument_node, trailer=None):
+        """
+        The argument_node is either a parser node or a list of evaluated
+        objects. Those evaluated objects may be lists of evaluated objects
+        themselves (one list for the first argument, one for the second, etc).
+
+        :param argument_node: May be an argument_node or a list of nodes.
+        """
+        self.argument_node = argument_node
+        self.context = context
+        self._evaluator = evaluator
+        self.trailer = trailer  # Can be None, e.g. in a class definition.
+
+    def _split(self):
+        if isinstance(self.argument_node, (tuple, list)):
+            for el in self.argument_node:
+                yield 0, el
+        else:
+            if not (self.argument_node.type == 'arglist' or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (self.argument_node.type == 'argument') and
+                     self.argument_node.children[0] in ('*', '**'))):
+                yield 0, self.argument_node
+                return
+
+            iterator = iter(self.argument_node.children)
+            for child in iterator:
+                if child == ',':
+                    continue
+                elif child in ('*', '**'):
+                    yield len(child.value), next(iterator)
+                elif child.type == 'argument' and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
+                else:
+                    yield 0, child
+
+    def unpack(self, funcdef=None):
+        named_args = []
+        for star_count, el in self._split():
+            if star_count == 1:
+                arrays = self.context.eval_node(el)
+                iterators = [_iterate_star_args(self.context, a, el, funcdef)
+                             for a in arrays]
+                iterators = list(iterators)
+                for values in list(zip_longest(*iterators)):
+                    # TODO zip_longest yields None, that means this would raise
+                    # an exception?
+                    yield None, get_merged_lazy_context(
+                        [v for v in values if v is not None]
+                    )
+            elif star_count == 2:
+                arrays = self._evaluator.eval_element(self.context, el)
+                for dct in arrays:
+                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
+                        yield key, values
+            else:
+                if el.type == 'argument':
+                    c = el.children
+                    if len(c) == 3:  # Keyword argument.
+                        named_args.append((c[0].value, LazyTreeContext(self.context, c[2]),))
+                    else:  # Generator comprehension.
+                        # Include the brackets with the parent.
+                        comp = iterable.GeneratorComprehension(
+                            self._evaluator, self.context, self.argument_node.parent)
+                        yield None, LazyKnownContext(comp)
+                else:
+                    yield None, LazyTreeContext(self.context, el)
+
+        # Reordering var_args is necessary, because star args sometimes appear
+        # after named argument, but in the actual order it's prepended.
+        for named_arg in named_args:
+            yield named_arg
+
+    def as_tree_tuple_objects(self):
+        for star_count, argument in self._split():
+            if argument.type == 'argument':
+                argument, default = argument.children[::2]
+            else:
+                default = None
+            yield argument, default, star_count
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
+
+    def get_calling_nodes(self):
+        from jedi.evaluate.dynamic import MergedExecutedParams
+        old_arguments_list = []
+        arguments = self
+
+        while arguments not in old_arguments_list:
+            if not isinstance(arguments, TreeArguments):
+                break
+
+            old_arguments_list.append(arguments)
+            for name, default, star_count in reversed(list(arguments.as_tree_tuple_objects())):
+                if not star_count or not isinstance(name, tree.Name):
+                    continue
+
+                names = self._evaluator.goto(arguments.context, name)
+                if len(names) != 1:
+                    break
+                if not isinstance(names[0], ParamName):
+                    break
+                param = names[0].get_param()
+                if isinstance(param, MergedExecutedParams):
+                    # For dynamic searches we don't even want to see errors.
+                    return []
+                if not isinstance(param, ExecutedParam):
+                    break
+                if param.var_args is None:
+                    break
+                arguments = param.var_args
+                break
+
+        return [arguments.argument_node or arguments.trailer]
+
+
+class ValuesArguments(AbstractArguments):
+    def __init__(self, values_list):
+        self._values_list = values_list
+
+    def unpack(self, funcdef=None):
+        for values in self._values_list:
+            yield None, LazyKnownContexts(values)
+
+    def get_calling_nodes(self):
+        return []
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
+
+
+def _iterate_star_args(context, array, input_node, funcdef=None):
+    try:
+        iter_ = array.py__iter__
+    except AttributeError:
+        if funcdef is not None:
+            # TODO this funcdef should not be needed.
+            m = "TypeError: %s() argument after * must be a sequence, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star', input_node, message=m)
+    else:
+        for lazy_context in iter_():
+            yield lazy_context
+
+
+def _star_star_dict(context, array, input_node, funcdef):
+    from jedi.evaluate.context.instance import CompiledInstance
+    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
+        # For now ignore this case. In the future add proper iterators and just
+        # make one call without crazy isinstance checks.
+        return {}
+    elif isinstance(array, iterable.AbstractIterable) and array.array_type == 'dict':
+        return array.exact_key_items()
+    else:
+        if funcdef is not None:
+            m = "TypeError: %s argument after ** must be a mapping, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star-star', input_node, message=m)
+        return {}

jedi/evaluate/base_context.py

@@ -0,0 +1,260 @@
+from parso.python.tree import ExprStmt, CompFor
+
+from jedi import debug
+from jedi._compatibility import Python3Method, zip_longest, unicode
+from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature
+from jedi.common import BaseContextSet, BaseContext
+
+
+class Context(BaseContext):
+    """
+    Should be defined, otherwise the API returns empty types.
+    """
+
+    predefined_names = {}
+    tree_node = None
+    """
+    To be defined by subclasses.
+    """
+
+    @property
+    def api_type(self):
+        # By default just lower name of the class. Can and should be
+        # overwritten.
+        return self.__class__.__name__.lower()
+
+    @debug.increase_indent
+    def execute(self, arguments):
+        """
+        In contrast to py__call__ this function is always available.
+
+        `hasattr(x, py__call__)` can also be checked to see if a context is
+        executable.
+        """
+        if self.evaluator.is_analysis:
+            arguments.eval_all()
+
+        debug.dbg('execute: %s %s', self, arguments)
+        from jedi.evaluate import stdlib
+        try:
+            # Some stdlib functions like super(), namedtuple(), etc. have been
+            # hard-coded in Jedi to support them.
+            return stdlib.execute(self.evaluator, self, arguments)
+        except stdlib.NotInStdLib:
+            pass
+
+        try:
+            func = self.py__call__
+        except AttributeError:
+            debug.warning("no execution possible %s", self)
+            return NO_CONTEXTS
+        else:
+            context_set = func(arguments)
+            debug.dbg('execute result: %s in %s', context_set, self)
+            return context_set
+
+        return self.evaluator.execute(self, arguments)
+
+    def execute_evaluated(self, *value_list):
+        """
+        Execute a function with already executed arguments.
+        """
+        from jedi.evaluate.arguments import ValuesArguments
+        arguments = ValuesArguments([ContextSet(value) for value in value_list])
+        return self.execute(arguments)
+
+    def iterate(self, contextualized_node=None):
+        debug.dbg('iterate')
+        try:
+            iter_method = self.py__iter__
+        except AttributeError:
+            if contextualized_node is not None:
+                from jedi.evaluate import analysis
+                analysis.add(
+                    contextualized_node.context,
+                    'type-error-not-iterable',
+                    contextualized_node.node,
+                    message="TypeError: '%s' object is not iterable" % self)
+            return iter([])
+        else:
+            return iter_method()
+
+    def get_item(self, index_contexts, contextualized_node):
+        from jedi.evaluate.compiled import CompiledObject
+        from jedi.evaluate.context.iterable import Slice, AbstractIterable
+        result = ContextSet()
+
+        for index in index_contexts:
+            if isinstance(index, (CompiledObject, Slice)):
+                index = index.obj
+
+            if type(index) not in (float, int, str, unicode, slice, type(Ellipsis)):
+                # If the index is not clearly defined, we have to get all the
+                # possiblities.
+                if isinstance(self, AbstractIterable) and self.array_type == 'dict':
+                    result |= self.dict_values()
+                else:
+                    result |= iterate_contexts(ContextSet(self))
+                continue
+
+            # The actual getitem call.
+            try:
+                getitem = self.py__getitem__
+            except AttributeError:
+                from jedi.evaluate import analysis
+                # TODO this context is probably not right.
+                analysis.add(
+                    contextualized_node.context,
+                    'type-error-not-subscriptable',
+                    contextualized_node.node,
+                    message="TypeError: '%s' object is not subscriptable" % self
+                )
+            else:
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= iterate_contexts(ContextSet(self))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= self.dict_values()
+        return result
+
+    def eval_node(self, node):
+        return self.evaluator.eval_element(self, node)
+
+    @Python3Method
+    def py__getattribute__(self, name_or_str, name_context=None, position=None,
+                           search_global=False, is_goto=False,
+                           analysis_errors=True):
+        """
+        :param position: Position of the last statement -> tuple of line, column
+        """
+        if name_context is None:
+            name_context = self
+        from jedi.evaluate import finder
+        f = finder.NameFinder(self.evaluator, self, name_context, name_or_str,
+                              position, analysis_errors=analysis_errors)
+        filters = f.get_filters(search_global)
+        if is_goto:
+            return f.filter_name(filters)
+        return f.find(filters, attribute_lookup=not search_global)
+
+        return self.evaluator.find_types(
+            self, name_or_str, name_context, position, search_global, is_goto,
+            analysis_errors)
+
+    def create_context(self, node, node_is_context=False, node_is_object=False):
+        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
+
+    def is_class(self):
+        return False
+
+    def py__bool__(self):
+        """
+        Since Wrapper is a super class for classes, functions and modules,
+        the return value will always be true.
+        """
+        return True
+
+    def py__doc__(self, include_call_signature=False):
+        try:
+            self.tree_node.get_doc_node
+        except AttributeError:
+            return ''
+        else:
+            if include_call_signature:
+                return get_doc_with_call_signature(self.tree_node)
+            else:
+                return clean_scope_docstring(self.tree_node)
+        return None
+
+
+def iterate_contexts(contexts, contextualized_node=None):
+    """
+    Calls `iterate`, on all contexts but ignores the ordering and just returns
+    all contexts that the iterate functions yield.
+    """
+    return ContextSet.from_sets(
+        lazy_context.infer()
+        for lazy_context in contexts.iterate(contextualized_node)
+    )
+
+
+class TreeContext(Context):
+    def __init__(self, evaluator, parent_context=None):
+        super(TreeContext, self).__init__(evaluator, parent_context)
+        self.predefined_names = {}
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
+
+
+class ContextualizedNode(object):
+    def __init__(self, context, node):
+        self.context = context
+        self.node = node
+
+    def get_root_context(self):
+        return self.context.get_root_context()
+
+    def infer(self):
+        return self.context.eval_node(self.node)
+
+
+class ContextualizedName(ContextualizedNode):
+    # TODO merge with TreeNameDefinition?!
+    @property
+    def name(self):
+        return self.node
+
+    def assignment_indexes(self):
+        """
+        Returns an array of tuple(int, node) of the indexes that are used in
+        tuple assignments.
+
+        For example if the name is ``y`` in the following code::
+
+            x, (y, z) = 2, ''
+
+        would result in ``[(1, xyz_node), (0, yz_node)]``.
+        """
+        indexes = []
+        node = self.node.parent
+        compare = self.node
+        while node is not None:
+            if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'):
+                for i, child in enumerate(node.children):
+                    if child == compare:
+                        indexes.insert(0, (int(i / 2), node))
+                        break
+                else:
+                    raise LookupError("Couldn't find the assignment.")
+            elif isinstance(node, (ExprStmt, CompFor)):
+                break
+
+            compare = node
+            node = node.parent
+        return indexes
+
+
+class ContextSet(BaseContextSet):
+    def py__class__(self):
+        return ContextSet.from_iterable(c.py__class__() for c in self._set)
+
+    def iterate(self, contextualized_node=None):
+        from jedi.evaluate.lazy_context import get_merged_lazy_context
+        type_iters = [c.iterate(contextualized_node) for c in self._set]
+        for lazy_contexts in zip_longest(*type_iters):
+            yield get_merged_lazy_context(
+                [l for l in lazy_contexts if l is not None]
+            )
+
+
+NO_CONTEXTS = ContextSet()
+
+
+def iterator_to_context_set(func):
+    def wrapper(*args, **kwargs):
+        return ContextSet.from_iterable(func(*args, **kwargs))
+
+    return wrapper

jedi/evaluate/cache.py

@@ -4,8 +4,6 @@
 - ``CachedMetaClass`` uses ``_memoize_default`` to do the same with classes.
 """
 
-import inspect
-
 _NO_DEFAULT = object()
 
 
@@ -40,8 +38,6 @@ def _memoize_default(default=_NO_DEFAULT, evaluator_is_first_arg=False, second_a
                 if default is not _NO_DEFAULT:
                     memo[key] = default
                 rv = function(obj, *args, **kwargs)
-                if inspect.isgenerator(rv):
-                    rv = list(rv)
                 memo[key] = rv
                 return rv
         return wrapper

jedi/evaluate/compiled/__init__.py

@@ -13,7 +13,8 @@ from jedi import debug
 from jedi.cache import underscore_memoization, memoize_method
 from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \
     ContextNameMixin
-from jedi.evaluate.context import Context, LazyKnownContext
+from jedi.evaluate.base_context import Context, ContextSet
+from jedi.evaluate.lazy_context import LazyKnownContext
 from jedi.evaluate.compiled.getattr_static import getattr_static
 from . import fake
 
@@ -82,10 +83,10 @@ class CompiledObject(Context):
     @CheckAttribute
     def py__call__(self, params):
         if inspect.isclass(self.obj):
-            from jedi.evaluate.instance import CompiledInstance
-            return set([CompiledInstance(self.evaluator, self.parent_context, self, params)])
+            from jedi.evaluate.context import CompiledInstance
+            return ContextSet(CompiledInstance(self.evaluator, self.parent_context, self, params))
         else:
-            return set(self._execute_function(params))
+            return ContextSet.from_iterable(self._execute_function(params))
 
     @CheckAttribute
     def py__class__(self):
@@ -221,9 +222,9 @@ class CompiledObject(Context):
     def py__getitem__(self, index):
         if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
             # Get rid of side effects, we won't call custom `__getitem__`s.
-            return set()
+            return ContextSet()
 
-        return set([create(self.evaluator, self.obj[index])])
+        return ContextSet(create(self.evaluator, self.obj[index]))
 
     @CheckAttribute
     def py__iter__(self):
@@ -266,7 +267,7 @@ class CompiledObject(Context):
                     # TODO do we?
                     continue
                 bltn_obj = create(self.evaluator, bltn_obj)
-                for result in self.evaluator.execute(bltn_obj, params):
+                for result in bltn_obj.execute(params):
                     yield result
         for type_ in docstrings.infer_return_types(self):
             yield type_
@@ -278,7 +279,9 @@ class CompiledObject(Context):
         return []  # Builtins don't have imports
 
     def dict_values(self):
-        return set(create(self.evaluator, v) for v in self.obj.values())
+        return ContextSet.from_iterable(
+            create(self.evaluator, v) for v in self.obj.values()
+        )
 
 
 class CompiledName(AbstractNameDefinition):
@@ -301,7 +304,9 @@ class CompiledName(AbstractNameDefinition):
     @underscore_memoization
     def infer(self):
         module = self.parent_context.get_root_context()
-        return [_create_from_name(self._evaluator, module, self.parent_context, self.string_name)]
+        return ContextSet(_create_from_name(
+            self._evaluator, module, self.parent_context, self.string_name
+        ))
 
 
 class SignatureParamName(AbstractNameDefinition):
@@ -318,13 +323,13 @@ class SignatureParamName(AbstractNameDefinition):
     def infer(self):
         p = self._signature_param
         evaluator = self.parent_context.evaluator
-        types = set()
+        contexts = ContextSet()
         if p.default is not p.empty:
-            types.add(create(evaluator, p.default))
+            contexts = ContextSet(create(evaluator, p.default))
         if p.annotation is not p.empty:
             annotation = create(evaluator, p.annotation)
-            types |= annotation.execute_evaluated()
-        return types
+            contexts |= annotation.execute_evaluated()
+        return contexts
 
 
 class UnresolvableParamName(AbstractNameDefinition):
@@ -335,7 +340,7 @@ class UnresolvableParamName(AbstractNameDefinition):
         self.string_name = name
 
     def infer(self):
-        return set()
+        return ContextSet()
 
 
 class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
@@ -356,7 +361,7 @@ class EmptyCompiledName(AbstractNameDefinition):
         self.string_name = name
 
     def infer(self):
-        return []
+        return ContextSet()
 
 
 class CompiledObjectFilter(AbstractFilter):
@@ -439,16 +444,12 @@ def dotted_from_fs_path(fs_path, sys_path):
 
 
 def load_module(evaluator, path=None, name=None):
-    sys_path = evaluator.sys_path
+    sys_path = list(evaluator.project.sys_path)
     if path is not None:
         dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
     else:
         dotted_path = name
 
-    if dotted_path is None:
-        p, _, dotted_path = path.partition(os.path.sep)
-        sys_path.insert(0, p)
-
     temp, sys.path = sys.path, sys_path
     try:
         __import__(dotted_path)
@@ -548,7 +549,7 @@ def _create_from_name(evaluator, module, compiled_object, name):
     try:
         faked = fake.get_faked(evaluator, module, obj, parent_context=compiled_object, name=name)
         if faked.type == 'funcdef':
-            from jedi.evaluate.representation import FunctionContext
+            from jedi.evaluate.context.function import FunctionContext
             return FunctionContext(evaluator, compiled_object, faked)
     except fake.FakeDoesNotExist:
         pass
@@ -629,7 +630,7 @@ def create(evaluator, obj, parent_context=None, module=None, faked=None):
         try:
             faked = fake.get_faked(evaluator, module, obj, parent_context=parent_context)
             if faked.type == 'funcdef':
-                from jedi.evaluate.representation import FunctionContext
+                from jedi.evaluate.context.function import FunctionContext
                 return FunctionContext(evaluator, parent_context, faked)
         except fake.FakeDoesNotExist:
             pass

jedi/evaluate/compiled/mixed.py

@@ -9,7 +9,8 @@ from jedi import settings
 from jedi.evaluate import compiled
 from jedi.cache import underscore_memoization
 from jedi.evaluate import imports
-from jedi.evaluate.context import Context
+from jedi.evaluate.base_context import Context, ContextSet
+from jedi.evaluate.context import ModuleContext
 from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate.compiled.getattr_static import getattr_static
 
@@ -41,9 +42,6 @@ class MixedObject(object):
     # We have to overwrite everything that has to do with trailers, name
     # lookups and filters to make it possible to route name lookups towards
     # compiled objects and the rest towards tree node contexts.
-    def eval_trailer(*args, **kwags):
-        return Context.eval_trailer(*args, **kwags)
-
     def py__getattribute__(*args, **kwargs):
         return Context.py__getattribute__(*args, **kwargs)
 
@@ -85,7 +83,9 @@ class MixedName(compiled.CompiledName):
             # PyQt4.QtGui.QStyleOptionComboBox.currentText
             # -> just set it to None
             obj = None
-        return [_create(self._evaluator, obj, parent_context=self.parent_context)]
+        return ContextSet(
+            _create(self._evaluator, obj, parent_context=self.parent_context)
+        )
 
     @property
     def api_type(self):
@@ -207,7 +207,6 @@ def _create(evaluator, obj, parent_context=None, *args):
     if parent_context.tree_node.get_root_node() == module_node:
         module_context = parent_context.get_root_context()
     else:
-        from jedi.evaluate.representation import ModuleContext
         module_context = ModuleContext(evaluator, module_node, path=path)
         # TODO this __name__ is probably wrong.
         name = compiled_object.get_root_context().py__name__()

jedi/evaluate/context.py

@@ -1,206 +0,0 @@
-from jedi._compatibility import Python3Method
-from jedi.common import unite
-from parso.python.tree import ExprStmt, CompFor
-from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature
-
-
-class Context(object):
-    """
-    Should be defined, otherwise the API returns empty types.
-    """
-
-    """
-    To be defined by subclasses.
-    """
-    predefined_names = {}
-    tree_node = None
-
-    def __init__(self, evaluator, parent_context=None):
-        self.evaluator = evaluator
-        self.parent_context = parent_context
-
-    @property
-    def api_type(self):
-        # By default just lower name of the class. Can and should be
-        # overwritten.
-        return self.__class__.__name__.lower()
-
-    def get_root_context(self):
-        context = self
-        while True:
-            if context.parent_context is None:
-                return context
-            context = context.parent_context
-
-    def execute(self, arguments):
-        return self.evaluator.execute(self, arguments)
-
-    def execute_evaluated(self, *value_list):
-        """
-        Execute a function with already executed arguments.
-        """
-        from jedi.evaluate.param import ValuesArguments
-        arguments = ValuesArguments([[value] for value in value_list])
-        return self.execute(arguments)
-
-    def eval_node(self, node):
-        return self.evaluator.eval_element(self, node)
-
-    def eval_stmt(self, stmt, seek_name=None):
-        return self.evaluator.eval_statement(self, stmt, seek_name)
-
-    def eval_trailer(self, types, trailer):
-        return self.evaluator.eval_trailer(self, types, trailer)
-
-    @Python3Method
-    def py__getattribute__(self, name_or_str, name_context=None, position=None,
-                           search_global=False, is_goto=False,
-                           analysis_errors=True):
-        if name_context is None:
-            name_context = self
-        return self.evaluator.find_types(
-            self, name_or_str, name_context, position, search_global, is_goto,
-            analysis_errors)
-
-    def create_context(self, node, node_is_context=False, node_is_object=False):
-        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
-
-    def is_class(self):
-        return False
-
-    def py__bool__(self):
-        """
-        Since Wrapper is a super class for classes, functions and modules,
-        the return value will always be true.
-        """
-        return True
-
-    def py__doc__(self, include_call_signature=False):
-        try:
-            self.tree_node.get_doc_node
-        except AttributeError:
-            return ''
-        else:
-            if include_call_signature:
-                return get_doc_with_call_signature(self.tree_node)
-            else:
-                return clean_scope_docstring(self.tree_node)
-        return None
-
-
-class TreeContext(Context):
-    def __init__(self, evaluator, parent_context=None):
-        super(TreeContext, self).__init__(evaluator, parent_context)
-        self.predefined_names = {}
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
-
-
-class AbstractLazyContext(object):
-    def __init__(self, data):
-        self.data = data
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.data)
-
-    def infer(self):
-        raise NotImplementedError
-
-
-class LazyKnownContext(AbstractLazyContext):
-    """data is a context."""
-    def infer(self):
-        return set([self.data])
-
-
-class LazyKnownContexts(AbstractLazyContext):
-    """data is a set of contexts."""
-    def infer(self):
-        return self.data
-
-
-class LazyUnknownContext(AbstractLazyContext):
-    def __init__(self):
-        super(LazyUnknownContext, self).__init__(None)
-
-    def infer(self):
-        return set()
-
-
-class LazyTreeContext(AbstractLazyContext):
-    def __init__(self, context, node):
-        super(LazyTreeContext, self).__init__(node)
-        self._context = context
-        # We need to save the predefined names. It's an unfortunate side effect
-        # that needs to be tracked otherwise results will be wrong.
-        self._predefined_names = dict(context.predefined_names)
-
-    def infer(self):
-        old, self._context.predefined_names = \
-            self._context.predefined_names, self._predefined_names
-        try:
-            return self._context.eval_node(self.data)
-        finally:
-            self._context.predefined_names = old
-
-
-def get_merged_lazy_context(lazy_contexts):
-    if len(lazy_contexts) > 1:
-        return MergedLazyContexts(lazy_contexts)
-    else:
-        return lazy_contexts[0]
-
-
-class MergedLazyContexts(AbstractLazyContext):
-    """data is a list of lazy contexts."""
-    def infer(self):
-        return unite(l.infer() for l in self.data)
-
-
-class ContextualizedNode(object):
-    def __init__(self, context, node):
-        self.context = context
-        self._node = node
-
-    def get_root_context(self):
-        return self.context.get_root_context()
-
-    def infer(self):
-        return self.context.eval_node(self._node)
-
-
-class ContextualizedName(ContextualizedNode):
-    # TODO merge with TreeNameDefinition?!
-    @property
-    def name(self):
-        return self._node
-
-    def assignment_indexes(self):
-        """
-        Returns an array of tuple(int, node) of the indexes that are used in
-        tuple assignments.
-
-        For example if the name is ``y`` in the following code::
-
-            x, (y, z) = 2, ''
-
-        would result in ``[(1, xyz_node), (0, yz_node)]``.
-        """
-        indexes = []
-        node = self._node.parent
-        compare = self._node
-        while node is not None:
-            if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'):
-                for i, child in enumerate(node.children):
-                    if child == compare:
-                        indexes.insert(0, (int(i / 2), node))
-                        break
-                else:
-                    raise LookupError("Couldn't find the assignment.")
-            elif isinstance(node, (ExprStmt, CompFor)):
-                break
-
-            compare = node
-            node = node.parent
-        return indexes

jedi/evaluate/context/__init__.py

@@ -0,0 +1,5 @@
+from jedi.evaluate.context.module import ModuleContext
+from jedi.evaluate.context.klass import ClassContext
+from jedi.evaluate.context.function import FunctionContext, FunctionExecutionContext
+from jedi.evaluate.context.instance import AnonymousInstance, BoundMethod, \
+    CompiledInstance, AbstractInstanceContext, TreeInstance

jedi/evaluate/context/function.py

@@ -0,0 +1,226 @@
+from parso.python import tree
+
+from jedi._compatibility import use_metaclass
+from jedi import debug
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import compiled
+from jedi.evaluate import recursion
+from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
+from jedi.evaluate import flow_analysis
+from jedi.evaluate import helpers
+from jedi.evaluate.arguments import AnonymousArguments
+from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \
+    ContextName, AbstractNameDefinition, ParamName
+from jedi.evaluate.base_context import ContextualizedNode, NO_CONTEXTS, \
+    ContextSet, TreeContext
+from jedi.evaluate.lazy_context import LazyKnownContexts, LazyKnownContext, \
+    LazyTreeContext
+from jedi.evaluate.context import iterable
+from jedi import parser_utils
+from jedi.evaluate.parser_cache import get_yield_exprs
+
+
+class LambdaName(AbstractNameDefinition):
+    string_name = '<lambda>'
+
+    def __init__(self, lambda_context):
+        self._lambda_context = lambda_context
+        self.parent_context = lambda_context.parent_context
+
+    def start_pos(self):
+        return self._lambda_context.tree_node.start_pos
+
+    def infer(self):
+        return ContextSet(self._lambda_context)
+
+
+class FunctionContext(use_metaclass(CachedMetaClass, TreeContext)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    api_type = 'function'
+
+    def __init__(self, evaluator, parent_context, funcdef):
+        """ This should not be called directly """
+        super(FunctionContext, self).__init__(evaluator, parent_context)
+        self.tree_node = funcdef
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        if search_global:
+            yield ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            )
+        else:
+            scope = self.py__class__()
+            for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
+                yield filter
+
+    def infer_function_execution(self, function_execution):
+        """
+        Created to be used by inheritance.
+        """
+        yield_exprs = get_yield_exprs(self.evaluator, self.tree_node)
+        if yield_exprs:
+            return ContextSet(iterable.Generator(self.evaluator, function_execution))
+        else:
+            return function_execution.get_return_values()
+
+    def get_function_execution(self, arguments=None):
+        if arguments is None:
+            arguments = AnonymousArguments()
+
+        return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
+
+    def py__call__(self, arguments):
+        function_execution = self.get_function_execution(arguments)
+        return self.infer_function_execution(function_execution)
+
+    def py__class__(self):
+        # This differentiation is only necessary for Python2. Python3 does not
+        # use a different method class.
+        if isinstance(parser_utils.get_parent_scope(self.tree_node), tree.Class):
+            name = 'METHOD_CLASS'
+        else:
+            name = 'FUNCTION_CLASS'
+        return compiled.get_special_object(self.evaluator, name)
+
+    @property
+    def name(self):
+        if self.tree_node.type == 'lambdef':
+            return LambdaName(self)
+        return ContextName(self, self.tree_node.name)
+
+    def get_param_names(self):
+        function_execution = self.get_function_execution()
+        return [ParamName(function_execution, param.name)
+                for param in self.tree_node.get_params()]
+
+
+class FunctionExecutionContext(TreeContext):
+    """
+    This class is used to evaluate functions and their returns.
+
+    This is the most complicated class, because it contains the logic to
+    transfer parameters. It is even more complicated, because there may be
+    multiple calls to functions and recursion has to be avoided. But this is
+    responsibility of the decorators.
+    """
+    function_execution_filter = FunctionExecutionFilter
+
+    def __init__(self, evaluator, parent_context, function_context, var_args):
+        super(FunctionExecutionContext, self).__init__(evaluator, parent_context)
+        self.function_context = function_context
+        self.tree_node = function_context.tree_node
+        self.var_args = var_args
+
+    @evaluator_method_cache(default=NO_CONTEXTS)
+    @recursion.execution_recursion_decorator()
+    def get_return_values(self, check_yields=False):
+        funcdef = self.tree_node
+        if funcdef.type == 'lambdef':
+            return self.evaluator.eval_element(self, funcdef.children[-1])
+
+        if check_yields:
+            context_set = NO_CONTEXTS
+            returns = get_yield_exprs(self.evaluator, funcdef)
+        else:
+            returns = funcdef.iter_return_stmts()
+            context_set = docstrings.infer_return_types(self.function_context)
+            context_set |= pep0484.infer_return_types(self.function_context)
+
+        for r in returns:
+            check = flow_analysis.reachability_check(self, funcdef, r)
+            if check is flow_analysis.UNREACHABLE:
+                debug.dbg('Return unreachable: %s', r)
+            else:
+                if check_yields:
+                    context_set |= ContextSet.from_sets(
+                        lazy_context.infer()
+                        for lazy_context in self._eval_yield(r)
+                    )
+                else:
+                    try:
+                        children = r.children
+                    except AttributeError:
+                        context_set |= ContextSet(compiled.create(self.evaluator, None))
+                    else:
+                        context_set |= self.eval_node(children[1])
+            if check is flow_analysis.REACHABLE:
+                debug.dbg('Return reachable: %s', r)
+                break
+        return context_set
+
+    def _eval_yield(self, yield_expr):
+        if yield_expr.type == 'keyword':
+            # `yield` just yields None.
+            yield LazyKnownContext(compiled.create(self.evaluator, None))
+            return
+
+        node = yield_expr.children[1]
+        if node.type == 'yield_arg':  # It must be a yield from.
+            cn = ContextualizedNode(self, node.children[1])
+            for lazy_context in cn.infer().iterate(cn):
+                yield lazy_context
+        else:
+            yield LazyTreeContext(self, node)
+
+    @recursion.execution_recursion_decorator(default=iter([]))
+    def get_yield_values(self):
+        for_parents = [(y, tree.search_ancestor(y, 'for_stmt', 'funcdef',
+                                                'while_stmt', 'if_stmt'))
+                       for y in get_yield_exprs(self.evaluator, self.tree_node)]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
+                    and parser_utils.for_stmt_defines_one_name(for_stmt):  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self.tree_node:
+                yields_order.append((None, [yield_]))
+            else:
+                types = self.get_return_values(check_yields=True)
+                if types:
+                    yield LazyKnownContexts(types)
+                return
+            last_for_stmt = for_stmt
+
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._eval_yield(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_testlist()
+                cn = ContextualizedNode(self, input_node)
+                ordered = cn.infer().iterate(cn)
+                ordered = list(ordered)
+                for lazy_context in ordered:
+                    dct = {str(for_stmt.children[1].value): lazy_context.infer()}
+                    with helpers.predefine_names(self, for_stmt, dct):
+                        for yield_in_same_for_stmt in yields:
+                            for result in self._eval_yield(yield_in_same_for_stmt):
+                                yield result
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield self.function_execution_filter(self.evaluator, self,
+                                             until_position=until_position,
+                                             origin_scope=origin_scope)
+
+    @evaluator_method_cache()
+    def get_params(self):
+        return self.var_args.get_params(self)

jedi/evaluate/context/instance.py

@@ -1,21 +1,23 @@
 from abc import abstractproperty
 
 from jedi._compatibility import is_py3
-from jedi.common import unite
 from jedi import debug
 from jedi.evaluate import compiled
 from jedi.evaluate import filters
-from jedi.evaluate.context import Context, LazyKnownContext, LazyKnownContexts
+from jedi.evaluate.base_context import Context, NO_CONTEXTS, ContextSet, \
+    iterator_to_context_set
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.evaluate.param import AbstractArguments, AnonymousArguments
+from jedi.evaluate.arguments import AbstractArguments, AnonymousArguments
 from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate import iterable
+from jedi.evaluate.context.function import FunctionExecutionContext, FunctionContext
+from jedi.evaluate.context.klass import ClassContext, apply_py__get__
+from jedi.evaluate.context import iterable
 from jedi.parser_utils import get_parent_scope
 
 
 
-class InstanceFunctionExecution(er.FunctionExecutionContext):
+class InstanceFunctionExecution(FunctionExecutionContext):
     def __init__(self, instance, parent_context, function_context, var_args):
         self.instance = instance
         var_args = InstanceVarArgs(self, var_args)
@@ -24,7 +26,7 @@ class InstanceFunctionExecution(er.FunctionExecutionContext):
             instance.evaluator, parent_context, function_context, var_args)
 
 
-class AnonymousInstanceFunctionExecution(er.FunctionExecutionContext):
+class AnonymousInstanceFunctionExecution(FunctionExecutionContext):
     function_execution_filter = filters.AnonymousInstanceFunctionExecutionFilter
 
     def __init__(self, instance, parent_context, function_context, var_args):
@@ -58,7 +60,7 @@ class AbstractInstanceContext(Context):
             raise AttributeError
 
         def execute(arguments):
-            return unite(name.execute(arguments) for name in names)
+            return ContextSet.from_sets(name.execute(arguments) for name in names)
 
         return execute
 
@@ -80,7 +82,7 @@ class AbstractInstanceContext(Context):
         return []
 
     def execute_function_slots(self, names, *evaluated_args):
-        return unite(
+        return ContextSet.from_sets(
             name.execute_evaluated(*evaluated_args)
             for name in names
         )
@@ -96,7 +98,7 @@ class AbstractInstanceContext(Context):
                 none_obj = compiled.create(self.evaluator, None)
                 return self.execute_function_slots(names, none_obj, obj)
         else:
-            return set([self])
+            return ContextSet(self)
 
     def get_filters(self, search_global=None, until_position=None,
                     origin_scope=None, include_self_names=True):
@@ -122,7 +124,7 @@ class AbstractInstanceContext(Context):
             names = self.get_function_slot_names('__getitem__')
         except KeyError:
             debug.warning('No __getitem__, cannot access the array.')
-            return set()
+            return NO_CONTEXTS
         else:
             index_obj = compiled.create(self.evaluator, index)
             return self.execute_function_slots(names, index_obj)
@@ -187,7 +189,7 @@ class AbstractInstanceContext(Context):
                     )
                     return bound_method.get_function_execution()
             elif scope.type == 'classdef':
-                class_context = er.ClassContext(self.evaluator, scope, parent_context)
+                class_context = ClassContext(self.evaluator, scope, parent_context)
                 return class_context
             elif scope.type == 'comp_for':
                 # Comprehensions currently don't have a special scope in Jedi.
@@ -250,9 +252,10 @@ class CompiledInstanceName(compiled.CompiledName):
         super(CompiledInstanceName, self).__init__(evaluator, parent_context, name)
         self._instance = instance
 
+    @iterator_to_context_set
     def infer(self):
         for result_context in super(CompiledInstanceName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
+            if isinstance(result_context, FunctionContext):
                 parent_context = result_context.parent_context
                 while parent_context.is_class():
                     parent_context = parent_context.parent_context
@@ -284,7 +287,7 @@ class CompiledInstanceClassFilter(compiled.CompiledObjectFilter):
             self._evaluator, self._instance, self._compiled_object, name)
 
 
-class BoundMethod(er.FunctionContext):
+class BoundMethod(FunctionContext):
     def __init__(self, evaluator, instance, class_context, *args, **kwargs):
         super(BoundMethod, self).__init__(evaluator, *args, **kwargs)
         self._instance = instance
@@ -311,9 +314,7 @@ class CompiledBoundMethod(compiled.CompiledObject):
 
 class InstanceNameDefinition(filters.TreeNameDefinition):
     def infer(self):
-        contexts = super(InstanceNameDefinition, self).infer()
-        for context in contexts:
-            yield context
+        return super(InstanceNameDefinition, self).infer()
 
 
 class LazyInstanceName(filters.TreeNameDefinition):
@@ -331,9 +332,10 @@ class LazyInstanceName(filters.TreeNameDefinition):
 
 
 class LazyInstanceClassName(LazyInstanceName):
+    @iterator_to_context_set
     def infer(self):
         for result_context in super(LazyInstanceClassName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
+            if isinstance(result_context, FunctionContext):
                 # Classes are never used to resolve anything within the
                 # functions. Only other functions and modules will resolve
                 # those things.
@@ -346,7 +348,7 @@ class LazyInstanceClassName(LazyInstanceName):
                     parent_context, result_context.tree_node
                 )
             else:
-                for c in er.apply_py__get__(result_context, self._instance):
+                for c in apply_py__get__(result_context, self._instance):
                     yield c
 
 

jedi/evaluate/context/iterable.py

@@ -22,28 +22,28 @@ It is important to note that:
 """
 from jedi import debug
 from jedi import settings
-from jedi import common
-from jedi.common import unite, safe_property
-from jedi._compatibility import unicode, zip_longest, is_py3
 from jedi.evaluate import compiled
-from jedi.evaluate import helpers
 from jedi.evaluate import analysis
-from jedi.evaluate import pep0484
-from jedi.evaluate import context
-from jedi.evaluate import precedence
 from jedi.evaluate import recursion
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext
+from jedi.evaluate.helpers import is_string, predefine_names, evaluate_call_of_leaf
+from jedi.evaluate.utils import safe_property
+from jedi.evaluate.utils import to_list
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \
-    ParserTreeFilter
+from jedi.evaluate.filters import ParserTreeFilter, has_builtin_methods, \
+    register_builtin_method, SpecialMethodFilter
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, Context, \
+    TreeContext, ContextualizedNode
 from jedi.parser_utils import get_comp_fors
 
 
-class AbstractSequence(context.Context):
+class AbstractIterable(Context):
     builtin_methods = {}
     api_type = 'instance'
 
     def __init__(self, evaluator):
-        super(AbstractSequence, self).__init__(evaluator, evaluator.BUILTINS)
+        super(AbstractIterable, self).__init__(evaluator, evaluator.BUILTINS)
 
     def get_filters(self, search_global, until_position=None, origin_scope=None):
         raise NotImplementedError
@@ -53,87 +53,6 @@ class AbstractSequence(context.Context):
         return compiled.CompiledContextName(self, self.array_type)
 
 
-class BuiltinMethod(object):
-    """``Generator.__next__`` ``dict.values`` methods and so on."""
-    def __init__(self, builtin_context, method, builtin_func):
-        self._builtin_context = builtin_context
-        self._method = method
-        self._builtin_func = builtin_func
-
-    def py__call__(self, params):
-        return self._method(self._builtin_context)
-
-    def __getattr__(self, name):
-        return getattr(self._builtin_func, name)
-
-
-class SpecialMethodFilter(DictFilter):
-    """
-    A filter for methods that are defined in this module on the corresponding
-    classes like Generator (for __next__, etc).
-    """
-    class SpecialMethodName(AbstractNameDefinition):
-        api_type = 'function'
-
-        def __init__(self, parent_context, string_name, callable_, builtin_context):
-            self.parent_context = parent_context
-            self.string_name = string_name
-            self._callable = callable_
-            self._builtin_context = builtin_context
-
-        def infer(self):
-            filter = next(self._builtin_context.get_filters())
-            # We can take the first index, because on builtin methods there's
-            # always only going to be one name. The same is true for the
-            # inferred values.
-            builtin_func = next(iter(filter.get(self.string_name)[0].infer()))
-            return set([BuiltinMethod(self.parent_context, self._callable, builtin_func)])
-
-    def __init__(self, context, dct, builtin_context):
-        super(SpecialMethodFilter, self).__init__(dct)
-        self.context = context
-        self._builtin_context = builtin_context
-        """
-        This context is what will be used to introspect the name, where as the
-        other context will be used to execute the function.
-
-        We distinguish, because we have to.
-        """
-
-    def _convert(self, name, value):
-        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
-
-
-def has_builtin_methods(cls):
-    base_dct = {}
-    # Need to care properly about inheritance. Builtin Methods should not get
-    # lost, just because they are not mentioned in a class.
-    for base_cls in reversed(cls.__bases__):
-        try:
-            base_dct.update(base_cls.builtin_methods)
-        except AttributeError:
-            pass
-
-    cls.builtin_methods = base_dct
-    for func in cls.__dict__.values():
-        try:
-            cls.builtin_methods.update(func.registered_builtin_methods)
-        except AttributeError:
-            pass
-    return cls
-
-
-def register_builtin_method(method_name, python_version_match=None):
-    def wrapper(func):
-        if python_version_match and python_version_match != 2 + int(is_py3):
-            # Some functions do only apply to certain versions.
-            return func
-        dct = func.__dict__.setdefault('registered_builtin_methods', {})
-        dct[method_name] = func
-        return func
-    return wrapper
-
-
 @has_builtin_methods
 class GeneratorMixin(object):
     array_type = None
@@ -143,7 +62,7 @@ class GeneratorMixin(object):
     @register_builtin_method('__next__', python_version_match=3)
     def py__next__(self):
         # TODO add TypeError if params are given.
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
 
     def get_filters(self, search_global, until_position=None, origin_scope=None):
         gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT')
@@ -163,7 +82,7 @@ class GeneratorMixin(object):
         return compiled.CompiledContextName(self, 'generator')
 
 
-class Generator(GeneratorMixin, context.Context):
+class Generator(GeneratorMixin, Context):
     """Handling of `yield` functions."""
     def __init__(self, evaluator, func_execution_context):
         super(Generator, self).__init__(evaluator, parent_context=evaluator.BUILTINS)
@@ -176,7 +95,7 @@ class Generator(GeneratorMixin, context.Context):
         return "<%s of %s>" % (type(self).__name__, self._func_execution_context)
 
 
-class CompForContext(context.TreeContext):
+class CompForContext(TreeContext):
     @classmethod
     def from_comp_for(cls, parent_context, comp_for):
         return cls(parent_context.evaluator, parent_context, comp_for)
@@ -192,7 +111,7 @@ class CompForContext(context.TreeContext):
         yield ParserTreeFilter(self.evaluator, self)
 
 
-class Comprehension(AbstractSequence):
+class Comprehension(AbstractIterable):
     @staticmethod
     def from_atom(evaluator, context, atom):
         bracket = atom.children[0]
@@ -234,14 +153,13 @@ class Comprehension(AbstractSequence):
         return CompForContext.from_comp_for(parent_context, comp_for)
 
     def _nested(self, comp_fors, parent_context=None):
-        evaluator = self.evaluator
         comp_for = comp_fors[0]
         input_node = comp_for.children[3]
         parent_context = parent_context or self._defining_context
         input_types = parent_context.eval_node(input_node)
 
-        cn = context.ContextualizedNode(parent_context, input_node)
-        iterated = py__iter__(evaluator, input_types, cn)
+        cn = ContextualizedNode(parent_context, input_node)
+        iterated = input_types.iterate(cn)
         exprlist = comp_for.children[1]
         for i, lazy_context in enumerate(iterated):
             types = lazy_context.infer()
@@ -250,7 +168,7 @@ class Comprehension(AbstractSequence):
                 parent_context,
                 comp_for,
             )
-            with helpers.predefine_names(context_, comp_for, dct):
+            with predefine_names(context_, comp_for, dct):
                 try:
                     for result in self._nested(comp_fors[1:], context_):
                         yield result
@@ -262,7 +180,7 @@ class Comprehension(AbstractSequence):
                         yield iterated
 
     @evaluator_method_cache(default=[])
-    @common.to_list
+    @to_list
     def _iterate(self):
         comp_fors = tuple(get_comp_fors(self._get_comp_for()))
         for result in self._nested(comp_fors):
@@ -270,7 +188,7 @@ class Comprehension(AbstractSequence):
 
     def py__iter__(self):
         for set_ in self._iterate():
-            yield context.LazyKnownContexts(set_)
+            yield LazyKnownContexts(set_)
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self._atom)
@@ -296,7 +214,10 @@ class ArrayMixin(object):
         return self.evaluator.BUILTINS
 
     def dict_values(self):
-        return unite(self._defining_context.eval_node(v) for k, v in self._items())
+        return ContextSet.from_sets(
+            self._defining_context.eval_node(v)
+            for k, v in self._items()
+        )
 
 
 class ListComprehension(ArrayMixin, Comprehension):
@@ -304,7 +225,7 @@ class ListComprehension(ArrayMixin, Comprehension):
 
     def py__getitem__(self, index):
         if isinstance(index, slice):
-            return set([self])
+            return ContextSet(self)
 
         all_types = list(self.py__iter__())
         return all_types[index].infer()
@@ -323,7 +244,7 @@ class DictComprehension(ArrayMixin, Comprehension):
 
     def py__iter__(self):
         for keys, values in self._iterate():
-            yield context.LazyKnownContexts(keys)
+            yield LazyKnownContexts(keys)
 
     def py__getitem__(self, index):
         for keys, values in self._iterate():
@@ -334,19 +255,19 @@ class DictComprehension(ArrayMixin, Comprehension):
         return self.dict_values()
 
     def dict_values(self):
-        return unite(values for keys, values in self._iterate())
+        return ContextSet.from_sets(values for keys, values in self._iterate())
 
     @register_builtin_method('values')
     def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
+        lazy_context = LazyKnownContexts(self.dict_values())
+        return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context]))
 
     @register_builtin_method('items')
     def _imitate_items(self):
-        items = set(
+        items = ContextSet.from_iterable(
             FakeSequence(
                 self.evaluator, 'tuple'
-                (context.LazyKnownContexts(keys), context.LazyKnownContexts(values))
+                (LazyKnownContexts(keys), LazyKnownContexts(values))
             ) for keys, values in self._iterate()
         )
 
@@ -357,7 +278,7 @@ class GeneratorComprehension(GeneratorMixin, Comprehension):
     pass
 
 
-class SequenceLiteralContext(ArrayMixin, AbstractSequence):
+class SequenceLiteralContext(ArrayMixin, AbstractIterable):
     mapping = {'(': 'tuple',
                '[': 'list',
                '{': 'set'}
@@ -385,7 +306,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
 
         # Can raise an IndexError
         if isinstance(index, slice):
-            return set([self])
+            return ContextSet(self)
         else:
             return self._defining_context.eval_node(self._items()[index])
 
@@ -396,16 +317,16 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
         """
         if self.array_type == 'dict':
             # Get keys.
-            types = set()
+            types = ContextSet()
             for k, _ in self._items():
                 types |= self._defining_context.eval_node(k)
             # We don't know which dict index comes first, therefore always
             # yield all the types.
             for _ in types:
-                yield context.LazyKnownContexts(types)
+                yield LazyKnownContexts(types)
         else:
             for node in self._items():
-                yield context.LazyTreeContext(self._defining_context, node)
+                yield LazyTreeContext(self._defining_context, node)
 
             for addition in check_array_additions(self._defining_context, self):
                 yield addition
@@ -413,7 +334,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
     def _values(self):
         """Returns a list of a list of node."""
         if self.array_type == 'dict':
-            return unite(v for k, v in self._items())
+            return ContextSet.from_sets(v for k, v in self._items())
         else:
             return self._items()
 
@@ -451,8 +372,8 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
         """
         for key_node, value in self._items():
             for key in self._defining_context.eval_node(key_node):
-                if precedence.is_string(key):
-                    yield key.obj, context.LazyTreeContext(self._defining_context, value)
+                if is_string(key):
+                    yield key.obj, LazyTreeContext(self._defining_context, value)
 
     def __repr__(self):
         return "<%s of %s>" % (self.__class__.__name__, self.atom)
@@ -469,20 +390,20 @@ class DictLiteralContext(SequenceLiteralContext):
 
     @register_builtin_method('values')
     def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
+        lazy_context = LazyKnownContexts(self.dict_values())
+        return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context]))
 
     @register_builtin_method('items')
     def _imitate_items(self):
         lazy_contexts = [
-            context.LazyKnownContext(FakeSequence(
+            LazyKnownContext(FakeSequence(
                 self.evaluator, 'tuple',
-                (context.LazyTreeContext(self._defining_context, key_node),
-                 context.LazyTreeContext(self._defining_context, value_node))
+                (LazyTreeContext(self._defining_context, key_node),
+                 LazyTreeContext(self._defining_context, value_node))
             )) for key_node, value_node in self._items()
         ]
 
-        return set([FakeSequence(self.evaluator, 'list', lazy_contexts)])
+        return ContextSet(FakeSequence(self.evaluator, 'list', lazy_contexts))
 
 
 class _FakeArray(SequenceLiteralContext):
@@ -502,7 +423,7 @@ class FakeSequence(_FakeArray):
         self._lazy_context_list = lazy_context_list
 
     def py__getitem__(self, index):
-        return set(self._lazy_context_list[index].infer())
+        return self._lazy_context_list[index].infer()
 
     def py__iter__(self):
         return self._lazy_context_list
@@ -521,13 +442,13 @@ class FakeDict(_FakeArray):
 
     def py__iter__(self):
         for key in self._dct:
-            yield context.LazyKnownContext(compiled.create(self.evaluator, key))
+            yield LazyKnownContext(compiled.create(self.evaluator, key))
 
     def py__getitem__(self, index):
         return self._dct[index].infer()
 
     def dict_values(self):
-        return unite(lazy_context.infer() for lazy_context in self._dct.values())
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self._dct.values())
 
     def exact_key_items(self):
         return self._dct.items()
@@ -544,7 +465,7 @@ class MergedArray(_FakeArray):
                 yield lazy_context
 
     def py__getitem__(self, index):
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
 
     def _items(self):
         for array in self._arrays:
@@ -568,7 +489,7 @@ def unpack_tuple_to_dict(context, types, exprlist):
         dct = {}
         parts = iter(exprlist.children[::2])
         n = 0
-        for lazy_context in py__iter__(context.evaluator, types, exprlist):
+        for lazy_context in types.iterate(exprlist):
             n += 1
             try:
                 part = next(parts)
@@ -595,103 +516,16 @@ def unpack_tuple_to_dict(context, types, exprlist):
     raise NotImplementedError
 
 
-def py__iter__(evaluator, types, contextualized_node=None):
-    debug.dbg('py__iter__')
-    type_iters = []
-    for typ in types:
-        try:
-            iter_method = typ.py__iter__
-        except AttributeError:
-            if contextualized_node is not None:
-                analysis.add(
-                    contextualized_node.context,
-                    'type-error-not-iterable',
-                    contextualized_node._node,
-                    message="TypeError: '%s' object is not iterable" % typ)
-        else:
-            type_iters.append(iter_method())
-
-    for lazy_contexts in zip_longest(*type_iters):
-        yield context.get_merged_lazy_context(
-            [l for l in lazy_contexts if l is not None]
-        )
-
-
-def py__iter__types(evaluator, types, contextualized_node=None):
-    """
-    Calls `py__iter__`, but ignores the ordering in the end and just returns
-    all types that it contains.
-    """
-    return unite(
-        lazy_context.infer()
-        for lazy_context in py__iter__(evaluator, types, contextualized_node)
-    )
-
-
-def py__getitem__(evaluator, context, types, trailer):
-    from jedi.evaluate.representation import ClassContext
-    from jedi.evaluate.instance import TreeInstance
-    result = set()
-
-    trailer_op, node, trailer_cl = trailer.children
-    assert trailer_op == "["
-    assert trailer_cl == "]"
-
-    # special case: PEP0484 typing module, see
-    # https://github.com/davidhalter/jedi/issues/663
-    for typ in list(types):
-        if isinstance(typ, (ClassContext, TreeInstance)):
-            typing_module_types = pep0484.py__getitem__(context, typ, node)
-            if typing_module_types is not None:
-                types.remove(typ)
-                result |= typing_module_types
-
-    if not types:
-        # all consumed by special cases
-        return result
-
-    for index in create_index_types(evaluator, context, node):
-        if isinstance(index, (compiled.CompiledObject, Slice)):
-            index = index.obj
-
-        if type(index) not in (float, int, str, unicode, slice, type(Ellipsis)):
-            # If the index is not clearly defined, we have to get all the
-            # possiblities.
-            for typ in list(types):
-                if isinstance(typ, AbstractSequence) and typ.array_type == 'dict':
-                    types.remove(typ)
-                    result |= typ.dict_values()
-            return result | py__iter__types(evaluator, types)
-
-        for typ in types:
-            # The actual getitem call.
-            try:
-                getitem = typ.py__getitem__
-            except AttributeError:
-                # TODO this context is probably not right.
-                analysis.add(context, 'type-error-not-subscriptable', trailer_op,
-                             message="TypeError: '%s' object is not subscriptable" % typ)
-            else:
-                try:
-                    result |= getitem(index)
-                except IndexError:
-                    result |= py__iter__types(evaluator, set([typ]))
-                except KeyError:
-                    # Must be a dict. Lists don't raise KeyErrors.
-                    result |= typ.dict_values()
-    return result
-
-
 def check_array_additions(context, sequence):
     """ Just a mapper function for the internal _check_array_additions """
     if sequence.array_type not in ('list', 'set'):
         # TODO also check for dict updates
-        return set()
+        return NO_CONTEXTS
 
     return _check_array_additions(context, sequence)
 
 
-@evaluator_method_cache(default=set())
+@evaluator_method_cache(default=NO_CONTEXTS)
 @debug.increase_indent
 def _check_array_additions(context, sequence):
     """
@@ -700,25 +534,25 @@ def _check_array_additions(context, sequence):
     >>> a = [""]
     >>> a.append(1)
     """
-    from jedi.evaluate import param
+    from jedi.evaluate import arguments
 
     debug.dbg('Dynamic array search for %s' % sequence, color='MAGENTA')
     module_context = context.get_root_context()
     if not settings.dynamic_array_additions or isinstance(module_context, compiled.CompiledObject):
         debug.dbg('Dynamic array search aborted.', color='MAGENTA')
-        return set()
+        return ContextSet()
 
     def find_additions(context, arglist, add_name):
-        params = list(param.TreeArguments(context.evaluator, context, arglist).unpack())
+        params = list(arguments.TreeArguments(context.evaluator, context, arglist).unpack())
         result = set()
         if add_name in ['insert']:
             params = params[1:]
         if add_name in ['append', 'add', 'insert']:
-            for key, lazy_context in params:
-                result.add(lazy_context)
+            for key, whatever in params:
+                result.add(whatever)
         elif add_name in ['extend', 'update']:
             for key, lazy_context in params:
-                result |= set(py__iter__(context.evaluator, lazy_context.infer()))
+                result |= set(lazy_context.infer().iterate())
         return result
 
     temp_param_add, settings.dynamic_params_for_other_modules = \
@@ -755,7 +589,7 @@ def _check_array_additions(context, sequence):
 
                 with recursion.execution_allowed(context.evaluator, power) as allowed:
                     if allowed:
-                        found = helpers.evaluate_call_of_leaf(
+                        found = evaluate_call_of_leaf(
                             random_context,
                             name,
                             cut_own_trailer=True
@@ -780,8 +614,8 @@ def get_dynamic_array_instance(instance):
         return instance.var_args
 
     ai = _ArrayInstance(instance)
-    from jedi.evaluate import param
-    return param.ValuesArguments([[ai]])
+    from jedi.evaluate import arguments
+    return arguments.ValuesArguments([ContextSet(ai)])
 
 
 class _ArrayInstance(object):
@@ -806,17 +640,20 @@ class _ArrayInstance(object):
         except StopIteration:
             pass
         else:
-            for lazy in py__iter__(self.instance.evaluator, lazy_context.infer()):
+            for lazy in lazy_context.infer().iterate():
                 yield lazy
 
-        from jedi.evaluate import param
-        if isinstance(var_args, param.TreeArguments):
+        from jedi.evaluate import arguments
+        if isinstance(var_args, arguments.TreeArguments):
             additions = _check_array_additions(var_args.context, self.instance)
             for addition in additions:
                 yield addition
 
+    def iterate(self, contextualized_node=None):
+        return self.py__iter__()
 
-class Slice(context.Context):
+
+class Slice(Context):
     def __init__(self, context, start, stop, step):
         super(Slice, self).__init__(
             context.evaluator,
@@ -852,33 +689,3 @@ class Slice(context.Context):
             return slice(get(self._start), get(self._stop), get(self._step))
         except IndexError:
             return slice(None, None, None)
-
-
-def create_index_types(evaluator, context, index):
-    """
-    Handles slices in subscript nodes.
-    """
-    if index == ':':
-        # Like array[:]
-        return set([Slice(context, None, None, None)])
-
-    elif index.type == 'subscript' and not index.children[0] == '.':
-        # subscript basically implies a slice operation, except for Python 2's
-        # Ellipsis.
-        # e.g. array[:3]
-        result = []
-        for el in index.children:
-            if el == ':':
-                if not result:
-                    result.append(None)
-            elif el.type == 'sliceop':
-                if len(el.children) == 2:
-                    result.append(el.children[1])
-            else:
-                result.append(el)
-        result += [None] * (3 - len(result))
-
-        return set([Slice(context, *result)])
-
-    # No slices
-    return context.eval_node(index)

jedi/evaluate/context/klass.py

@@ -0,0 +1,197 @@
+"""
+Like described in the :mod:`parso.python.tree` module,
+there's a need for an ast like module to represent the states of parsed
+modules.
+
+But now there are also structures in Python that need a little bit more than
+that. An ``Instance`` for example is only a ``Class`` before it is
+instantiated. This class represents these cases.
+
+So, why is there also a ``Class`` class here? Well, there are decorators and
+they change classes in Python 3.
+
+Representation modules also define "magic methods". Those methods look like
+``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
+and others. Here's a list:
+
+====================================== ========================================
+**Method**                             **Description**
+-------------------------------------- ----------------------------------------
+py__call__(params: Array)              On callable objects, returns types.
+py__bool__()                           Returns True/False/None; None means that
+                                       there's no certainty.
+py__bases__()                          Returns a list of base classes.
+py__mro__()                            Returns a list of classes (the mro).
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__getitem__(index: int/str)          Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__file__()                           Only on modules. Returns None if does
+                                       not exist.
+py__package__()                        Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
+py__get__(call_object)                 Only on instances. Simulates
+                                       descriptors.
+py__doc__(include_call_signature:      Returns the docstring for a context.
+          bool)
+====================================== ========================================
+
+"""
+from jedi._compatibility import use_metaclass
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import compiled
+from jedi.evaluate.lazy_context import LazyKnownContext
+from jedi.evaluate.filters import ParserTreeFilter, TreeNameDefinition, \
+    ContextName, AnonymousInstanceParamName
+from jedi.evaluate.base_context import ContextSet, iterator_to_context_set, \
+    TreeContext
+
+
+def apply_py__get__(context, base_context):
+    try:
+        method = context.py__get__
+    except AttributeError:
+        yield context
+    else:
+        for descriptor_context in method(base_context):
+            yield descriptor_context
+
+
+class ClassName(TreeNameDefinition):
+    def __init__(self, parent_context, tree_name, name_context):
+        super(ClassName, self).__init__(parent_context, tree_name)
+        self._name_context = name_context
+
+    @iterator_to_context_set
+    def infer(self):
+        # TODO this _name_to_types might get refactored and be a part of the
+        # parent class. Once it is, we can probably just overwrite method to
+        # achieve this.
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        inferred = tree_name_to_contexts(
+            self.parent_context.evaluator, self._name_context, self.tree_name)
+
+        for result_context in inferred:
+            for c in apply_py__get__(result_context, self.parent_context):
+                yield c
+
+
+class ClassFilter(ParserTreeFilter):
+    name_class = ClassName
+
+    def _convert_names(self, names):
+        return [self.name_class(self.context, name, self._node_context)
+                for name in names]
+
+
+class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
+    """
+    This class is not only important to extend `tree.Class`, it is also a
+    important for descriptors (if the descriptor methods are evaluated or not).
+    """
+    api_type = 'class'
+
+    def __init__(self, evaluator, parent_context, classdef):
+        super(ClassContext, self).__init__(evaluator, parent_context=parent_context)
+        self.tree_node = classdef
+
+    @evaluator_method_cache(default=())
+    def py__mro__(self):
+        def add(cls):
+            if cls not in mro:
+                mro.append(cls)
+
+        mro = [self]
+        # TODO Do a proper mro resolution. Currently we are just listing
+        # classes. However, it's a complicated algorithm.
+        for lazy_cls in self.py__bases__():
+            # TODO there's multiple different mro paths possible if this yields
+            # multiple possibilities. Could be changed to be more correct.
+            for cls in lazy_cls.infer():
+                # TODO detect for TypeError: duplicate base class str,
+                # e.g.  `class X(str, str): pass`
+                try:
+                    mro_method = cls.py__mro__
+                except AttributeError:
+                    # TODO add a TypeError like:
+                    """
+                    >>> class Y(lambda: test): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: function() argument 1 must be code, not str
+                    >>> class Y(1): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: int() takes at most 2 arguments (3 given)
+                    """
+                    pass
+                else:
+                    add(cls)
+                    for cls_new in mro_method():
+                        add(cls_new)
+        return tuple(mro)
+
+    @evaluator_method_cache(default=())
+    def py__bases__(self):
+        arglist = self.tree_node.get_super_arglist()
+        if arglist:
+            from jedi.evaluate import arguments
+            args = arguments.TreeArguments(self.evaluator, self, arglist)
+            return [value for key, value in args.unpack() if key is None]
+        else:
+            return [LazyKnownContext(compiled.create(self.evaluator, object))]
+
+    def py__call__(self, params):
+        from jedi.evaluate.context import TreeInstance
+        return ContextSet(TreeInstance(self.evaluator, self.parent_context, self, params))
+
+    def py__class__(self):
+        return compiled.create(self.evaluator, type)
+
+    def get_params(self):
+        from jedi.evaluate.context import AnonymousInstance
+        anon = AnonymousInstance(self.evaluator, self.parent_context, self)
+        return [AnonymousInstanceParamName(anon, param.name) for param in self.funcdef.get_params()]
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
+        if search_global:
+            yield ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            )
+        else:
+            for cls in self.py__mro__():
+                if isinstance(cls, compiled.CompiledObject):
+                    for filter in cls.get_filters(is_instance=is_instance):
+                        yield filter
+                else:
+                    yield ClassFilter(
+                        self.evaluator, self, node_context=cls,
+                        origin_scope=origin_scope)
+
+    def is_class(self):
+        return True
+
+    def get_function_slot_names(self, name):
+        for filter in self.get_filters(search_global=False):
+            names = filter.get(name)
+            if names:
+                return names
+        return []
+
+    def get_param_names(self):
+        for name in self.get_function_slot_names('__init__'):
+            for context_ in name.infer():
+                try:
+                    method = context_.get_param_names
+                except AttributeError:
+                    pass
+                else:
+                    return list(method())[1:]
+        return []
+
+    @property
+    def name(self):
+        return ContextName(self, self.tree_node.name)

jedi/evaluate/context/module.py

@@ -0,0 +1,213 @@
+import pkgutil
+import imp
+import re
+import os
+
+from parso import python_bytes_to_unicode
+
+from jedi._compatibility import use_metaclass
+from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache
+from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
+    AbstractNameDefinition, ParserTreeFilter, DictFilter
+from jedi.evaluate import compiled
+from jedi.evaluate.base_context import TreeContext
+from jedi.evaluate.imports import SubModuleName, infer_import
+
+
+class _ModuleAttributeName(AbstractNameDefinition):
+    """
+    For module attributes like __file__, __str__ and so on.
+    """
+    api_type = 'instance'
+
+    def __init__(self, parent_module, string_name):
+        self.parent_context = parent_module
+        self.string_name = string_name
+
+    def infer(self):
+        return compiled.create(self.parent_context.evaluator, str).execute_evaluated()
+
+
+class ModuleName(ContextNameMixin, AbstractNameDefinition):
+    start_pos = 1, 0
+
+    def __init__(self, context, name):
+        self._context = context
+        self._name = name
+
+    @property
+    def string_name(self):
+        return self._name
+
+
+class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
+    api_type = 'module'
+    parent_context = None
+
+    def __init__(self, evaluator, module_node, path):
+        super(ModuleContext, self).__init__(evaluator, parent_context=None)
+        self.tree_node = module_node
+        self._path = path
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield ParserTreeFilter(
+            self.evaluator,
+            context=self,
+            until_position=until_position,
+            origin_scope=origin_scope
+        )
+        yield GlobalNameFilter(self, self.tree_node)
+        yield DictFilter(self._sub_modules_dict())
+        yield DictFilter(self._module_attributes_dict())
+        for star_module in self.star_imports():
+            yield next(star_module.get_filters(search_global))
+
+    # I'm not sure if the star import cache is really that effective anymore
+    # with all the other really fast import caches. Recheck. Also we would need
+    # to push the star imports into Evaluator.modules, if we reenable this.
+    @evaluator_method_cache([])
+    def star_imports(self):
+        modules = []
+        for i in self.tree_node.iter_imports():
+            if i.is_star_import():
+                name = i.get_paths()[-1][-1]
+                new = infer_import(self, name)
+                for module in new:
+                    if isinstance(module, ModuleContext):
+                        modules += module.star_imports()
+                modules += new
+        return modules
+
+    @evaluator_method_cache()
+    def _module_attributes_dict(self):
+        names = ['__file__', '__package__', '__doc__', '__name__']
+        # All the additional module attributes are strings.
+        return dict((n, _ModuleAttributeName(self, n)) for n in names)
+
+    @property
+    def _string_name(self):
+        """ This is used for the goto functions. """
+        if self._path is None:
+            return ''  # no path -> empty name
+        else:
+            sep = (re.escape(os.path.sep),) * 2
+            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path)
+            # Remove PEP 3149 names
+            return re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        return ModuleName(self, self._string_name)
+
+    def _get_init_directory(self):
+        """
+        :return: The path to the directory of a package. None in case it's not
+                 a package.
+        """
+        for suffix, _, _ in imp.get_suffixes():
+            ending = '__init__' + suffix
+            py__file__ = self.py__file__()
+            if py__file__ is not None and py__file__.endswith(ending):
+                # Remove the ending, including the separator.
+                return self.py__file__()[:-len(ending) - 1]
+        return None
+
+    def py__name__(self):
+        for name, module in self.evaluator.modules.items():
+            if module == self and name != '':
+                return name
+
+        return '__main__'
+
+    def py__file__(self):
+        """
+        In contrast to Python's __file__ can be None.
+        """
+        if self._path is None:
+            return None
+
+        return os.path.abspath(self._path)
+
+    def py__package__(self):
+        if self._get_init_directory() is None:
+            return re.sub(r'\.?[^\.]+$', '', self.py__name__())
+        else:
+            return self.py__name__()
+
+    def _py__path__(self):
+        search_path = self.evaluator.project.sys_path
+        init_path = self.py__file__()
+        if os.path.basename(init_path) == '__init__.py':
+            with open(init_path, 'rb') as f:
+                content = python_bytes_to_unicode(f.read(), errors='replace')
+                # these are strings that need to be used for namespace packages,
+                # the first one is ``pkgutil``, the second ``pkg_resources``.
+                options = ('declare_namespace(__name__)', 'extend_path(__path__')
+                if options[0] in content or options[1] in content:
+                    # It is a namespace, now try to find the rest of the
+                    # modules on sys_path or whatever the search_path is.
+                    paths = set()
+                    for s in search_path:
+                        other = os.path.join(s, self.name.string_name)
+                        if os.path.isdir(other):
+                            paths.add(other)
+                    if paths:
+                        return list(paths)
+                    # TODO I'm not sure if this is how nested namespace
+                    # packages work. The tests are not really good enough to
+                    # show that.
+        # Default to this.
+        return [self._get_init_directory()]
+
+    @property
+    def py__path__(self):
+        """
+        Not seen here, since it's a property. The callback actually uses a
+        variable, so use it like::
+
+            foo.py__path__(sys_path)
+
+        In case of a package, this returns Python's __path__ attribute, which
+        is a list of paths (strings).
+        Raises an AttributeError if the module is not a package.
+        """
+        path = self._get_init_directory()
+
+        if path is None:
+            raise AttributeError('Only packages have __path__ attributes.')
+        else:
+            return self._py__path__
+
+    @evaluator_method_cache()
+    def _sub_modules_dict(self):
+        """
+        Lists modules in the directory of this module (if this module is a
+        package).
+        """
+        path = self._path
+        names = {}
+        if path is not None and path.endswith(os.path.sep + '__init__.py'):
+            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            for module_loader, name, is_pkg in mods:
+                # It's obviously a relative import to the current module.
+                names[name] = SubModuleName(self, name)
+
+        # TODO add something like this in the future, its cleaner than the
+        #   import hacks.
+        # ``os.path`` is a hardcoded exception, because it's a
+        # ``sys.modules`` modification.
+        # if str(self.name) == 'os':
+        #     names.append(Name('path', parent_context=self))
+
+        return names
+
+    def py__class__(self):
+        return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')
+
+    def __repr__(self):
+        return "<%s: %s@%s-%s>" % (
+            self.__class__.__name__, self._string_name,
+            self.tree_node.start_pos[0], self.tree_node.end_pos[0])
+
+

jedi/evaluate/context/namespace.py

@@ -0,0 +1,74 @@
+import os
+from itertools import chain
+
+from jedi._compatibility import use_metaclass
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import imports
+from jedi.evaluate.filters import DictFilter, AbstractNameDefinition
+from jedi.evaluate.base_context import NO_CONTEXTS, TreeContext
+
+
+class ImplicitNSName(AbstractNameDefinition):
+    """
+    Accessing names for implicit namespace packages should infer to nothing.
+    This object will prevent Jedi from raising exceptions
+    """
+    def __init__(self, implicit_ns_context, string_name):
+        self.implicit_ns_context = implicit_ns_context
+        self.string_name = string_name
+
+    def infer(self):
+        return NO_CONTEXTS
+
+    def get_root_context(self):
+        return self.implicit_ns_context
+
+
+class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, TreeContext)):
+    """
+    Provides support for implicit namespace packages
+    """
+    api_type = 'module'
+    parent_context = None
+
+    def __init__(self, evaluator, fullname):
+        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
+        self.evaluator = evaluator
+        self.fullname = fullname
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield DictFilter(self._sub_modules_dict())
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        string_name = self.py__package__().rpartition('.')[-1]
+        return ImplicitNSName(self, string_name)
+
+    def py__file__(self):
+        return None
+
+    def py__package__(self):
+        """Return the fullname
+        """
+        return self.fullname
+
+    @property
+    def py__path__(self):
+        return lambda: [self.paths]
+
+    @evaluator_method_cache()
+    def _sub_modules_dict(self):
+        names = {}
+
+        paths = self.paths
+        file_names = chain.from_iterable(os.listdir(path) for path in paths)
+        mods = [
+            file_name.rpartition('.')[0] if '.' in file_name else file_name
+            for file_name in file_names
+            if file_name != '__pycache__'
+        ]
+
+        for name in mods:
+            names[name] = imports.SubModuleName(self, name)
+        return names

jedi/evaluate/docstrings.py

@@ -21,11 +21,11 @@ from textwrap import dedent
 from parso import parse
 
 from jedi._compatibility import u
-from jedi.common import unite
-from jedi.evaluate import context
+from jedi.evaluate.utils import indent_block
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.common import indent_block
-from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence
+from jedi.evaluate.base_context import iterator_to_context_set, ContextSet, \
+    NO_CONTEXTS
+from jedi.evaluate.lazy_context import LazyKnownContexts
 
 
 DOCSTRING_PARAM_PATTERNS = [
@@ -202,7 +202,7 @@ def _evaluate_for_statement_string(module_context, string):
     except (AttributeError, IndexError):
         return []
 
-    from jedi.evaluate.representation import FunctionContext
+    from jedi.evaluate.context import FunctionContext
     function_context = FunctionContext(
         module_context.evaluator,
         module_context,
@@ -223,7 +223,10 @@ def _execute_types_in_stmt(module_context, stmt):
     contain is executed. (Used as type information).
     """
     definitions = module_context.eval_node(stmt)
-    return unite(_execute_array_values(module_context.evaluator, d) for d in definitions)
+    return ContextSet.from_sets(
+        _execute_array_values(module_context.evaluator, d)
+        for d in definitions
+    )
 
 
 def _execute_array_values(evaluator, array):
@@ -231,11 +234,15 @@ def _execute_array_values(evaluator, array):
     Tuples indicate that there's not just one return value, but the listed
     ones.  `(str, int)` means that it returns a tuple with both types.
     """
+    from jedi.evaluate.context.iterable import SequenceLiteralContext, FakeSequence
     if isinstance(array, SequenceLiteralContext):
         values = []
         for lazy_context in array.py__iter__():
-            objects = unite(_execute_array_values(evaluator, typ) for typ in lazy_context.infer())
-            values.append(context.LazyKnownContexts(objects))
+            objects = ContextSet.from_sets(
+                _execute_array_values(evaluator, typ)
+                for typ in lazy_context.infer()
+            )
+            values.append(LazyKnownContexts(objects))
         return set([FakeSequence(evaluator, array.array_type, values)])
     else:
         return array.execute_evaluated()
@@ -243,10 +250,10 @@ def _execute_array_values(evaluator, array):
 
 @evaluator_method_cache()
 def infer_param(execution_context, param):
-    from jedi.evaluate.instance import AnonymousInstanceFunctionExecution
+    from jedi.evaluate.context.instance import AnonymousInstanceFunctionExecution
 
     def eval_docstring(docstring):
-        return set(
+        return ContextSet.from_iterable(
             p
             for param_str in _search_param_in_docstr(docstring, param.name.value)
             for p in _evaluate_for_statement_string(module_context, param_str)
@@ -254,7 +261,7 @@ def infer_param(execution_context, param):
     module_context = execution_context.get_root_context()
     func = param.get_parent_function()
     if func.type == 'lambdef':
-        return set()
+        return NO_CONTEXTS
 
     types = eval_docstring(execution_context.py__doc__())
     if isinstance(execution_context, AnonymousInstanceFunctionExecution) and \
@@ -266,6 +273,7 @@ def infer_param(execution_context, param):
 
 
 @evaluator_method_cache()
+@iterator_to_context_set
 def infer_return_types(function_context):
     def search_return_in_docstr(code):
         for p in DOCSTRING_RETURN_PATTERNS:
@@ -279,4 +287,3 @@ def infer_return_types(function_context):
     for type_str in search_return_in_docstr(function_context.py__doc__()):
         for type_eval in _evaluate_for_statement_string(function_context.get_root_context(), type_str):
             yield type_eval
-

jedi/evaluate/dynamic.py

@@ -14,7 +14,7 @@ It works as follows:
 
 - |Jedi| sees a param
 - search for function calls named ``foo``
-- execute these calls and check the input. This work with a ``ParamListener``.
+- execute these calls and check the input.
 """
 
 from parso.python import tree
@@ -22,26 +22,19 @@ from jedi import settings
 from jedi import debug
 from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate import imports
-from jedi.evaluate.param import TreeArguments, create_default_params
+from jedi.evaluate.arguments import TreeArguments
+from jedi.evaluate.param import create_default_params
 from jedi.evaluate.helpers import is_stdlib_path
-from jedi.common import to_list, unite
+from jedi.evaluate.utils import to_list
 from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.context import ModuleContext, instance
+from jedi.evaluate.base_context import ContextSet
+
 
 
 MAX_PARAM_SEARCHES = 20
 
 
-class ParamListener(object):
-    """
-    This listener is used to get the params for a function.
-    """
-    def __init__(self):
-        self.param_possibilities = []
-
-    def execute(self, params):
-        self.param_possibilities += params
-
-
 class MergedExecutedParams(object):
     """
     Simulates being a parameter while actually just being multiple params.
@@ -50,7 +43,7 @@ class MergedExecutedParams(object):
         self._executed_params = executed_params
 
     def infer(self):
-        return unite(p.infer() for p in self._executed_params)
+        return ContextSet.from_sets(p.infer() for p in self._executed_params)
 
 
 @debug.increase_indent
@@ -103,14 +96,12 @@ def search_params(evaluator, execution_context, funcdef):
         evaluator.dynamic_params_depth -= 1
 
 
-@evaluator_function_cache(default=[])
+@evaluator_function_cache(default=None)
 @to_list
 def _search_function_executions(evaluator, module_context, funcdef):
     """
     Returns a list of param names.
     """
-    from jedi.evaluate import representation as er
-
     func_string_name = funcdef.name.value
     compare_node = funcdef
     if func_string_name == '__init__':
@@ -123,7 +114,7 @@ def _search_function_executions(evaluator, module_context, funcdef):
     i = 0
     for for_mod_context in imports.get_modules_containing_name(
             evaluator, [module_context], func_string_name):
-        if not isinstance(module_context, er.ModuleContext):
+        if not isinstance(module_context, ModuleContext):
             return
         for name, trailer in _get_possible_nodes(for_mod_context, func_string_name):
             i += 1
@@ -160,7 +151,7 @@ def _get_possible_nodes(module_context, func_string_name):
 
 
 def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
-    from jedi.evaluate import representation as er, instance
+    from jedi.evaluate.context.function import FunctionExecutionContext
 
     def create_func_excs():
         arglist = trailer.children[1]
@@ -184,7 +175,7 @@ def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
         if compare_node == value_node:
             for func_execution in create_func_excs():
                 yield func_execution
-        elif isinstance(value.parent_context, er.FunctionExecutionContext) and \
+        elif isinstance(value.parent_context, FunctionExecutionContext) and \
                 compare_node.type == 'funcdef':
             # Here we're trying to find decorators by checking the first
             # parameter. It's not very generic though. Should find a better

jedi/evaluate/filters.py

@@ -5,9 +5,12 @@ are needed for name resolution.
 from abc import abstractmethod
 
 from parso.tree import search_ancestor
+
+from jedi._compatibility import is_py3
 from jedi.evaluate import flow_analysis
-from jedi.common import to_list, unite
+from jedi.evaluate.base_context import ContextSet, Context
 from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.utils import to_list
 
 
 class AbstractNameDefinition(object):
@@ -35,10 +38,10 @@ class AbstractNameDefinition(object):
         return '<%s: %s@%s>' % (self.__class__.__name__, self.string_name, self.start_pos)
 
     def execute(self, arguments):
-        return unite(context.execute(arguments) for context in self.infer())
+        return self.infer().execute(arguments)
 
     def execute_evaluated(self, *args, **kwargs):
-        return unite(context.execute_evaluated(*args, **kwargs) for context in self.infer())
+        return self.infer().execute_evaluated(*args, **kwargs)
 
     @property
     def api_type(self):
@@ -64,7 +67,7 @@ class AbstractTreeName(AbstractNameDefinition):
 
 class ContextNameMixin(object):
     def infer(self):
-        return set([self._context])
+        return ContextSet(self._context)
 
     def get_root_context(self):
         if self.parent_context is None:
@@ -93,8 +96,8 @@ class TreeNameDefinition(AbstractTreeName):
 
     def infer(self):
         # Refactor this, should probably be here.
-        from jedi.evaluate.finder import _name_to_types
-        return _name_to_types(self.parent_context.evaluator, self.parent_context, self.tree_name)
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        return tree_name_to_contexts(self.parent_context.evaluator, self.parent_context, self.tree_name)
 
     @property
     def api_type(self):
@@ -128,7 +131,7 @@ class AnonymousInstanceParamName(ParamName):
         if param_node.position_index == 0:
             # This is a speed optimization, to return the self param (because
             # it's known). This only affects anonymous instances.
-            return set([self.parent_context.instance])
+            return ContextSet(self.parent_context.instance)
         else:
             return self.get_param().infer()
 
@@ -276,6 +279,92 @@ class DictFilter(AbstractFilter):
         return value
 
 
+class _BuiltinMappedMethod(Context):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    api_type = 'function'
+
+    def __init__(self, builtin_context, method, builtin_func):
+        super(_BuiltinMappedMethod, self).__init__(
+            builtin_context.evaluator,
+            parent_context=builtin_context
+        )
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, params):
+        return self._method(self.parent_context)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+class SpecialMethodFilter(DictFilter):
+    """
+    A filter for methods that are defined in this module on the corresponding
+    classes like Generator (for __next__, etc).
+    """
+    class SpecialMethodName(AbstractNameDefinition):
+        api_type = 'function'
+
+        def __init__(self, parent_context, string_name, callable_, builtin_context):
+            self.parent_context = parent_context
+            self.string_name = string_name
+            self._callable = callable_
+            self._builtin_context = builtin_context
+
+        def infer(self):
+            filter = next(self._builtin_context.get_filters())
+            # We can take the first index, because on builtin methods there's
+            # always only going to be one name. The same is true for the
+            # inferred values.
+            builtin_func = next(iter(filter.get(self.string_name)[0].infer()))
+            return ContextSet(_BuiltinMappedMethod(self.parent_context, self._callable, builtin_func))
+
+    def __init__(self, context, dct, builtin_context):
+        super(SpecialMethodFilter, self).__init__(dct)
+        self.context = context
+        self._builtin_context = builtin_context
+        """
+        This context is what will be used to introspect the name, where as the
+        other context will be used to execute the function.
+
+        We distinguish, because we have to.
+        """
+
+    def _convert(self, name, value):
+        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
+
+
+def has_builtin_methods(cls):
+    base_dct = {}
+    # Need to care properly about inheritance. Builtin Methods should not get
+    # lost, just because they are not mentioned in a class.
+    for base_cls in reversed(cls.__bases__):
+        try:
+            base_dct.update(base_cls.builtin_methods)
+        except AttributeError:
+            pass
+
+    cls.builtin_methods = base_dct
+    for func in cls.__dict__.values():
+        try:
+            cls.builtin_methods.update(func.registered_builtin_methods)
+        except AttributeError:
+            pass
+    return cls
+
+
+def register_builtin_method(method_name, python_version_match=None):
+    def wrapper(func):
+        if python_version_match and python_version_match != 2 + int(is_py3):
+            # Some functions do only apply to certain versions.
+            return func
+        dct = func.__dict__.setdefault('registered_builtin_methods', {})
+        dct[method_name] = func
+        return func
+    return wrapper
+
+
 def get_global_filters(evaluator, context, until_position, origin_scope):
     """
     Returns all filters in order of priority for name resolution.
@@ -326,7 +415,7 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
     >>> filters[4].values()                              #doctest: +ELLIPSIS
     [<CompiledName: ...>, ...]
     """
-    from jedi.evaluate.representation import FunctionExecutionContext
+    from jedi.evaluate.context.function import FunctionExecutionContext
     while context is not None:
         # Names in methods cannot be resolved within the class.
         for filter in context.get_filters(

jedi/evaluate/finder.py

@@ -18,20 +18,16 @@ check for -> a is a string). There's big potential in these checks.
 from parso.python import tree
 from parso.tree import search_ancestor
 from jedi import debug
-from jedi.common import unite
 from jedi import settings
-from jedi.evaluate import representation as er
-from jedi.evaluate.instance import AbstractInstanceContext
+from jedi.evaluate.context import AbstractInstanceContext
 from jedi.evaluate import compiled
-from jedi.evaluate import pep0484
-from jedi.evaluate import iterable
-from jedi.evaluate import imports
 from jedi.evaluate import analysis
 from jedi.evaluate import flow_analysis
-from jedi.evaluate import param
+from jedi.evaluate.arguments import TreeArguments
 from jedi.evaluate import helpers
+from jedi.evaluate.context import iterable
 from jedi.evaluate.filters import get_global_filters, TreeNameDefinition
-from jedi.evaluate.context import ContextualizedName, ContextualizedNode
+from jedi.evaluate.base_context import ContextSet
 from jedi.parser_utils import is_scope, get_parent_scope
 
 
@@ -62,7 +58,7 @@ class NameFinder(object):
             check = flow_analysis.reachability_check(
                 self._context, self._context.tree_node, self._name)
             if check is flow_analysis.UNREACHABLE:
-                return set()
+                return ContextSet()
             return self._found_predefined_types
 
         types = self._names_to_types(names, attribute_lookup)
@@ -158,22 +154,20 @@ class NameFinder(object):
         return inst.execute_function_slots(names, name)
 
     def _names_to_types(self, names, attribute_lookup):
-        types = set()
+        contexts = ContextSet.from_sets(name.infer() for name in names)
 
-        types = unite(name.infer() for name in names)
-
-        debug.dbg('finder._names_to_types: %s -> %s', names, types)
+        debug.dbg('finder._names_to_types: %s -> %s', names, contexts)
         if not names and isinstance(self._context, AbstractInstanceContext):
             # handling __getattr__ / __getattribute__
             return self._check_getattr(self._context)
 
         # Add isinstance and other if/assert knowledge.
-        if not types and isinstance(self._name, tree.Name) and \
+        if not contexts and isinstance(self._name, tree.Name) and \
                 not isinstance(self._name_context, AbstractInstanceContext):
             flow_scope = self._name
             base_node = self._name_context.tree_node
             if base_node.type == 'comp_for':
-                return types
+                return contexts
             while True:
                 flow_scope = get_parent_scope(flow_scope, include_flows=True)
                 n = _check_flow_information(self._name_context, flow_scope,
@@ -182,132 +176,7 @@ class NameFinder(object):
                     return n
                 if flow_scope == base_node:
                     break
-        return types
-
-
-def _name_to_types(evaluator, context, tree_name):
-    types = []
-    node = tree_name.get_definition(import_name_always=True)
-    if node is None:
-        node = tree_name.parent
-        if node.type == 'global_stmt':
-            context = evaluator.create_context(context, tree_name)
-            finder = NameFinder(evaluator, context, context, tree_name.value)
-            filters = finder.get_filters(search_global=True)
-            # For global_stmt lookups, we only need the first possible scope,
-            # which means the function itself.
-            filters = [next(filters)]
-            return finder.find(filters, attribute_lookup=False)
-        elif node.type not in ('import_from', 'import_name'):
-            raise ValueError("Should not happen.")
-
-    typ = node.type
-    if typ == 'for_stmt':
-        types = pep0484.find_type_from_comment_hint_for(context, node, tree_name)
-        if types:
-            return types
-    if typ == 'with_stmt':
-        types = pep0484.find_type_from_comment_hint_with(context, node, tree_name)
-        if types:
-            return types
-    if typ in ('for_stmt', 'comp_for'):
-        try:
-            types = context.predefined_names[node][tree_name.value]
-        except KeyError:
-            cn = ContextualizedNode(context, node.children[3])
-            for_types = iterable.py__iter__types(evaluator, cn.infer(), cn)
-            c_node = ContextualizedName(context, tree_name)
-            types = check_tuple_assignments(evaluator, c_node, for_types)
-    elif typ == 'expr_stmt':
-        types = _remove_statements(evaluator, context, node, tree_name)
-    elif typ == 'with_stmt':
-        context_managers = context.eval_node(node.get_test_node_from_name(tree_name))
-        enter_methods = unite(
-            context_manager.py__getattribute__('__enter__')
-            for context_manager in context_managers
-        )
-        types = unite(method.execute_evaluated() for method in enter_methods)
-    elif typ in ('import_from', 'import_name'):
-        types = imports.infer_import(context, tree_name)
-    elif typ in ('funcdef', 'classdef'):
-        types = _apply_decorators(evaluator, context, node)
-    elif typ == 'try_stmt':
-        # TODO an exception can also be a tuple. Check for those.
-        # TODO check for types that are not classes and add it to
-        # the static analysis report.
-        exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling())
-        types = unite(
-            evaluator.execute(t, param.ValuesArguments([]))
-            for t in exceptions
-        )
-    else:
-        raise ValueError("Should not happen.")
-    return types
-
-
-def _apply_decorators(evaluator, context, node):
-    """
-    Returns the function, that should to be executed in the end.
-    This is also the places where the decorators are processed.
-    """
-    if node.type == 'classdef':
-        decoratee_context = er.ClassContext(
-            evaluator,
-            parent_context=context,
-            classdef=node
-        )
-    else:
-        decoratee_context = er.FunctionContext(
-            evaluator,
-            parent_context=context,
-            funcdef=node
-        )
-    initial = values = set([decoratee_context])
-    for dec in reversed(node.get_decorators()):
-        debug.dbg('decorator: %s %s', dec, values)
-        dec_values = context.eval_node(dec.children[1])
-        trailer_nodes = dec.children[2:-1]
-        if trailer_nodes:
-            # Create a trailer and evaluate it.
-            trailer = tree.PythonNode('trailer', trailer_nodes)
-            trailer.parent = dec
-            dec_values = evaluator.eval_trailer(context, dec_values, trailer)
-
-        if not len(dec_values):
-            debug.warning('decorator not found: %s on %s', dec, node)
-            return initial
-
-        values = unite(dec_value.execute(param.ValuesArguments([values]))
-                       for dec_value in dec_values)
-        if not len(values):
-            debug.warning('not possible to resolve wrappers found %s', node)
-            return initial
-
-        debug.dbg('decorator end %s', values)
-    return values
-
-
-def _remove_statements(evaluator, context, stmt, name):
-    """
-    This is the part where statements are being stripped.
-
-    Due to lazy evaluation, statements like a = func; b = a; b() have to be
-    evaluated.
-    """
-    types = set()
-    check_instance = None
-
-    pep0484types = \
-        pep0484.find_type_from_comment_hint_assign(context, stmt, name)
-    if pep0484types:
-        return pep0484types
-    types |= context.eval_stmt(stmt, seek_name=name)
-
-    if check_instance is not None:
-        # class renames
-        types = set([er.get_instance_el(evaluator, check_instance, a, True)
-                     if isinstance(a, er.Function) else a for a in types])
-    return types
+        return contexts
 
 
 def _check_flow_information(context, flow, search_name, pos):
@@ -362,7 +231,7 @@ def _check_isinstance_type(context, element, search_name):
 
         # arglist stuff
         arglist = trailer.children[1]
-        args = param.TreeArguments(context.evaluator, context, arglist, trailer)
+        args = TreeArguments(context.evaluator, context, arglist, trailer)
         param_list = list(args.unpack())
         # Disallow keyword arguments
         assert len(param_list) == 2
@@ -377,34 +246,13 @@ def _check_isinstance_type(context, element, search_name):
     except AssertionError:
         return None
 
-    result = set()
+    context_set = ContextSet()
     for cls_or_tup in lazy_context_cls.infer():
-        if isinstance(cls_or_tup, iterable.AbstractSequence) and \
+        if isinstance(cls_or_tup, iterable.AbstractIterable) and \
                 cls_or_tup.array_type == 'tuple':
             for lazy_context in cls_or_tup.py__iter__():
                 for context in lazy_context.infer():
-                    result |= context.execute_evaluated()
+                    context_set |= context.execute_evaluated()
         else:
-            result |= cls_or_tup.execute_evaluated()
-    return result
-
-
-def check_tuple_assignments(evaluator, contextualized_name, types):
-    """
-    Checks if tuples are assigned.
-    """
-    lazy_context = None
-    for index, node in contextualized_name.assignment_indexes():
-        cn = ContextualizedNode(contextualized_name.context, node)
-        iterated = iterable.py__iter__(evaluator, types, cn)
-        for _ in range(index + 1):
-            try:
-                lazy_context = next(iterated)
-            except StopIteration:
-                # We could do this with the default param in next. But this
-                # would allow this loop to run for a very long time if the
-                # index number is high. Therefore break if the loop is
-                # finished.
-                return set()
-        types = lazy_context.infer()
-    return types
+            context_set |= cls_or_tup.execute_evaluated()
+    return context_set

jedi/evaluate/helpers.py

@@ -6,7 +6,10 @@ from itertools import chain
 from contextlib import contextmanager
 
 from parso.python import tree
+
+from jedi._compatibility import unicode
 from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.compiled import CompiledObject
 
 
 def is_stdlib_path(path):
@@ -55,8 +58,11 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
 
     If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
 
-    # TODO remove cut_own_trailer option, since its always used with it. Just
-    #      ignore it, It's not what we want anyway. Or document it better?
+    We use this function for two purposes. Given an expression ``bar.foo``,
+    we may want to
+      - infer the type of ``foo`` to offer completions after foo
+      - infer the type of ``bar`` to be able to jump to the definition of foo
+    The option ``cut_own_trailer`` must be set to true for the second purpose.
     """
     trailer = leaf.parent
     # The leaf may not be the last or first child, because there exist three
@@ -86,9 +92,14 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
         base = power.children[0]
         trailers = power.children[1:cut]
 
+    if base == 'await':
+        base = trailers[0]
+        trailers = trailers[1:]
+
     values = context.eval_node(base)
+    from jedi.evaluate.syntax_tree import eval_trailer
     for trailer in trailers:
-        values = context.eval_trailer(values, trailer)
+        values = eval_trailer(context, values, trailer)
     return values
 
 
@@ -172,3 +183,19 @@ def predefine_names(context, flow_scope, dct):
         yield
     finally:
         del predefined[flow_scope]
+
+
+def is_compiled(context):
+    return isinstance(context, CompiledObject)
+
+
+def is_string(context):
+    return is_compiled(context) and isinstance(context.obj, (str, unicode))
+
+
+def is_literal(context):
+    return is_number(context) or is_string(context)
+
+
+def is_number(context):
+    return is_compiled(context) and isinstance(context.obj, (int, float))

jedi/evaluate/imports.py

@@ -24,18 +24,19 @@ from parso import python_bytes_to_unicode
 from jedi._compatibility import find_module, unicode, ImplicitNSInfo
 from jedi import debug
 from jedi import settings
-from jedi.common import unite
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi.evaluate import compiled
 from jedi.evaluate import analysis
+from jedi.evaluate.utils import unite
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.filters import AbstractNameDefinition
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
 
 
 # This memoization is needed, because otherwise we will infinitely loop on
 # certain imports.
-@evaluator_method_cache(default=set())
+@evaluator_method_cache(default=NO_CONTEXTS)
 def infer_import(context, tree_name, is_goto=False):
     module_context = context.get_root_context()
     import_node = search_ancestor(tree_name, 'import_name', 'import_from')
@@ -63,7 +64,7 @@ def infer_import(context, tree_name, is_goto=False):
     #    scopes = [NestedImportModule(module, import_node)]
 
     if not types:
-        return set()
+        return NO_CONTEXTS
 
     if from_import_name is not None:
         types = unite(
@@ -72,8 +73,11 @@ def infer_import(context, tree_name, is_goto=False):
                 name_context=context,
                 is_goto=is_goto,
                 analysis_errors=False
-            ) for t in types
+            )
+            for t in types
         )
+        if not is_goto:
+            types = ContextSet.from_set(types)
 
         if not types:
             path = import_path + [from_import_name]
@@ -248,10 +252,8 @@ class Importer(object):
 
     def sys_path_with_modifications(self):
         in_path = []
-        sys_path_mod = list(sys_path.sys_path_with_modifications(
-            self._evaluator,
-            self.module_context
-        ))
+        sys_path_mod = self._evaluator.project.sys_path \
+                       + sys_path.check_sys_path_modifications(self.module_context)
         if self.file_path is not None:
             # If you edit e.g. gunicorn, there will be imports like this:
             # `from gunicorn import something`. But gunicorn is not in the
@@ -270,7 +272,7 @@ class Importer(object):
 
     def follow(self):
         if not self.import_path:
-            return set()
+            return NO_CONTEXTS
         return self._do_import(self.import_path, self.sys_path_with_modifications())
 
     def _do_import(self, import_path, sys_path):
@@ -296,7 +298,7 @@ class Importer(object):
 
         module_name = '.'.join(import_parts)
         try:
-            return set([self._evaluator.modules[module_name]])
+            return ContextSet(self._evaluator.modules[module_name])
         except KeyError:
             pass
 
@@ -305,7 +307,7 @@ class Importer(object):
             # the module cache.
             bases = self._do_import(import_path[:-1], sys_path)
             if not bases:
-                return set()
+                return NO_CONTEXTS
             # We can take the first element, because only the os special
             # case yields multiple modules, which is not important for
             # further imports.
@@ -323,7 +325,7 @@ class Importer(object):
             except AttributeError:
                 # The module is not a package.
                 _add_error(self.module_context, import_path[-1])
-                return set()
+                return NO_CONTEXTS
             else:
                 paths = method()
                 debug.dbg('search_module %s in paths %s', module_name, paths)
@@ -340,7 +342,7 @@ class Importer(object):
                         module_path = None
                 if module_path is None:
                     _add_error(self.module_context, import_path[-1])
-                    return set()
+                    return NO_CONTEXTS
         else:
             parent_module = None
             try:
@@ -356,7 +358,7 @@ class Importer(object):
             except ImportError:
                 # The module is not a package.
                 _add_error(self.module_context, import_path[-1])
-                return set()
+                return NO_CONTEXTS
 
         code = None
         if is_pkg:
@@ -371,7 +373,7 @@ class Importer(object):
             module_file.close()
 
         if isinstance(module_path, ImplicitNSInfo):
-            from jedi.evaluate.representation import ImplicitNamespaceContext
+            from jedi.evaluate.context.namespace import ImplicitNamespaceContext
             fullname, paths = module_path.name, module_path.paths
             module = ImplicitNamespaceContext(self._evaluator, fullname=fullname)
             module.paths = paths
@@ -383,10 +385,10 @@ class Importer(object):
         if module is None:
             # The file might raise an ImportError e.g. and therefore not be
             # importable.
-            return set()
+            return NO_CONTEXTS
 
         self._evaluator.modules[module_name] = module
-        return set([module])
+        return ContextSet(module)
 
     def _generate_name(self, name, in_module=None):
         # Create a pseudo import to be able to follow them.
@@ -416,7 +418,8 @@ class Importer(object):
         :param only_modules: Indicates wheter it's possible to import a
             definition that is not defined in a module.
         """
-        from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext
+        from jedi.evaluate.context import ModuleContext
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
         names = []
         if self.import_path:
             # flask
@@ -475,7 +478,7 @@ class Importer(object):
 
 def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None):
     if sys_path is None:
-        sys_path = evaluator.sys_path
+        sys_path = evaluator.project.sys_path
 
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
@@ -485,7 +488,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N
             code=code, path=path, cache=True, diff_cache=True,
             cache_path=settings.cache_directory)
 
-        from jedi.evaluate.representation import ModuleContext
+        from jedi.evaluate.context import ModuleContext
         return ModuleContext(evaluator, module_node, path=path)
     else:
         return compiled.load_module(evaluator, path)
@@ -504,7 +507,17 @@ def get_modules_containing_name(evaluator, modules, name):
     """
     Search a name in the directories of modules.
     """
-    from jedi.evaluate import representation as er
+    from jedi.evaluate.context import ModuleContext
+    def check_directories(paths):
+        for p in paths:
+            if p is not None:
+                # We need abspath, because the seetings paths might not already
+                # have been converted to absolute paths.
+                d = os.path.dirname(os.path.abspath(p))
+                for file_name in os.listdir(d):
+                    path = os.path.join(d, file_name)
+                    if file_name.endswith('.py'):
+                        yield path
 
     def check_python_file(path):
         try:
@@ -517,7 +530,7 @@ def get_modules_containing_name(evaluator, modules, name):
                 return None
         else:
             module_node = node_cache_item.node
-            return er.ModuleContext(evaluator, module_node, path=path)
+            return ModuleContext(evaluator, module_node, path=path)
 
     def check_fs(path):
         with open(path, 'rb') as f:
@@ -525,7 +538,7 @@ def get_modules_containing_name(evaluator, modules, name):
             if name in code:
                 module = _load_module(evaluator, path, code)
 
-                module_name = sys_path.dotted_path_in_sys_path(evaluator.sys_path, path)
+                module_name = sys_path.dotted_path_in_sys_path(evaluator.project.sys_path, path)
                 if module_name is not None:
                     add_module(evaluator, module_name, module)
                 return module
@@ -544,17 +557,10 @@ def get_modules_containing_name(evaluator, modules, name):
     if not settings.dynamic_params_for_other_modules:
         return
 
-    paths = set(settings.additional_dynamic_modules)
-    for p in used_mod_paths:
-        if p is not None:
-            # We need abspath, because the seetings paths might not already
-            # have been converted to absolute paths.
-            d = os.path.dirname(os.path.abspath(p))
-            for file_name in os.listdir(d):
-                path = os.path.join(d, file_name)
-                if path not in used_mod_paths and path not in paths:
-                    if file_name.endswith('.py'):
-                        paths.add(path)
+    additional = set(os.path.abspath(p) for p in settings.additional_dynamic_modules)
+    # Check the directories of used modules.
+    paths = (additional | set(check_directories(used_mod_paths))) \
+            - used_mod_paths
 
     # Sort here to make issues less random.
     for p in sorted(paths):

jedi/evaluate/lazy_context.py

@@ -0,0 +1,61 @@
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+
+class AbstractLazyContext(object):
+    def __init__(self, data):
+        self.data = data
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.data)
+
+    def infer(self):
+        raise NotImplementedError
+
+
+class LazyKnownContext(AbstractLazyContext):
+    """data is a context."""
+    def infer(self):
+        return ContextSet(self.data)
+
+
+class LazyKnownContexts(AbstractLazyContext):
+    """data is a ContextSet."""
+    def infer(self):
+        return self.data
+
+
+class LazyUnknownContext(AbstractLazyContext):
+    def __init__(self):
+        super(LazyUnknownContext, self).__init__(None)
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class LazyTreeContext(AbstractLazyContext):
+    def __init__(self, context, node):
+        super(LazyTreeContext, self).__init__(node)
+        self._context = context
+        # We need to save the predefined names. It's an unfortunate side effect
+        # that needs to be tracked otherwise results will be wrong.
+        self._predefined_names = dict(context.predefined_names)
+
+    def infer(self):
+        old, self._context.predefined_names = \
+            self._context.predefined_names, self._predefined_names
+        try:
+            return self._context.eval_node(self.data)
+        finally:
+            self._context.predefined_names = old
+
+
+def get_merged_lazy_context(lazy_contexts):
+    if len(lazy_contexts) > 1:
+        return MergedLazyContexts(lazy_contexts)
+    else:
+        return lazy_contexts[0]
+
+
+class MergedLazyContexts(AbstractLazyContext):
+    """data is a list of lazy contexts."""
+    def infer(self):
+        return ContextSet.from_sets(l.infer() for l in self.data)

jedi/evaluate/param.py

@@ -1,230 +1,22 @@
 from collections import defaultdict
 
-from jedi._compatibility import zip_longest
-from jedi import debug
-from jedi import common
-from parso.python import tree
-from jedi.evaluate import iterable
+from jedi.evaluate.utils import PushBackIterator
 from jedi.evaluate import analysis
-from jedi.evaluate import context
+from jedi.evaluate.lazy_context import LazyKnownContext, \
+    LazyTreeContext, LazyUnknownContext
 from jedi.evaluate import docstrings
 from jedi.evaluate import pep0484
-from jedi.evaluate.filters import ParamName
+from jedi.evaluate.context import iterable
 
 
-def add_argument_issue(parent_context, error_name, lazy_context, message):
-    if isinstance(lazy_context, context.LazyTreeContext):
+def _add_argument_issue(parent_context, error_name, lazy_context, message):
+    if isinstance(lazy_context, LazyTreeContext):
         node = lazy_context.data
         if node.parent.type == 'argument':
             node = node.parent
         analysis.add(parent_context, error_name, node, message)
 
 
-def try_iter_content(types, depth=0):
-    """Helper method for static analysis."""
-    if depth > 10:
-        # It's possible that a loop has references on itself (especially with
-        # CompiledObject). Therefore don't loop infinitely.
-        return
-
-    for typ in types:
-        try:
-            f = typ.py__iter__
-        except AttributeError:
-            pass
-        else:
-            for lazy_context in f():
-                try_iter_content(lazy_context.infer(), depth + 1)
-
-
-class AbstractArguments():
-    context = None
-
-    def eval_argument_clinic(self, parameters):
-        """Uses a list with argument clinic information (see PEP 436)."""
-        iterator = self.unpack()
-        for i, (name, optional, allow_kwargs) in enumerate(parameters):
-            key, argument = next(iterator, (None, None))
-            if key is not None:
-                raise NotImplementedError
-            if argument is None and not optional:
-                debug.warning('TypeError: %s expected at least %s arguments, got %s',
-                              name, len(parameters), i)
-                raise ValueError
-            values = set() if argument is None else argument.infer()
-
-            if not values and not optional:
-                # For the stdlib we always want values. If we don't get them,
-                # that's ok, maybe something is too hard to resolve, however,
-                # we will not proceed with the evaluation of that function.
-                debug.warning('argument_clinic "%s" not resolvable.', name)
-                raise ValueError
-            yield values
-
-    def eval_all(self, funcdef=None):
-        """
-        Evaluates all arguments as a support for static analysis
-        (normally Jedi).
-        """
-        for key, lazy_context in self.unpack():
-            types = lazy_context.infer()
-            try_iter_content(types)
-
-    def get_calling_nodes(self):
-        raise NotImplementedError
-
-    def unpack(self, funcdef=None):
-        raise NotImplementedError
-
-    def get_params(self, execution_context):
-        return get_params(execution_context, self)
-
-
-class AnonymousArguments(AbstractArguments):
-    def get_params(self, execution_context):
-        from jedi.evaluate.dynamic import search_params
-        return search_params(
-            execution_context.evaluator,
-            execution_context,
-            execution_context.tree_node
-        )
-
-
-class TreeArguments(AbstractArguments):
-    def __init__(self, evaluator, context, argument_node, trailer=None):
-        """
-        The argument_node is either a parser node or a list of evaluated
-        objects. Those evaluated objects may be lists of evaluated objects
-        themselves (one list for the first argument, one for the second, etc).
-
-        :param argument_node: May be an argument_node or a list of nodes.
-        """
-        self.argument_node = argument_node
-        self.context = context
-        self._evaluator = evaluator
-        self.trailer = trailer  # Can be None, e.g. in a class definition.
-
-    def _split(self):
-        if isinstance(self.argument_node, (tuple, list)):
-            for el in self.argument_node:
-                yield 0, el
-        else:
-            if not (self.argument_node.type == 'arglist' or (
-                    # in python 3.5 **arg is an argument, not arglist
-                    (self.argument_node.type == 'argument') and
-                     self.argument_node.children[0] in ('*', '**'))):
-                yield 0, self.argument_node
-                return
-
-            iterator = iter(self.argument_node.children)
-            for child in iterator:
-                if child == ',':
-                    continue
-                elif child in ('*', '**'):
-                    yield len(child.value), next(iterator)
-                elif child.type == 'argument' and \
-                        child.children[0] in ('*', '**'):
-                    assert len(child.children) == 2
-                    yield len(child.children[0].value), child.children[1]
-                else:
-                    yield 0, child
-
-    def unpack(self, funcdef=None):
-        named_args = []
-        for star_count, el in self._split():
-            if star_count == 1:
-                arrays = self.context.eval_node(el)
-                iterators = [_iterate_star_args(self.context, a, el, funcdef)
-                             for a in arrays]
-                iterators = list(iterators)
-                for values in list(zip_longest(*iterators)):
-                    # TODO zip_longest yields None, that means this would raise
-                    # an exception?
-                    yield None, context.get_merged_lazy_context(
-                        [v for v in values if v is not None]
-                    )
-            elif star_count == 2:
-                arrays = self._evaluator.eval_element(self.context, el)
-                for dct in arrays:
-                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
-                        yield key, values
-            else:
-                if el.type == 'argument':
-                    c = el.children
-                    if len(c) == 3:  # Keyword argument.
-                        named_args.append((c[0].value, context.LazyTreeContext(self.context, c[2]),))
-                    else:  # Generator comprehension.
-                        # Include the brackets with the parent.
-                        comp = iterable.GeneratorComprehension(
-                            self._evaluator, self.context, self.argument_node.parent)
-                        yield None, context.LazyKnownContext(comp)
-                else:
-                    yield None, context.LazyTreeContext(self.context, el)
-
-        # Reordering var_args is necessary, because star args sometimes appear
-        # after named argument, but in the actual order it's prepended.
-        for named_arg in named_args:
-            yield named_arg
-
-    def as_tree_tuple_objects(self):
-        for star_count, argument in self._split():
-            if argument.type == 'argument':
-                argument, default = argument.children[::2]
-            else:
-                default = None
-            yield argument, default, star_count
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
-
-    def get_calling_nodes(self):
-        from jedi.evaluate.dynamic import MergedExecutedParams
-        old_arguments_list = []
-        arguments = self
-
-        while arguments not in old_arguments_list:
-            if not isinstance(arguments, TreeArguments):
-                break
-
-            old_arguments_list.append(arguments)
-            for name, default, star_count in reversed(list(arguments.as_tree_tuple_objects())):
-                if not star_count or not isinstance(name, tree.Name):
-                    continue
-
-                names = self._evaluator.goto(arguments.context, name)
-                if len(names) != 1:
-                    break
-                if not isinstance(names[0], ParamName):
-                    break
-                param = names[0].get_param()
-                if isinstance(param, MergedExecutedParams):
-                    # For dynamic searches we don't even want to see errors.
-                    return []
-                if not isinstance(param, ExecutedParam):
-                    break
-                if param.var_args is None:
-                    break
-                arguments = param.var_args
-                break
-
-        return [arguments.argument_node or arguments.trailer]
-
-
-class ValuesArguments(AbstractArguments):
-    def __init__(self, values_list):
-        self._values_list = values_list
-
-    def unpack(self, funcdef=None):
-        for values in self._values_list:
-            yield None, context.LazyKnownContexts(values)
-
-    def get_calling_nodes(self):
-        return []
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
-
-
 class ExecutedParam(object):
     """Fake a param and give it values."""
     def __init__(self, execution_context, param_node, lazy_context):
@@ -237,7 +29,7 @@ class ExecutedParam(object):
         pep0484_hints = pep0484.infer_param(self._execution_context, self._param_node)
         doc_params = docstrings.infer_param(self._execution_context, self._param_node)
         if pep0484_hints or doc_params:
-            return list(set(pep0484_hints) | set(doc_params))
+            return pep0484_hints | doc_params
 
         return self._lazy_context.infer()
 
@@ -258,7 +50,7 @@ def get_params(execution_context, var_args):
     for param in funcdef.get_params():
         param_dict[param.name.value] = param
     unpacked_va = list(var_args.unpack(funcdef))
-    var_arg_iterator = common.PushBackIterator(iter(unpacked_va))
+    var_arg_iterator = PushBackIterator(iter(unpacked_va))
 
     non_matching_keys = defaultdict(lambda: [])
     keys_used = {}
@@ -306,30 +98,30 @@ def get_params(execution_context, var_args):
                         break
                     lazy_context_list.append(argument)
             seq = iterable.FakeSequence(execution_context.evaluator, 'tuple', lazy_context_list)
-            result_arg = context.LazyKnownContext(seq)
+            result_arg = LazyKnownContext(seq)
         elif param.star_count == 2:
             # **kwargs param
             dct = iterable.FakeDict(execution_context.evaluator, dict(non_matching_keys))
-            result_arg = context.LazyKnownContext(dct)
+            result_arg = LazyKnownContext(dct)
             non_matching_keys = {}
         else:
             # normal param
             if argument is None:
                 # No value: Return an empty container
                 if param.default is None:
-                    result_arg = context.LazyUnknownContext()
+                    result_arg = LazyUnknownContext()
                     if not keys_only:
                         for node in var_args.get_calling_nodes():
                             m = _error_argument_count(funcdef, len(unpacked_va))
                             analysis.add(parent_context, 'type-error-too-few-arguments',
                                          node, message=m)
                 else:
-                    result_arg = context.LazyTreeContext(parent_context, param.default)
+                    result_arg = LazyTreeContext(parent_context, param.default)
             else:
                 result_arg = argument
 
         result_params.append(ExecutedParam(execution_context, param, result_arg))
-        if not isinstance(result_arg, context.LazyUnknownContext):
+        if not isinstance(result_arg, LazyUnknownContext):
             keys_used[param.name.value] = result_params[-1]
 
     if keys_only:
@@ -350,7 +142,7 @@ def get_params(execution_context, var_args):
     for key, lazy_context in non_matching_keys.items():
         m = "TypeError: %s() got an unexpected keyword argument '%s'." \
             % (funcdef.name, key)
-        add_argument_issue(
+        _add_argument_issue(
             parent_context,
             'type-error-keyword-argument',
             lazy_context,
@@ -365,40 +157,10 @@ def get_params(execution_context, var_args):
         first_key, lazy_context = remaining_arguments[0]
         if var_args.get_calling_nodes():
             # There might not be a valid calling node so check for that first.
-            add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m)
+            _add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m)
     return result_params
 
 
-def _iterate_star_args(context, array, input_node, funcdef=None):
-    try:
-        iter_ = array.py__iter__
-    except AttributeError:
-        if funcdef is not None:
-            # TODO this funcdef should not be needed.
-            m = "TypeError: %s() argument after * must be a sequence, not %s" \
-                % (funcdef.name.value, array)
-            analysis.add(context, 'type-error-star', input_node, message=m)
-    else:
-        for lazy_context in iter_():
-            yield lazy_context
-
-
-def _star_star_dict(context, array, input_node, funcdef):
-    from jedi.evaluate.instance import CompiledInstance
-    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
-        # For now ignore this case. In the future add proper iterators and just
-        # make one call without crazy isinstance checks.
-        return {}
-    elif isinstance(array, iterable.AbstractSequence) and array.array_type == 'dict':
-        return array.exact_key_items()
-    else:
-        if funcdef is not None:
-            m = "TypeError: %s argument after ** must be a mapping, not %s" \
-                % (funcdef.name.value, array)
-            analysis.add(context, 'type-error-star-star', input_node, message=m)
-        return {}
-
-
 def _error_argument_count(funcdef, actual_count):
     params = funcdef.get_params()
     default_arguments = sum(1 for p in params if p.default or p.star_count)
@@ -413,17 +175,17 @@ def _error_argument_count(funcdef, actual_count):
 
 def _create_default_param(execution_context, param):
     if param.star_count == 1:
-        result_arg = context.LazyKnownContext(
+        result_arg = LazyKnownContext(
             iterable.FakeSequence(execution_context.evaluator, 'tuple', [])
         )
     elif param.star_count == 2:
-        result_arg = context.LazyKnownContext(
+        result_arg = LazyKnownContext(
             iterable.FakeDict(execution_context.evaluator, {})
         )
     elif param.default is None:
-        result_arg = context.LazyUnknownContext()
+        result_arg = LazyUnknownContext()
     else:
-        result_arg = context.LazyTreeContext(execution_context.parent_context, param.default)
+        result_arg = LazyTreeContext(execution_context.parent_context, param.default)
     return ExecutedParam(execution_context, param, result_arg)
 
 

jedi/evaluate/pep0484.py

@@ -19,17 +19,17 @@ x support for type hint comments for functions, `# type: (int, str) -> int`.
     See comment from Guido https://github.com/davidhalter/jedi/issues/662
 """
 
-import itertools
 import os
 import re
 
 from parso import ParserSyntaxError
 from parso.python import tree
 
-from jedi.common import unite
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate import compiled
-from jedi.evaluate.context import LazyTreeContext
+from jedi.evaluate.base_context import NO_CONTEXTS, ContextSet
+from jedi.evaluate.lazy_context import LazyTreeContext
+from jedi.evaluate.context import ModuleContext
 from jedi import debug
 from jedi import _compatibility
 from jedi import parser_utils
@@ -42,16 +42,15 @@ def _evaluate_for_annotation(context, annotation, index=None):
     and we're interested in that index
     """
     if annotation is not None:
-        definitions = context.eval_node(
-            _fix_forward_reference(context, annotation))
+        context_set = context.eval_node(_fix_forward_reference(context, annotation))
         if index is not None:
-            definitions = list(itertools.chain.from_iterable(
-                definition.py__getitem__(index) for definition in definitions
-                if definition.array_type == 'tuple' and
-                len(list(definition.py__iter__())) >= index))
-        return unite(d.execute_evaluated() for d in definitions)
+            context_set = context_set.filter(
+                lambda context: context.array_type == 'tuple' \
+                                and len(list(context.py__iter__())) >= index
+            ).py__getitem__(index)
+        return context_set.execute_evaluated()
     else:
-        return set()
+        return NO_CONTEXTS
 
 
 def _fix_forward_reference(context, node):
@@ -147,13 +146,12 @@ def py__getitem__(context, typ, node):
     if type_name in ("Union", '_Union'):
         # In Python 3.6 it's still called typing.Union but it's an instance
         # called _Union.
-        return unite(context.eval_node(node) for node in nodes)
+        return ContextSet.from_sets(context.eval_node(node) for node in nodes)
     if type_name in ("Optional", '_Optional'):
         # Here we have the same issue like in Union. Therefore we also need to
         # check for the instance typing._Optional (Python 3.6).
         return context.eval_node(nodes[0])
 
-    from jedi.evaluate.representation import ModuleContext
     typing = ModuleContext(
         context.evaluator,
         module_node=_get_typing_replacement_module(context.evaluator.latest_grammar),
@@ -171,7 +169,7 @@ def py__getitem__(context, typ, node):
         return None
     compiled_classname = compiled.create(context.evaluator, type_name)
 
-    from jedi.evaluate.iterable import FakeSequence
+    from jedi.evaluate.context.iterable import FakeSequence
     args = FakeSequence(
         context.evaluator,
         "tuple",

jedi/evaluate/precedence.py

@@ -1,178 +0,0 @@
-"""
-Handles operator precedence.
-"""
-import operator as op
-
-from jedi._compatibility import unicode
-from jedi import debug
-from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
-from jedi.evaluate import analysis
-
-# Maps Python syntax to the operator module.
-COMPARISON_OPERATORS = {
-    '==': op.eq,
-    '!=': op.ne,
-    'is': op.is_,
-    'is not': op.is_not,
-    '<': op.lt,
-    '<=': op.le,
-    '>': op.gt,
-    '>=': op.ge,
-}
-
-
-def literals_to_types(evaluator, result):
-    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
-    # int(), float(), etc).
-    new_result = set()
-    for typ in result:
-        if is_literal(typ):
-            # Literals are only valid as long as the operations are
-            # correct. Otherwise add a value-free instance.
-            cls = builtin_from_name(evaluator, typ.name.string_name)
-            new_result |= cls.execute_evaluated()
-        else:
-            new_result.add(typ)
-    return new_result
-
-
-def calculate_children(evaluator, context, children):
-    """
-    Calculate a list of children with operators.
-    """
-    iterator = iter(children)
-    types = context.eval_node(next(iterator))
-    for operator in iterator:
-        right = next(iterator)
-        if operator.type == 'comp_op':  # not in / is not
-            operator = ' '.join(c.value for c in operator.children)
-
-        # handle lazy evaluation of and/or here.
-        if operator in ('and', 'or'):
-            left_bools = set([left.py__bool__() for left in types])
-            if left_bools == set([True]):
-                if operator == 'and':
-                    types = context.eval_node(right)
-            elif left_bools == set([False]):
-                if operator != 'and':
-                    types = context.eval_node(right)
-            # Otherwise continue, because of uncertainty.
-        else:
-            types = calculate(evaluator, context, types, operator,
-                              context.eval_node(right))
-    debug.dbg('calculate_children types %s', types)
-    return types
-
-
-def calculate(evaluator, context, left_result, operator, right_result):
-    result = set()
-    if not left_result or not right_result:
-        # illegal slices e.g. cause left/right_result to be None
-        result = (left_result or set()) | (right_result or set())
-        result = literals_to_types(evaluator, result)
-    else:
-        # I don't think there's a reasonable chance that a string
-        # operation is still correct, once we pass something like six
-        # objects.
-        if len(left_result) * len(right_result) > 6:
-            result = literals_to_types(evaluator, left_result | right_result)
-        else:
-            for left in left_result:
-                for right in right_result:
-                    result |= _element_calculate(evaluator, context, left, operator, right)
-    return result
-
-
-def factor_calculate(evaluator, types, operator):
-    """
-    Calculates `+`, `-`, `~` and `not` prefixes.
-    """
-    for typ in types:
-        if operator == '-':
-            if _is_number(typ):
-                yield create(evaluator, -typ.obj)
-        elif operator == 'not':
-            value = typ.py__bool__()
-            if value is None:  # Uncertainty.
-                return
-            yield create(evaluator, not value)
-        else:
-            yield typ
-
-
-def _is_number(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (int, float))
-
-
-def is_string(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (str, unicode))
-
-
-def is_literal(obj):
-    return _is_number(obj) or is_string(obj)
-
-
-def _is_tuple(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'tuple'
-
-
-def _is_list(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'list'
-
-
-def _element_calculate(evaluator, context, left, operator, right):
-    from jedi.evaluate import iterable, instance
-    l_is_num = _is_number(left)
-    r_is_num = _is_number(right)
-    if operator == '*':
-        # for iterables, ignore * operations
-        if isinstance(left, iterable.AbstractSequence) or is_string(left):
-            return set([left])
-        elif isinstance(right, iterable.AbstractSequence) or is_string(right):
-            return set([right])
-    elif operator == '+':
-        if l_is_num and r_is_num or is_string(left) and is_string(right):
-            return set([create(evaluator, left.obj + right.obj)])
-        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
-            return set([iterable.MergedArray(evaluator, (left, right))])
-    elif operator == '-':
-        if l_is_num and r_is_num:
-            return set([create(evaluator, left.obj - right.obj)])
-    elif operator == '%':
-        # With strings and numbers the left type typically remains. Except for
-        # `int() % float()`.
-        return set([left])
-    elif operator in COMPARISON_OPERATORS:
-        operation = COMPARISON_OPERATORS[operator]
-        if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
-            # Possible, because the return is not an option. Just compare.
-            left = left.obj
-            right = right.obj
-
-        try:
-            result = operation(left, right)
-        except TypeError:
-            # Could be True or False.
-            return set([create(evaluator, True), create(evaluator, False)])
-        else:
-            return set([create(evaluator, result)])
-    elif operator == 'in':
-        return set()
-
-    def check(obj):
-        """Checks if a Jedi object is either a float or an int."""
-        return isinstance(obj, instance.CompiledInstance) and \
-            obj.name.string_name in ('int', 'float')
-
-    # Static analysis, one is a number, the other one is not.
-    if operator in ('+', '-') and l_is_num != r_is_num \
-            and not (check(left) or check(right)):
-        message = "TypeError: unsupported operand type(s) for +: %s and %s"
-        analysis.add(context, 'type-error-operation', operator,
-                     message % (left, right))
-
-    return set([left, right])

jedi/evaluate/project.py

@@ -0,0 +1,40 @@
+import os
+import sys
+
+from jedi.evaluate.sys_path import get_venv_path, detect_additional_paths
+from jedi.cache import underscore_memoization
+
+
+class Project(object):
+    def __init__(self, sys_path=None):
+        if sys_path is not None:
+            self._sys_path = sys_path
+
+        venv = os.getenv('VIRTUAL_ENV')
+        if venv:
+            sys_path = get_venv_path(venv)
+
+        if sys_path is None:
+            sys_path = sys.path
+
+        base_sys_path = list(sys_path)
+        try:
+            base_sys_path.remove('')
+        except ValueError:
+            pass
+
+        self._base_sys_path = base_sys_path
+
+    def add_script_path(self, script_path):
+        self._script_path = script_path
+
+    def add_evaluator(self, evaluator):
+        self._evaluator = evaluator
+
+    @property
+    @underscore_memoization
+    def sys_path(self):
+        if self._script_path is None:
+            return self._base_sys_path
+
+        return self._base_sys_path + detect_additional_paths(self._evaluator, self._script_path)

jedi/evaluate/recursion.py

@@ -29,6 +29,7 @@ therefore the quality might not always be maximal.
 from contextlib import contextmanager
 
 from jedi import debug
+from jedi.evaluate.base_context import NO_CONTEXTS
 
 
 recursion_limit = 15
@@ -71,7 +72,7 @@ def execution_allowed(evaluator, node):
         pushed_nodes.pop()
 
 
-def execution_recursion_decorator(default=set()):
+def execution_recursion_decorator(default=NO_CONTEXTS):
     def decorator(func):
         def wrapper(execution, **kwargs):
             detector = execution.evaluator.execution_recursion_detector

jedi/evaluate/representation.py

@@ -1,684 +0,0 @@
-"""
-Like described in the :mod:`parso.python.tree` module,
-there's a need for an ast like module to represent the states of parsed
-modules.
-
-But now there are also structures in Python that need a little bit more than
-that. An ``Instance`` for example is only a ``Class`` before it is
-instantiated. This class represents these cases.
-
-So, why is there also a ``Class`` class here? Well, there are decorators and
-they change classes in Python 3.
-
-Representation modules also define "magic methods". Those methods look like
-``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
-and others. Here's a list:
-
-====================================== ========================================
-**Method**                             **Description**
--------------------------------------- ----------------------------------------
-py__call__(params: Array)              On callable objects, returns types.
-py__bool__()                           Returns True/False/None; None means that
-                                       there's no certainty.
-py__bases__()                          Returns a list of base classes.
-py__mro__()                            Returns a list of classes (the mro).
-py__iter__()                           Returns a generator of a set of types.
-py__class__()                          Returns the class of an instance.
-py__getitem__(index: int/str)          Returns a a set of types of the index.
-                                       Can raise an IndexError/KeyError.
-py__file__()                           Only on modules. Returns None if does
-                                       not exist.
-py__package__()                        Only on modules. For the import system.
-py__path__()                           Only on modules. For the import system.
-py__get__(call_object)                 Only on instances. Simulates
-                                       descriptors.
-py__doc__(include_call_signature:      Returns the docstring for a context.
-          bool)
-====================================== ========================================
-
-"""
-import os
-import pkgutil
-import imp
-import re
-from itertools import chain
-
-from parso.python import tree
-from parso import python_bytes_to_unicode
-
-from jedi._compatibility import use_metaclass
-from jedi import debug
-from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
-from jedi.evaluate import compiled
-from jedi.evaluate import recursion
-from jedi.evaluate import iterable
-from jedi.evaluate import docstrings
-from jedi.evaluate import pep0484
-from jedi.evaluate import param
-from jedi.evaluate import flow_analysis
-from jedi.evaluate import imports
-from jedi.evaluate import helpers
-from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \
-    GlobalNameFilter, DictFilter, ContextName, AbstractNameDefinition, \
-    ParamName, AnonymousInstanceParamName, TreeNameDefinition, \
-    ContextNameMixin
-from jedi.evaluate import context
-from jedi.evaluate.context import ContextualizedNode
-from jedi import parser_utils
-from jedi.evaluate.parser_cache import get_yield_exprs
-
-
-def apply_py__get__(context, base_context):
-    try:
-        method = context.py__get__
-    except AttributeError:
-        yield context
-    else:
-        for descriptor_context in method(base_context):
-            yield descriptor_context
-
-
-class ClassName(TreeNameDefinition):
-    def __init__(self, parent_context, tree_name, name_context):
-        super(ClassName, self).__init__(parent_context, tree_name)
-        self._name_context = name_context
-
-    def infer(self):
-        # TODO this _name_to_types might get refactored and be a part of the
-        # parent class. Once it is, we can probably just overwrite method to
-        # achieve this.
-        from jedi.evaluate.finder import _name_to_types
-        inferred = _name_to_types(
-            self.parent_context.evaluator, self._name_context, self.tree_name)
-
-        for result_context in inferred:
-            for c in apply_py__get__(result_context, self.parent_context):
-                yield c
-
-
-class ClassFilter(ParserTreeFilter):
-    name_class = ClassName
-
-    def _convert_names(self, names):
-        return [self.name_class(self.context, name, self._node_context)
-                for name in names]
-
-
-class ClassContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    This class is not only important to extend `tree.Class`, it is also a
-    important for descriptors (if the descriptor methods are evaluated or not).
-    """
-    api_type = 'class'
-
-    def __init__(self, evaluator, classdef, parent_context):
-        super(ClassContext, self).__init__(evaluator, parent_context=parent_context)
-        self.tree_node = classdef
-
-    @evaluator_method_cache(default=())
-    def py__mro__(self):
-        def add(cls):
-            if cls not in mro:
-                mro.append(cls)
-
-        mro = [self]
-        # TODO Do a proper mro resolution. Currently we are just listing
-        # classes. However, it's a complicated algorithm.
-        for lazy_cls in self.py__bases__():
-            # TODO there's multiple different mro paths possible if this yields
-            # multiple possibilities. Could be changed to be more correct.
-            for cls in lazy_cls.infer():
-                # TODO detect for TypeError: duplicate base class str,
-                # e.g.  `class X(str, str): pass`
-                try:
-                    mro_method = cls.py__mro__
-                except AttributeError:
-                    # TODO add a TypeError like:
-                    """
-                    >>> class Y(lambda: test): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: function() argument 1 must be code, not str
-                    >>> class Y(1): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: int() takes at most 2 arguments (3 given)
-                    """
-                    pass
-                else:
-                    add(cls)
-                    for cls_new in mro_method():
-                        add(cls_new)
-        return tuple(mro)
-
-    @evaluator_method_cache(default=())
-    def py__bases__(self):
-        arglist = self.tree_node.get_super_arglist()
-        if arglist:
-            args = param.TreeArguments(self.evaluator, self, arglist)
-            return [value for key, value in args.unpack() if key is None]
-        else:
-            return [context.LazyKnownContext(compiled.create(self.evaluator, object))]
-
-    def py__call__(self, params):
-        from jedi.evaluate.instance import TreeInstance
-        return set([TreeInstance(self.evaluator, self.parent_context, self, params)])
-
-    def py__class__(self):
-        return compiled.create(self.evaluator, type)
-
-    def get_params(self):
-        from jedi.evaluate.instance import AnonymousInstance
-        anon = AnonymousInstance(self.evaluator, self.parent_context, self)
-        return [AnonymousInstanceParamName(anon, param.name) for param in self.funcdef.get_params()]
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            for cls in self.py__mro__():
-                if isinstance(cls, compiled.CompiledObject):
-                    for filter in cls.get_filters(is_instance=is_instance):
-                        yield filter
-                else:
-                    yield ClassFilter(
-                        self.evaluator, self, node_context=cls,
-                        origin_scope=origin_scope)
-
-    def is_class(self):
-        return True
-
-    def get_function_slot_names(self, name):
-        for filter in self.get_filters(search_global=False):
-            names = filter.get(name)
-            if names:
-                return names
-        return []
-
-    def get_param_names(self):
-        for name in self.get_function_slot_names('__init__'):
-            for context_ in name.infer():
-                try:
-                    method = context_.get_param_names
-                except AttributeError:
-                    pass
-                else:
-                    return list(method())[1:]
-        return []
-
-    @property
-    def name(self):
-        return ContextName(self, self.tree_node.name)
-
-
-class LambdaName(AbstractNameDefinition):
-    string_name = '<lambda>'
-
-    def __init__(self, lambda_context):
-        self._lambda_context = lambda_context
-        self.parent_context = lambda_context.parent_context
-
-    def start_pos(self):
-        return self._lambda_context.tree_node.start_pos
-
-    def infer(self):
-        return set([self._lambda_context])
-
-
-class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    Needed because of decorators. Decorators are evaluated here.
-    """
-    api_type = 'function'
-
-    def __init__(self, evaluator, parent_context, funcdef):
-        """ This should not be called directly """
-        super(FunctionContext, self).__init__(evaluator, parent_context)
-        self.tree_node = funcdef
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            scope = self.py__class__()
-            for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
-                yield filter
-
-    def infer_function_execution(self, function_execution):
-        """
-        Created to be used by inheritance.
-        """
-        yield_exprs = get_yield_exprs(self.evaluator, self.tree_node)
-        if yield_exprs:
-            return set([iterable.Generator(self.evaluator, function_execution)])
-        else:
-            return function_execution.get_return_values()
-
-    def get_function_execution(self, arguments=None):
-        if arguments is None:
-            arguments = param.AnonymousArguments()
-
-        return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
-
-    def py__call__(self, arguments):
-        function_execution = self.get_function_execution(arguments)
-        return self.infer_function_execution(function_execution)
-
-    def py__class__(self):
-        # This differentiation is only necessary for Python2. Python3 does not
-        # use a different method class.
-        if isinstance(parser_utils.get_parent_scope(self.tree_node), tree.Class):
-            name = 'METHOD_CLASS'
-        else:
-            name = 'FUNCTION_CLASS'
-        return compiled.get_special_object(self.evaluator, name)
-
-    @property
-    def name(self):
-        if self.tree_node.type == 'lambdef':
-            return LambdaName(self)
-        return ContextName(self, self.tree_node.name)
-
-    def get_param_names(self):
-        function_execution = self.get_function_execution()
-        return [ParamName(function_execution, param.name)
-                for param in self.tree_node.get_params()]
-
-
-class FunctionExecutionContext(context.TreeContext):
-    """
-    This class is used to evaluate functions and their returns.
-
-    This is the most complicated class, because it contains the logic to
-    transfer parameters. It is even more complicated, because there may be
-    multiple calls to functions and recursion has to be avoided. But this is
-    responsibility of the decorators.
-    """
-    function_execution_filter = FunctionExecutionFilter
-
-    def __init__(self, evaluator, parent_context, function_context, var_args):
-        super(FunctionExecutionContext, self).__init__(evaluator, parent_context)
-        self.function_context = function_context
-        self.tree_node = function_context.tree_node
-        self.var_args = var_args
-
-    @evaluator_method_cache(default=set())
-    @recursion.execution_recursion_decorator()
-    def get_return_values(self, check_yields=False):
-        funcdef = self.tree_node
-        if funcdef.type == 'lambdef':
-            return self.evaluator.eval_element(self, funcdef.children[-1])
-
-        if check_yields:
-            types = set()
-            returns = get_yield_exprs(self.evaluator, funcdef)
-        else:
-            returns = funcdef.iter_return_stmts()
-            types = set(docstrings.infer_return_types(self.function_context))
-            types |= set(pep0484.infer_return_types(self.function_context))
-
-        for r in returns:
-            check = flow_analysis.reachability_check(self, funcdef, r)
-            if check is flow_analysis.UNREACHABLE:
-                debug.dbg('Return unreachable: %s', r)
-            else:
-                if check_yields:
-                    types |= set(self._eval_yield(r))
-                else:
-                    try:
-                        children = r.children
-                    except AttributeError:
-                        types.add(compiled.create(self.evaluator, None))
-                    else:
-                        types |= self.eval_node(children[1])
-            if check is flow_analysis.REACHABLE:
-                debug.dbg('Return reachable: %s', r)
-                break
-        return types
-
-    def _eval_yield(self, yield_expr):
-        if yield_expr.type == 'keyword':
-            # `yield` just yields None.
-            yield context.LazyKnownContext(compiled.create(self.evaluator, None))
-            return
-
-        node = yield_expr.children[1]
-        if node.type == 'yield_arg':  # It must be a yield from.
-            cn = ContextualizedNode(self, node.children[1])
-            for lazy_context in iterable.py__iter__(self.evaluator, cn.infer(), cn):
-                yield lazy_context
-        else:
-            yield context.LazyTreeContext(self, node)
-
-    @recursion.execution_recursion_decorator(default=iter([]))
-    def get_yield_values(self):
-        for_parents = [(y, tree.search_ancestor(y, 'for_stmt', 'funcdef',
-                                                'while_stmt', 'if_stmt'))
-                       for y in get_yield_exprs(self.evaluator, self.tree_node)]
-
-        # Calculate if the yields are placed within the same for loop.
-        yields_order = []
-        last_for_stmt = None
-        for yield_, for_stmt in for_parents:
-            # For really simple for loops we can predict the order. Otherwise
-            # we just ignore it.
-            parent = for_stmt.parent
-            if parent.type == 'suite':
-                parent = parent.parent
-            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
-                    and parser_utils.for_stmt_defines_one_name(for_stmt):  # Simplicity for now.
-                if for_stmt == last_for_stmt:
-                    yields_order[-1][1].append(yield_)
-                else:
-                    yields_order.append((for_stmt, [yield_]))
-            elif for_stmt == self.tree_node:
-                yields_order.append((None, [yield_]))
-            else:
-                types = self.get_return_values(check_yields=True)
-                if types:
-                    yield context.get_merged_lazy_context(list(types))
-                return
-            last_for_stmt = for_stmt
-
-        evaluator = self.evaluator
-        for for_stmt, yields in yields_order:
-            if for_stmt is None:
-                # No for_stmt, just normal yields.
-                for yield_ in yields:
-                    for result in self._eval_yield(yield_):
-                        yield result
-            else:
-                input_node = for_stmt.get_testlist()
-                cn = ContextualizedNode(self, input_node)
-                ordered = iterable.py__iter__(evaluator, cn.infer(), cn)
-                ordered = list(ordered)
-                for lazy_context in ordered:
-                    dct = {str(for_stmt.children[1].value): lazy_context.infer()}
-                    with helpers.predefine_names(self, for_stmt, dct):
-                        for yield_in_same_for_stmt in yields:
-                            for result in self._eval_yield(yield_in_same_for_stmt):
-                                yield result
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield self.function_execution_filter(self.evaluator, self,
-                                             until_position=until_position,
-                                             origin_scope=origin_scope)
-
-    @evaluator_method_cache()
-    def get_params(self):
-        return self.var_args.get_params(self)
-
-
-class ModuleAttributeName(AbstractNameDefinition):
-    """
-    For module attributes like __file__, __str__ and so on.
-    """
-    api_type = 'instance'
-
-    def __init__(self, parent_module, string_name):
-        self.parent_context = parent_module
-        self.string_name = string_name
-
-    def infer(self):
-        return compiled.create(self.parent_context.evaluator, str).execute(
-            param.ValuesArguments([])
-        )
-
-
-class ModuleName(ContextNameMixin, AbstractNameDefinition):
-    start_pos = 1, 0
-
-    def __init__(self, context, name):
-        self._context = context
-        self._name = name
-
-    @property
-    def string_name(self):
-        return self._name
-
-
-class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    api_type = 'module'
-    parent_context = None
-
-    def __init__(self, evaluator, module_node, path):
-        super(ModuleContext, self).__init__(evaluator, parent_context=None)
-        self.tree_node = module_node
-        self._path = path
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield ParserTreeFilter(
-            self.evaluator,
-            context=self,
-            until_position=until_position,
-            origin_scope=origin_scope
-        )
-        yield GlobalNameFilter(self, self.tree_node)
-        yield DictFilter(self._sub_modules_dict())
-        yield DictFilter(self._module_attributes_dict())
-        for star_module in self.star_imports():
-            yield next(star_module.get_filters(search_global))
-
-    # I'm not sure if the star import cache is really that effective anymore
-    # with all the other really fast import caches. Recheck. Also we would need
-    # to push the star imports into Evaluator.modules, if we reenable this.
-    @evaluator_method_cache([])
-    def star_imports(self):
-        modules = []
-        for i in self.tree_node.iter_imports():
-            if i.is_star_import():
-                name = i.get_paths()[-1][-1]
-                new = imports.infer_import(self, name)
-                for module in new:
-                    if isinstance(module, ModuleContext):
-                        modules += module.star_imports()
-                modules += new
-        return modules
-
-    @evaluator_method_cache()
-    def _module_attributes_dict(self):
-        names = ['__file__', '__package__', '__doc__', '__name__']
-        # All the additional module attributes are strings.
-        return dict((n, ModuleAttributeName(self, n)) for n in names)
-
-    @property
-    def _string_name(self):
-        """ This is used for the goto functions. """
-        if self._path is None:
-            return ''  # no path -> empty name
-        else:
-            sep = (re.escape(os.path.sep),) * 2
-            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path)
-            # Remove PEP 3149 names
-            return re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
-
-    @property
-    @evaluator_method_cache()
-    def name(self):
-        return ModuleName(self, self._string_name)
-
-    def _get_init_directory(self):
-        """
-        :return: The path to the directory of a package. None in case it's not
-                 a package.
-        """
-        for suffix, _, _ in imp.get_suffixes():
-            ending = '__init__' + suffix
-            py__file__ = self.py__file__()
-            if py__file__ is not None and py__file__.endswith(ending):
-                # Remove the ending, including the separator.
-                return self.py__file__()[:-len(ending) - 1]
-        return None
-
-    def py__name__(self):
-        for name, module in self.evaluator.modules.items():
-            if module == self and name != '':
-                return name
-
-        return '__main__'
-
-    def py__file__(self):
-        """
-        In contrast to Python's __file__ can be None.
-        """
-        if self._path is None:
-            return None
-
-        return os.path.abspath(self._path)
-
-    def py__package__(self):
-        if self._get_init_directory() is None:
-            return re.sub(r'\.?[^\.]+$', '', self.py__name__())
-        else:
-            return self.py__name__()
-
-    def _py__path__(self):
-        search_path = self.evaluator.sys_path
-        init_path = self.py__file__()
-        if os.path.basename(init_path) == '__init__.py':
-            with open(init_path, 'rb') as f:
-                content = python_bytes_to_unicode(f.read(), errors='replace')
-                # these are strings that need to be used for namespace packages,
-                # the first one is ``pkgutil``, the second ``pkg_resources``.
-                options = ('declare_namespace(__name__)', 'extend_path(__path__')
-                if options[0] in content or options[1] in content:
-                    # It is a namespace, now try to find the rest of the
-                    # modules on sys_path or whatever the search_path is.
-                    paths = set()
-                    for s in search_path:
-                        other = os.path.join(s, self.name.string_name)
-                        if os.path.isdir(other):
-                            paths.add(other)
-                    if paths:
-                        return list(paths)
-                    # TODO I'm not sure if this is how nested namespace
-                    # packages work. The tests are not really good enough to
-                    # show that.
-        # Default to this.
-        return [self._get_init_directory()]
-
-    @property
-    def py__path__(self):
-        """
-        Not seen here, since it's a property. The callback actually uses a
-        variable, so use it like::
-
-            foo.py__path__(sys_path)
-
-        In case of a package, this returns Python's __path__ attribute, which
-        is a list of paths (strings).
-        Raises an AttributeError if the module is not a package.
-        """
-        path = self._get_init_directory()
-
-        if path is None:
-            raise AttributeError('Only packages have __path__ attributes.')
-        else:
-            return self._py__path__
-
-    @evaluator_method_cache()
-    def _sub_modules_dict(self):
-        """
-        Lists modules in the directory of this module (if this module is a
-        package).
-        """
-        path = self._path
-        names = {}
-        if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
-            for module_loader, name, is_pkg in mods:
-                # It's obviously a relative import to the current module.
-                names[name] = imports.SubModuleName(self, name)
-
-        # TODO add something like this in the future, its cleaner than the
-        #   import hacks.
-        # ``os.path`` is a hardcoded exception, because it's a
-        # ``sys.modules`` modification.
-        # if str(self.name) == 'os':
-        #     names.append(Name('path', parent_context=self))
-
-        return names
-
-    def py__class__(self):
-        return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')
-
-    def __repr__(self):
-        return "<%s: %s@%s-%s>" % (
-            self.__class__.__name__, self._string_name,
-            self.tree_node.start_pos[0], self.tree_node.end_pos[0])
-
-
-class ImplicitNSName(AbstractNameDefinition):
-    """
-    Accessing names for implicit namespace packages should infer to nothing.
-    This object will prevent Jedi from raising exceptions
-    """
-    def __init__(self, implicit_ns_context, string_name):
-        self.implicit_ns_context = implicit_ns_context
-        self.string_name = string_name
-
-    def infer(self):
-        return []
-
-    def get_root_context(self):
-        return self.implicit_ns_context
-
-
-class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    Provides support for implicit namespace packages
-    """
-    api_type = 'module'
-    parent_context = None
-
-    def __init__(self, evaluator, fullname):
-        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
-        self.evaluator = evaluator
-        self.fullname = fullname
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield DictFilter(self._sub_modules_dict())
-
-    @property
-    @evaluator_method_cache()
-    def name(self):
-        string_name = self.py__package__().rpartition('.')[-1]
-        return ImplicitNSName(self, string_name)
-
-    def py__file__(self):
-        return None
-
-    def py__package__(self):
-        """Return the fullname
-        """
-        return self.fullname
-
-    @property
-    def py__path__(self):
-        return lambda: [self.paths]
-
-    @evaluator_method_cache()
-    def _sub_modules_dict(self):
-        names = {}
-
-        paths = self.paths
-        file_names = chain.from_iterable(os.listdir(path) for path in paths)
-        mods = [
-            file_name.rpartition('.')[0] if '.' in file_name else file_name
-            for file_name in file_names
-            if file_name != '__pycache__'
-        ]
-
-        for name in mods:
-            names[name] = imports.SubModuleName(self, name)
-        return names

jedi/evaluate/stdlib.py

@@ -12,18 +12,19 @@ compiled module that returns the types for C-builtins.
 import collections
 import re
 
-from jedi.common import unite
+from jedi import debug
+from jedi.evaluate.arguments import ValuesArguments
+from jedi.evaluate import analysis
 from jedi.evaluate import compiled
-from jedi.evaluate import representation as er
-from jedi.evaluate.instance import InstanceFunctionExecution, \
+from jedi.evaluate.context.instance import InstanceFunctionExecution, \
     AbstractInstanceContext, CompiledInstance, BoundMethod, \
     AnonymousInstanceFunctionExecution
-from jedi.evaluate import iterable
-from jedi import debug
-from jedi.evaluate import precedence
-from jedi.evaluate import param
-from jedi.evaluate import analysis
-from jedi.evaluate.context import LazyTreeContext, ContextualizedNode
+from jedi.evaluate.base_context import ContextualizedNode, \
+    NO_CONTEXTS, ContextSet
+from jedi.evaluate.context import ClassContext, ModuleContext
+from jedi.evaluate.context import iterable
+from jedi.evaluate.lazy_context import LazyTreeContext
+from jedi.evaluate.syntax_tree import is_string
 
 # Now this is all part of fake tuples in Jedi. However super doesn't work on
 # __init__ and __new__ doesn't work at all. So adding this to nametuples is
@@ -58,7 +59,7 @@ def execute(evaluator, obj, arguments):
     else:
         if obj.parent_context == evaluator.BUILTINS:
             module_name = 'builtins'
-        elif isinstance(obj.parent_context, er.ModuleContext):
+        elif isinstance(obj.parent_context, ModuleContext):
             module_name = obj.parent_context.name.string_name
         else:
             module_name = ''
@@ -77,7 +78,7 @@ def _follow_param(evaluator, arguments, index):
     try:
         key, lazy_context = list(arguments.unpack())[index]
     except IndexError:
-        return set()
+        return NO_CONTEXTS
     else:
         return lazy_context.infer()
 
@@ -109,7 +110,7 @@ def argument_clinic(string, want_obj=False, want_context=False, want_arguments=F
             try:
                 lst = list(arguments.eval_argument_clinic(clinic_args))
             except ValueError:
-                return set()
+                return NO_CONTEXTS
             else:
                 kwargs = {}
                 if want_context:
@@ -137,15 +138,16 @@ def builtins_next(evaluator, iterators, defaults):
     else:
         name = '__next__'
 
-    types = set()
+    context_set = NO_CONTEXTS
     for iterator in iterators:
         if isinstance(iterator, AbstractInstanceContext):
-            for filter in iterator.get_filters(include_self_names=True):
-                for n in filter.get(name):
-                    for context in n.infer():
-                        types |= context.execute_evaluated()
-    if types:
-        return types
+            context_set = ContextSet.from_sets(
+                n.infer()
+                for filter in iterator.get_filters(include_self_names=True)
+                for n in filter.get(name)
+            ).execute_evaluated()
+    if context_set:
+        return context_set
     return defaults
 
 
@@ -154,21 +156,21 @@ def builtins_getattr(evaluator, objects, names, defaults=None):
     # follow the first param
     for obj in objects:
         for name in names:
-            if precedence.is_string(name):
+            if is_string(name):
                 return obj.py__getattribute__(name.obj)
             else:
                 debug.warning('getattr called without str')
                 continue
-    return set()
+    return NO_CONTEXTS
 
 
 @argument_clinic('object[, bases, dict], /')
 def builtins_type(evaluator, objects, bases, dicts):
     if bases or dicts:
         # It's a type creation... maybe someday...
-        return set()
+        return NO_CONTEXTS
     else:
-        return set([o.py__class__() for o in objects])
+        return objects.py__class__()
 
 
 class SuperInstance(AbstractInstanceContext):
@@ -184,8 +186,8 @@ def builtins_super(evaluator, types, objects, context):
     if isinstance(context, (InstanceFunctionExecution,
                             AnonymousInstanceFunctionExecution)):
         su = context.instance.py__class__().py__bases__()
-        return unite(context.execute_evaluated() for context in su[0].infer())
-    return set()
+        return su[0].infer().execute_evaluated()
+    return NO_CONTEXTS
 
 
 @argument_clinic('sequence, /', want_obj=True, want_arguments=True)
@@ -198,7 +200,7 @@ def builtins_reversed(evaluator, sequences, obj, arguments):
     if isinstance(lazy_context, LazyTreeContext):
         # TODO access private
         cn = ContextualizedNode(lazy_context._context, lazy_context.data)
-    ordered = list(iterable.py__iter__(evaluator, sequences, cn))
+    ordered = list(sequences.iterate(cn))
 
     rev = list(reversed(ordered))
     # Repack iterator values and then run it the normal way. This is
@@ -206,13 +208,13 @@ def builtins_reversed(evaluator, sequences, obj, arguments):
     # would fail in certain cases like `reversed(x).__iter__` if we
     # just returned the result directly.
     seq = iterable.FakeSequence(evaluator, 'list', rev)
-    arguments = param.ValuesArguments([[seq]])
-    return set([CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments)])
+    arguments = ValuesArguments([ContextSet(seq)])
+    return ContextSet(CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments))
 
 
 @argument_clinic('obj, type, /', want_arguments=True)
 def builtins_isinstance(evaluator, objects, types, arguments):
-    bool_results = set([])
+    bool_results = set()
     for o in objects:
         try:
             mro_func = o.py__class__().py__mro__
@@ -220,7 +222,7 @@ def builtins_isinstance(evaluator, objects, types, arguments):
             # This is temporary. Everything should have a class attribute in
             # Python?! Maybe we'll leave it here, because some numpy objects or
             # whatever might not.
-            return set([compiled.create(True), compiled.create(False)])
+            return ContextSet(compiled.create(True), compiled.create(False))
 
         mro = mro_func()
 
@@ -230,9 +232,9 @@ def builtins_isinstance(evaluator, objects, types, arguments):
             elif cls_or_tup.name.string_name == 'tuple' \
                     and cls_or_tup.get_root_context() == evaluator.BUILTINS:
                 # Check for tuples.
-                classes = unite(
+                classes = ContextSet.from_sets(
                     lazy_context.infer()
-                    for lazy_context in cls_or_tup.py__iter__()
+                    for lazy_context in cls_or_tup.iterate()
                 )
                 bool_results.add(any(cls in mro for cls in classes))
             else:
@@ -244,7 +246,7 @@ def builtins_isinstance(evaluator, objects, types, arguments):
                               'not %s.' % cls_or_tup
                     analysis.add(lazy_context._context, 'type-error-isinstance', node, message)
 
-    return set(compiled.create(evaluator, x) for x in bool_results)
+    return ContextSet.from_iterable(compiled.create(evaluator, x) for x in bool_results)
 
 
 def collections_namedtuple(evaluator, obj, arguments):
@@ -259,7 +261,7 @@ def collections_namedtuple(evaluator, obj, arguments):
     """
     # Namedtuples are not supported on Python 2.6
     if not hasattr(collections, '_class_template'):
-        return set()
+        return NO_CONTEXTS
 
     # Process arguments
     # TODO here we only use one of the types, we should use all.
@@ -267,14 +269,14 @@ def collections_namedtuple(evaluator, obj, arguments):
     _fields = list(_follow_param(evaluator, arguments, 1))[0]
     if isinstance(_fields, compiled.CompiledObject):
         fields = _fields.obj.replace(',', ' ').split()
-    elif isinstance(_fields, iterable.AbstractSequence):
+    elif isinstance(_fields, iterable.AbstractIterable):
         fields = [
             v.obj
             for lazy_context in _fields.py__iter__()
             for v in lazy_context.infer() if hasattr(v, 'obj')
         ]
     else:
-        return set()
+        return NO_CONTEXTS
 
     base = collections._class_template
     base += _NAMEDTUPLE_INIT
@@ -292,8 +294,8 @@ def collections_namedtuple(evaluator, obj, arguments):
     # Parse source
     module = evaluator.grammar.parse(source)
     generated_class = next(module.iter_classdefs())
-    parent_context = er.ModuleContext(evaluator, module, '')
-    return set([er.ClassContext(evaluator, generated_class, parent_context)])
+    parent_context = ModuleContext(evaluator, module, '')
+    return ContextSet(ClassContext(evaluator, parent_context, generated_class))
 
 
 @argument_clinic('first, /')
@@ -314,8 +316,8 @@ _implemented = {
         'deepcopy': _return_first_param,
     },
     'json': {
-        'load': lambda *args: set(),
-        'loads': lambda *args: set(),
+        'load': lambda *args: NO_CONTEXTS,
+        'loads': lambda *args: NO_CONTEXTS,
     },
     'collections': {
         'namedtuple': collections_namedtuple,

jedi/evaluate/syntax_tree.py

@@ -0,0 +1,588 @@
+"""
+Functions evaluating the syntax tree.
+"""
+import copy
+import operator as op
+
+from parso.python import tree
+
+from jedi import debug
+from jedi import parser_utils
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, ContextualizedNode, \
+    ContextualizedName, iterator_to_context_set, iterate_contexts
+from jedi.evaluate import compiled
+from jedi.evaluate import pep0484
+from jedi.evaluate import recursion
+from jedi.evaluate import helpers
+from jedi.evaluate import analysis
+from jedi.evaluate import imports
+from jedi.evaluate import arguments
+from jedi.evaluate.context import ClassContext, FunctionContext
+from jedi.evaluate.context import iterable
+from jedi.evaluate.context import TreeInstance, CompiledInstance
+from jedi.evaluate.finder import NameFinder
+from jedi.evaluate.helpers import is_string, is_literal, is_number, is_compiled
+
+
+def _limit_context_infers(func):
+    """
+    This is for now the way how we limit type inference going wild. There are
+    other ways to ensure recursion limits as well. This is mostly necessary
+    because of instance (self) access that can be quite tricky to limit.
+
+    I'm still not sure this is the way to go, but it looks okay for now and we
+    can still go anther way in the future. Tests are there. ~ dave
+    """
+    def wrapper(context, *args, **kwargs):
+        n = context.tree_node
+        evaluator = context.evaluator
+        try:
+            evaluator.inferred_element_counts[n] += 1
+            if evaluator.inferred_element_counts[n] > 300:
+                debug.warning('In context %s there were too many inferences.', n)
+                return NO_CONTEXTS
+        except KeyError:
+            evaluator.inferred_element_counts[n] = 1
+        return func(context, *args, **kwargs)
+
+    return wrapper
+
+
+@debug.increase_indent
+@_limit_context_infers
+def eval_node(context, element):
+    debug.dbg('eval_element %s@%s', element, element.start_pos)
+    evaluator = context.evaluator
+    typ = element.type
+    if typ in ('name', 'number', 'string', 'atom'):
+        return eval_atom(context, element)
+    elif typ == 'keyword':
+        # For False/True/None
+        if element.value in ('False', 'True', 'None'):
+            return ContextSet(compiled.builtin_from_name(evaluator, element.value))
+        # else: print e.g. could be evaluated like this in Python 2.7
+        return NO_CONTEXTS
+    elif typ == 'lambdef':
+        return ContextSet(FunctionContext(evaluator, context, element))
+    elif typ == 'expr_stmt':
+        return eval_expr_stmt(context, element)
+    elif typ in ('power', 'atom_expr'):
+        first_child = element.children[0]
+        if not (first_child.type == 'keyword' and first_child.value == 'await'):
+            context_set = eval_atom(context, first_child)
+            for trailer in element.children[1:]:
+                if trailer == '**':  # has a power operation.
+                    right = evaluator.eval_element(context, element.children[2])
+                    context_set = _eval_comparison(
+                        evaluator,
+                        context,
+                        context_set,
+                        trailer,
+                        right
+                    )
+                    break
+                context_set = eval_trailer(context, context_set, trailer)
+            return context_set
+        return NO_CONTEXTS
+    elif typ in ('testlist_star_expr', 'testlist',):
+        # The implicit tuple in statements.
+        return ContextSet(iterable.SequenceLiteralContext(evaluator, context, element))
+    elif typ in ('not_test', 'factor'):
+        context_set = context.eval_node(element.children[-1])
+        for operator in element.children[:-1]:
+            context_set = eval_factor(context_set, operator)
+        return context_set
+    elif typ == 'test':
+        # `x if foo else y` case.
+        return (context.eval_node(element.children[0]) |
+                context.eval_node(element.children[-1]))
+    elif typ == 'operator':
+        # Must be an ellipsis, other operators are not evaluated.
+        # In Python 2 ellipsis is coded as three single dot tokens, not
+        # as one token 3 dot token.
+        assert element.value in ('.', '...')
+        return ContextSet(compiled.create(evaluator, Ellipsis))
+    elif typ == 'dotted_name':
+        context_set = eval_atom(context, element.children[0])
+        for next_name in element.children[2::2]:
+            # TODO add search_global=True?
+            context_set = context_set.py__getattribute__(next_name, name_context=context)
+        return context_set
+    elif typ == 'eval_input':
+        return eval_node(context, element.children[0])
+    elif typ == 'annassign':
+        return pep0484._evaluate_for_annotation(context, element.children[1])
+    else:
+        return eval_or_test(context, element)
+
+
+def eval_trailer(context, base_contexts, trailer):
+    trailer_op, node = trailer.children[:2]
+    if node == ')':  # `arglist` is optional.
+        node = ()
+
+    if trailer_op == '[':
+        trailer_op, node, _ = trailer.children
+
+        # TODO It's kind of stupid to cast this from a context set to a set.
+        foo = set(base_contexts)
+        # special case: PEP0484 typing module, see
+        # https://github.com/davidhalter/jedi/issues/663
+        result = ContextSet()
+        for typ in list(foo):
+            if isinstance(typ, (ClassContext, TreeInstance)):
+                typing_module_types = pep0484.py__getitem__(context, typ, node)
+                if typing_module_types is not None:
+                    foo.remove(typ)
+                    result |= typing_module_types
+
+        return result | base_contexts.get_item(
+            eval_subscript_list(context.evaluator, context, node),
+            ContextualizedNode(context, trailer)
+        )
+    else:
+        debug.dbg('eval_trailer: %s in %s', trailer, base_contexts)
+        if trailer_op == '.':
+            return base_contexts.py__getattribute__(
+                name_context=context,
+                name_or_str=node
+            )
+        else:
+            assert trailer_op == '('
+            args = arguments.TreeArguments(context.evaluator, context, node, trailer)
+            return base_contexts.execute(args)
+
+
+def eval_atom(context, atom):
+    """
+    Basically to process ``atom`` nodes. The parser sometimes doesn't
+    generate the node (because it has just one child). In that case an atom
+    might be a name or a literal as well.
+    """
+    if atom.type == 'name':
+        # This is the first global lookup.
+        stmt = tree.search_ancestor(
+            atom, 'expr_stmt', 'lambdef'
+        ) or atom
+        if stmt.type == 'lambdef':
+            stmt = atom
+        return context.py__getattribute__(
+            name_or_str=atom,
+            position=stmt.start_pos,
+            search_global=True
+        )
+
+    elif isinstance(atom, tree.Literal):
+        string = parser_utils.safe_literal_eval(atom.value)
+        return ContextSet(compiled.create(context.evaluator, string))
+    else:
+        c = atom.children
+        if c[0].type == 'string':
+            # Will be one string.
+            context_set = eval_atom(context, c[0])
+            for string in c[1:]:
+                right = eval_atom(context, string)
+                context_set = _eval_comparison(context.evaluator, context, context_set, '+', right)
+            return context_set
+        # Parentheses without commas are not tuples.
+        elif c[0] == '(' and not len(c) == 2 \
+                and not(c[1].type == 'testlist_comp' and
+                        len(c[1].children) > 1):
+            return context.eval_node(c[1])
+
+        try:
+            comp_for = c[1].children[1]
+        except (IndexError, AttributeError):
+            pass
+        else:
+            if comp_for == ':':
+                # Dict comprehensions have a colon at the 3rd index.
+                try:
+                    comp_for = c[1].children[3]
+                except IndexError:
+                    pass
+
+            if comp_for.type == 'comp_for':
+                return ContextSet(iterable.Comprehension.from_atom(context.evaluator, context, atom))
+
+        # It's a dict/list/tuple literal.
+        array_node = c[1]
+        try:
+            array_node_c = array_node.children
+        except AttributeError:
+            array_node_c = []
+        if c[0] == '{' and (array_node == '}' or ':' in array_node_c):
+            context = iterable.DictLiteralContext(context.evaluator, context, atom)
+        else:
+            context = iterable.SequenceLiteralContext(context.evaluator, context, atom)
+        return ContextSet(context)
+
+
+@_limit_context_infers
+def eval_expr_stmt(context, stmt, seek_name=None):
+    with recursion.execution_allowed(context.evaluator, stmt) as allowed:
+        if allowed or context.get_root_context() == context.evaluator.BUILTINS:
+            return _eval_expr_stmt(context, stmt, seek_name)
+    return NO_CONTEXTS
+
+
+@debug.increase_indent
+def _eval_expr_stmt(context, stmt, seek_name=None):
+    """
+    The starting point of the completion. A statement always owns a call
+    list, which are the calls, that a statement does. In case multiple
+    names are defined in the statement, `seek_name` returns the result for
+    this name.
+
+    :param stmt: A `tree.ExprStmt`.
+    """
+    debug.dbg('eval_expr_stmt %s (%s)', stmt, seek_name)
+    rhs = stmt.get_rhs()
+    context_set = context.eval_node(rhs)
+
+    if seek_name:
+        c_node = ContextualizedName(context, seek_name)
+        context_set = check_tuple_assignments(context.evaluator, c_node, context_set)
+
+    first_operator = next(stmt.yield_operators(), None)
+    if first_operator not in ('=', None) and first_operator.type == 'operator':
+        # `=` is always the last character in aug assignments -> -1
+        operator = copy.copy(first_operator)
+        operator.value = operator.value[:-1]
+        name = stmt.get_defined_names()[0].value
+        left = context.py__getattribute__(
+            name, position=stmt.start_pos, search_global=True)
+
+        for_stmt = tree.search_ancestor(stmt, 'for_stmt')
+        if for_stmt is not None and for_stmt.type == 'for_stmt' and context_set \
+                and parser_utils.for_stmt_defines_one_name(for_stmt):
+            # Iterate through result and add the values, that's possible
+            # only in for loops without clutter, because they are
+            # predictable. Also only do it, if the variable is not a tuple.
+            node = for_stmt.get_testlist()
+            cn = ContextualizedNode(context, node)
+            ordered = list(cn.infer().iterate(cn))
+
+            for lazy_context in ordered:
+                dct = {for_stmt.children[1].value: lazy_context.infer()}
+                with helpers.predefine_names(context, for_stmt, dct):
+                    t = context.eval_node(rhs)
+                    left = _eval_comparison(context.evaluator, context, left, operator, t)
+            context_set = left
+        else:
+            context_set = _eval_comparison(context.evaluator, context, left, operator, context_set)
+    debug.dbg('eval_expr_stmt result %s', context_set)
+    return context_set
+
+
+def eval_or_test(context, or_test):
+    iterator = iter(or_test.children)
+    types = context.eval_node(next(iterator))
+    for operator in iterator:
+        right = next(iterator)
+        if operator.type == 'comp_op':  # not in / is not
+            operator = ' '.join(c.value for c in operator.children)
+
+        # handle lazy evaluation of and/or here.
+        if operator in ('and', 'or'):
+            left_bools = set(left.py__bool__() for left in types)
+            if left_bools == set([True]):
+                if operator == 'and':
+                    types = context.eval_node(right)
+            elif left_bools == set([False]):
+                if operator != 'and':
+                    types = context.eval_node(right)
+            # Otherwise continue, because of uncertainty.
+        else:
+            types = _eval_comparison(context.evaluator, context, types, operator,
+                                         context.eval_node(right))
+    debug.dbg('eval_or_test types %s', types)
+    return types
+
+
+@iterator_to_context_set
+def eval_factor(context_set, operator):
+    """
+    Calculates `+`, `-`, `~` and `not` prefixes.
+    """
+    for context in context_set:
+        if operator == '-':
+            if is_number(context):
+                yield compiled.create(context.evaluator, -context.obj)
+        elif operator == 'not':
+            value = context.py__bool__()
+            if value is None:  # Uncertainty.
+                return
+            yield compiled.create(context.evaluator, not value)
+        else:
+            yield context
+
+
+# Maps Python syntax to the operator module.
+COMPARISON_OPERATORS = {
+    '==': op.eq,
+    '!=': op.ne,
+    'is': op.is_,
+    'is not': op.is_not,
+    '<': op.lt,
+    '<=': op.le,
+    '>': op.gt,
+    '>=': op.ge,
+}
+
+
+def _literals_to_types(evaluator, result):
+    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
+    # int(), float(), etc).
+    new_result = NO_CONTEXTS
+    for typ in result:
+        if is_literal(typ):
+            # Literals are only valid as long as the operations are
+            # correct. Otherwise add a value-free instance.
+            cls = compiled.builtin_from_name(evaluator, typ.name.string_name)
+            new_result |= cls.execute_evaluated()
+        else:
+            new_result |= ContextSet(typ)
+    return new_result
+
+
+def _eval_comparison(evaluator, context, left_contexts, operator, right_contexts):
+    if not left_contexts or not right_contexts:
+        # illegal slices e.g. cause left/right_result to be None
+        result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS)
+        return _literals_to_types(evaluator, result)
+    else:
+        # I don't think there's a reasonable chance that a string
+        # operation is still correct, once we pass something like six
+        # objects.
+        if len(left_contexts) * len(right_contexts) > 6:
+            return _literals_to_types(evaluator, left_contexts | right_contexts)
+        else:
+            return ContextSet.from_sets(
+                _eval_comparison_part(evaluator, context, left, operator, right)
+                for left in left_contexts
+                for right in right_contexts
+            )
+
+
+def _is_tuple(context):
+    return isinstance(context, iterable.AbstractIterable) and context.array_type == 'tuple'
+
+
+def _is_list(context):
+    return isinstance(context, iterable.AbstractIterable) and context.array_type == 'list'
+
+
+def _eval_comparison_part(evaluator, context, left, operator, right):
+    l_is_num = is_number(left)
+    r_is_num = is_number(right)
+    if operator == '*':
+        # for iterables, ignore * operations
+        if isinstance(left, iterable.AbstractIterable) or is_string(left):
+            return ContextSet(left)
+        elif isinstance(right, iterable.AbstractIterable) or is_string(right):
+            return ContextSet(right)
+    elif operator == '+':
+        if l_is_num and r_is_num or is_string(left) and is_string(right):
+            return ContextSet(compiled.create(evaluator, left.obj + right.obj))
+        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
+            return ContextSet(iterable.MergedArray(evaluator, (left, right)))
+    elif operator == '-':
+        if l_is_num and r_is_num:
+            return ContextSet(compiled.create(evaluator, left.obj - right.obj))
+    elif operator == '%':
+        # With strings and numbers the left type typically remains. Except for
+        # `int() % float()`.
+        return ContextSet(left)
+    elif operator in COMPARISON_OPERATORS:
+        operation = COMPARISON_OPERATORS[operator]
+        if is_compiled(left) and is_compiled(right):
+            # Possible, because the return is not an option. Just compare.
+            left = left.obj
+            right = right.obj
+
+        try:
+            result = operation(left, right)
+        except TypeError:
+            # Could be True or False.
+            return ContextSet(compiled.create(evaluator, True), compiled.create(evaluator, False))
+        else:
+            return ContextSet(compiled.create(evaluator, result))
+    elif operator == 'in':
+        return NO_CONTEXTS
+
+    def check(obj):
+        """Checks if a Jedi object is either a float or an int."""
+        return isinstance(obj, CompiledInstance) and \
+            obj.name.string_name in ('int', 'float')
+
+    # Static analysis, one is a number, the other one is not.
+    if operator in ('+', '-') and l_is_num != r_is_num \
+            and not (check(left) or check(right)):
+        message = "TypeError: unsupported operand type(s) for +: %s and %s"
+        analysis.add(context, 'type-error-operation', operator,
+                     message % (left, right))
+
+    return ContextSet(left, right)
+
+
+def _remove_statements(evaluator, context, stmt, name):
+    """
+    This is the part where statements are being stripped.
+
+    Due to lazy evaluation, statements like a = func; b = a; b() have to be
+    evaluated.
+    """
+    pep0484_contexts = \
+        pep0484.find_type_from_comment_hint_assign(context, stmt, name)
+    if pep0484_contexts:
+        return pep0484_contexts
+
+    return eval_expr_stmt(context, stmt, seek_name=name)
+
+
+def tree_name_to_contexts(evaluator, context, tree_name):
+    types = []
+    node = tree_name.get_definition(import_name_always=True)
+    if node is None:
+        node = tree_name.parent
+        if node.type == 'global_stmt':
+            context = evaluator.create_context(context, tree_name)
+            finder = NameFinder(evaluator, context, context, tree_name.value)
+            filters = finder.get_filters(search_global=True)
+            # For global_stmt lookups, we only need the first possible scope,
+            # which means the function itself.
+            filters = [next(filters)]
+            return finder.find(filters, attribute_lookup=False)
+        elif node.type not in ('import_from', 'import_name'):
+            raise ValueError("Should not happen.")
+
+    typ = node.type
+    if typ == 'for_stmt':
+        types = pep0484.find_type_from_comment_hint_for(context, node, tree_name)
+        if types:
+            return types
+    if typ == 'with_stmt':
+        types = pep0484.find_type_from_comment_hint_with(context, node, tree_name)
+        if types:
+            return types
+
+    if typ in ('for_stmt', 'comp_for'):
+        try:
+            types = context.predefined_names[node][tree_name.value]
+        except KeyError:
+            cn = ContextualizedNode(context, node.children[3])
+            for_types = iterate_contexts(cn.infer(), cn)
+            c_node = ContextualizedName(context, tree_name)
+            types = check_tuple_assignments(evaluator, c_node, for_types)
+    elif typ == 'expr_stmt':
+        types = _remove_statements(evaluator, context, node, tree_name)
+    elif typ == 'with_stmt':
+        context_managers = context.eval_node(node.get_test_node_from_name(tree_name))
+        enter_methods = context_managers.py__getattribute__('__enter__')
+        return enter_methods.execute_evaluated()
+    elif typ in ('import_from', 'import_name'):
+        types = imports.infer_import(context, tree_name)
+    elif typ in ('funcdef', 'classdef'):
+        types = _apply_decorators(context, node)
+    elif typ == 'try_stmt':
+        # TODO an exception can also be a tuple. Check for those.
+        # TODO check for types that are not classes and add it to
+        # the static analysis report.
+        exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling())
+        types = exceptions.execute_evaluated()
+    else:
+        raise ValueError("Should not happen.")
+    return types
+
+
+def _apply_decorators(context, node):
+    """
+    Returns the function, that should to be executed in the end.
+    This is also the places where the decorators are processed.
+    """
+    if node.type == 'classdef':
+        decoratee_context = ClassContext(
+            context.evaluator,
+            parent_context=context,
+            classdef=node
+        )
+    else:
+        decoratee_context = FunctionContext(
+            context.evaluator,
+            parent_context=context,
+            funcdef=node
+        )
+    initial = values = ContextSet(decoratee_context)
+    for dec in reversed(node.get_decorators()):
+        debug.dbg('decorator: %s %s', dec, values)
+        dec_values = context.eval_node(dec.children[1])
+        trailer_nodes = dec.children[2:-1]
+        if trailer_nodes:
+            # Create a trailer and evaluate it.
+            trailer = tree.PythonNode('trailer', trailer_nodes)
+            trailer.parent = dec
+            dec_values = eval_trailer(context, dec_values, trailer)
+
+        if not len(dec_values):
+            debug.warning('decorator not found: %s on %s', dec, node)
+            return initial
+
+        values = dec_values.execute(arguments.ValuesArguments([values]))
+        if not len(values):
+            debug.warning('not possible to resolve wrappers found %s', node)
+            return initial
+
+        debug.dbg('decorator end %s', values)
+    return values
+
+
+def check_tuple_assignments(evaluator, contextualized_name, context_set):
+    """
+    Checks if tuples are assigned.
+    """
+    lazy_context = None
+    for index, node in contextualized_name.assignment_indexes():
+        cn = ContextualizedNode(contextualized_name.context, node)
+        iterated = context_set.iterate(cn)
+        for _ in range(index + 1):
+            try:
+                lazy_context = next(iterated)
+            except StopIteration:
+                # We could do this with the default param in next. But this
+                # would allow this loop to run for a very long time if the
+                # index number is high. Therefore break if the loop is
+                # finished.
+                return ContextSet()
+        context_set = lazy_context.infer()
+    return context_set
+
+
+def eval_subscript_list(evaluator, context, index):
+    """
+    Handles slices in subscript nodes.
+    """
+    if index == ':':
+        # Like array[:]
+        return ContextSet(iterable.Slice(context, None, None, None))
+
+    elif index.type == 'subscript' and not index.children[0] == '.':
+        # subscript basically implies a slice operation, except for Python 2's
+        # Ellipsis.
+        # e.g. array[:3]
+        result = []
+        for el in index.children:
+            if el == ':':
+                if not result:
+                    result.append(None)
+            elif el.type == 'sliceop':
+                if len(el.children) == 2:
+                    result.append(el.children[1])
+            else:
+                result.append(el)
+        result += [None] * (3 - len(result))
+
+        return ContextSet(iterable.Slice(context, *result))
+
+    # No slices
+    return context.eval_node(index)

jedi/evaluate/sys_path.py

@@ -4,19 +4,19 @@ import sys
 import imp
 from jedi.evaluate.site import addsitedir
 
-from jedi._compatibility import exec_function, unicode
-from jedi.evaluate.cache import evaluator_function_cache
-from jedi.evaluate.compiled import CompiledObject
-from jedi.evaluate.context import ContextualizedNode
+from jedi._compatibility import unicode
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.base_context import ContextualizedNode
+from jedi.evaluate.helpers import is_string
 from jedi import settings
 from jedi import debug
-from jedi import common
+from jedi.evaluate.utils import ignored
 
 
 def get_venv_path(venv):
     """Get sys.path for specified virtual environment."""
     sys_path = _get_venv_path_dirs(venv)
-    with common.ignored(ValueError):
+    with ignored(ValueError):
         sys_path.remove('')
     sys_path = _get_sys_path_with_egglinks(sys_path)
     # As of now, get_venv_path_dirs does not scan built-in pythonpath and
@@ -69,21 +69,18 @@ def _get_venv_sitepackages(venv):
     return p
 
 
-def _execute_code(module_path, code):
-    c = "import os; from os.path import *; result=%s"
-    variables = {'__file__': module_path}
-    try:
-        exec_function(c % code, variables)
-    except Exception:
-        debug.warning('sys.path manipulation detected, but failed to evaluate.')
-    else:
-        try:
-            res = variables['result']
-            if isinstance(res, str):
-                return [os.path.abspath(res)]
-        except KeyError:
-            pass
-    return []
+def _abs_path(module_context, path):
+    module_path = module_context.py__file__()
+    if os.path.isabs(path):
+        return path
+
+    if module_path is None:
+        # In this case we have no idea where we actually are in the file
+        # system.
+        return None
+
+    base_dir = os.path.dirname(module_path)
+    return os.path.abspath(os.path.join(base_dir, path))
 
 
 def _paths_from_assignment(module_context, expr_stmt):
@@ -108,8 +105,8 @@ def _paths_from_assignment(module_context, expr_stmt):
             assert trailer.children[0] == '.' and trailer.children[1].value == 'path'
             # TODO Essentially we're not checking details on sys.path
             # manipulation. Both assigment of the sys.path and changing/adding
-            # parts of the sys.path are the same: They get added to the current
-            # sys.path.
+            # parts of the sys.path are the same: They get added to the end of
+            # the current sys.path.
             """
             execution = c[2]
             assert execution.children[0] == '['
@@ -120,34 +117,40 @@ def _paths_from_assignment(module_context, expr_stmt):
         except AssertionError:
             continue
 
-        from jedi.evaluate.iterable import py__iter__
-        from jedi.evaluate.precedence import is_string
         cn = ContextualizedNode(module_context.create_context(expr_stmt), expr_stmt)
-        for lazy_context in py__iter__(module_context.evaluator, cn.infer(), cn):
+        for lazy_context in cn.infer().iterate(cn):
             for context in lazy_context.infer():
                 if is_string(context):
-                    yield context.obj
+                    abs_path = _abs_path(module_context, context.obj)
+                    if abs_path is not None:
+                        yield abs_path
 
 
-def _paths_from_list_modifications(module_path, trailer1, trailer2):
+def _paths_from_list_modifications(module_context, trailer1, trailer2):
     """ extract the path from either "sys.path.append" or "sys.path.insert" """
     # Guarantee that both are trailers, the first one a name and the second one
     # a function execution with at least one param.
     if not (trailer1.type == 'trailer' and trailer1.children[0] == '.'
             and trailer2.type == 'trailer' and trailer2.children[0] == '('
             and len(trailer2.children) == 3):
-        return []
+        return
 
     name = trailer1.children[1].value
     if name not in ['insert', 'append']:
-        return []
+        return
     arg = trailer2.children[1]
     if name == 'insert' and len(arg.children) in (3, 4):  # Possible trailing comma.
         arg = arg.children[2]
-    return _execute_code(module_path, arg.get_code())
+
+    for context in module_context.create_context(arg).eval_node(arg):
+        if is_string(context):
+            abs_path = _abs_path(module_context, context.obj)
+            if abs_path is not None:
+                yield abs_path
 
 
-def _check_module(module_context):
+@evaluator_method_cache(default=[])
+def check_sys_path_modifications(module_context):
     """
     Detect sys.path modifications within module.
     """
@@ -162,10 +165,10 @@ def _check_module(module_context):
                     if n.type == 'name' and n.value == 'path':
                         yield name, power
 
-    sys_path = list(module_context.evaluator.sys_path)  # copy
-    if isinstance(module_context, CompiledObject):
-        return sys_path
+    if module_context.tree_node is None:
+        return []
 
+    added = []
     try:
         possible_names = module_context.tree_node.get_used_names()['path']
     except KeyError:
@@ -174,39 +177,29 @@ def _check_module(module_context):
         for name, power in get_sys_path_powers(possible_names):
             expr_stmt = power.parent
             if len(power.children) >= 4:
-                sys_path.extend(
+                added.extend(
                     _paths_from_list_modifications(
-                        module_context.py__file__(), *power.children[2:4]
+                        module_context, *power.children[2:4]
                     )
                 )
             elif expr_stmt is not None and expr_stmt.type == 'expr_stmt':
-                sys_path.extend(_paths_from_assignment(module_context, expr_stmt))
-    return sys_path
+                added.extend(_paths_from_assignment(module_context, expr_stmt))
+    return added
 
 
-@evaluator_function_cache(default=[])
 def sys_path_with_modifications(evaluator, module_context):
-    path = module_context.py__file__()
-    if path is None:
-        # Support for modules without a path is bad, therefore return the
-        # normal path.
-        return list(evaluator.sys_path)
+    return evaluator.project.sys_path + check_sys_path_modifications(module_context)
 
-    curdir = os.path.abspath(os.curdir)
-    #TODO why do we need a chdir?
-    with common.ignored(OSError):
-        os.chdir(os.path.dirname(path))
 
+def detect_additional_paths(evaluator, script_path):
+    django_paths = _detect_django_path(script_path)
     buildout_script_paths = set()
 
-    result = _check_module(module_context)
-    result += _detect_django_path(path)
-    for buildout_script_path in _get_buildout_script_paths(path):
+    for buildout_script_path in _get_buildout_script_paths(script_path):
         for path in _get_paths_from_buildout_script(evaluator, buildout_script_path):
             buildout_script_paths.add(path)
-    # cleanup, back to old directory
-    os.chdir(curdir)
-    return list(result) + list(buildout_script_paths)
+
+    return django_paths + list(buildout_script_paths)
 
 
 def _get_paths_from_buildout_script(evaluator, buildout_script_path):
@@ -220,8 +213,9 @@ def _get_paths_from_buildout_script(evaluator, buildout_script_path):
         debug.warning('Error trying to read buildout_script: %s', buildout_script_path)
         return
 
-    from jedi.evaluate.representation import ModuleContext
-    for path in _check_module(ModuleContext(evaluator, module_node, buildout_script_path)):
+    from jedi.evaluate.context import ModuleContext
+    module = ModuleContext(evaluator, module_node, buildout_script_path)
+    for path in check_sys_path_modifications(module):
         yield path
 
 
@@ -246,7 +240,7 @@ def _detect_django_path(module_path):
     result = []
 
     for parent in traverse_parents(module_path):
-        with common.ignored(IOError):
+        with ignored(IOError):
             with open(parent + os.path.sep + 'manage.py'):
                 debug.dbg('Found django path: %s', module_path)
                 result.append(parent)

jedi/evaluate/usages.py

@@ -0,0 +1,62 @@
+from jedi.evaluate import imports
+from jedi.evaluate.filters import TreeNameDefinition
+from jedi.evaluate.context import ModuleContext
+
+
+def _resolve_names(definition_names, avoid_names=()):
+    for name in definition_names:
+        if name in avoid_names:
+            # Avoiding recursions here, because goto on a module name lands
+            # on the same module.
+            continue
+
+        if not isinstance(name, imports.SubModuleName):
+            # SubModuleNames are not actually existing names but created
+            # names when importing something like `import foo.bar.baz`.
+            yield name
+
+        if name.api_type == 'module':
+            for name in _resolve_names(name.goto(), definition_names):
+                yield name
+
+
+def _dictionarize(names):
+    return dict(
+        (n if n.tree_name is None else n.tree_name, n)
+        for n in names
+    )
+
+
+def _find_names(module_context, tree_name):
+    context = module_context.create_context(tree_name)
+    name = TreeNameDefinition(context, tree_name)
+    found_names = set(name.goto())
+    found_names.add(name)
+    return _dictionarize(_resolve_names(found_names))
+
+
+def usages(module_context, tree_name):
+    search_name = tree_name.value
+    found_names = _find_names(module_context, tree_name)
+    modules = set(d.get_root_context() for d in found_names.values())
+    modules = set(m for m in modules if isinstance(m, ModuleContext))
+
+    non_matching_usage_maps = {}
+    for m in imports.get_modules_containing_name(module_context.evaluator, modules, search_name):
+        for name_leaf in m.tree_node.get_used_names().get(search_name, []):
+            new = _find_names(m, name_leaf)
+            if any(tree_name in found_names for tree_name in new):
+                found_names.update(new)
+                for tree_name in new:
+                    for dct in non_matching_usage_maps.get(tree_name, []):
+                        # A usage that was previously searched for matches with
+                        # a now found name. Merge.
+                        found_names.update(dct)
+                    try:
+                        del non_matching_usage_maps[tree_name]
+                    except KeyError:
+                        pass
+            else:
+                for name in new:
+                    non_matching_usage_maps.setdefault(name, []).append(new)
+    return found_names.values()

jedi/evaluate/utils.py

@@ -4,7 +4,17 @@ import contextlib
 import functools
 
 from jedi._compatibility import reraise
-from jedi import settings
+
+
+def to_list(func):
+    def wrapper(*args, **kwargs):
+        return list(func(*args, **kwargs))
+    return wrapper
+
+
+def unite(iterable):
+    """Turns a two dimensional array into a one dimensional."""
+    return set(typ for types in iterable for typ in types)
 
 
 class UncaughtAttributeError(Exception):
@@ -78,16 +88,6 @@ class PushBackIterator(object):
         return self.current
 
 
-def indent_block(text, indention='    '):
-    """This function indents a text block with a default of four spaces."""
-    temp = ''
-    while text and text[-1] == '\n':
-        temp += text[-1]
-        text = text[:-1]
-    lines = text.split('\n')
-    return '\n'.join(map(lambda s: indention + s, lines)) + temp
-
-
 @contextlib.contextmanager
 def ignored(*exceptions):
     """
@@ -100,12 +100,11 @@ def ignored(*exceptions):
         pass
 
 
-def unite(iterable):
-    """Turns a two dimensional array into a one dimensional."""
-    return set(typ for types in iterable for typ in types)
-
-
-def to_list(func):
-    def wrapper(*args, **kwargs):
-        return list(func(*args, **kwargs))
-    return wrapper
+def indent_block(text, indention='    '):
+    """This function indents a text block with a default of four spaces."""
+    temp = ''
+    while text and text[-1] == '\n':
+        temp += text[-1]
+        text = text[:-1]
+    lines = text.split('\n')
+    return '\n'.join(map(lambda s: indention + s, lines)) + temp

jedi/refactoring.py

@@ -14,7 +14,6 @@ following functions (sometimes bug-prone):
 """
 import difflib
 
-from jedi import common
 from parso import python_bytes_to_unicode, split_lines
 from jedi.evaluate import helpers
 
@@ -165,38 +164,37 @@ def inline(script):
     dct = {}
 
     definitions = script.goto_assignments()
-    with common.ignored(AssertionError):
-        assert len(definitions) == 1
-        stmt = definitions[0]._definition
-        usages = script.usages()
-        inlines = [r for r in usages
-                   if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
-        inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
-                         reverse=True)
-        expression_list = stmt.expression_list()
-        # don't allow multiline refactorings for now.
-        assert stmt.start_pos[0] == stmt.end_pos[0]
-        index = stmt.start_pos[0] - 1
+    assert len(definitions) == 1
+    stmt = definitions[0]._definition
+    usages = script.usages()
+    inlines = [r for r in usages
+               if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
+    inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
+                     reverse=True)
+    expression_list = stmt.expression_list()
+    # don't allow multiline refactorings for now.
+    assert stmt.start_pos[0] == stmt.end_pos[0]
+    index = stmt.start_pos[0] - 1
 
-        line = new_lines[index]
-        replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
-        replace_str = replace_str.strip()
-        # tuples need parentheses
-        if expression_list and isinstance(expression_list[0], pr.Array):
-            arr = expression_list[0]
-            if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
-                replace_str = '(%s)' % replace_str
+    line = new_lines[index]
+    replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
+    replace_str = replace_str.strip()
+    # tuples need parentheses
+    if expression_list and isinstance(expression_list[0], pr.Array):
+        arr = expression_list[0]
+        if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
+            replace_str = '(%s)' % replace_str
 
-        # if it's the only assignment, remove the statement
-        if len(stmt.get_defined_names()) == 1:
-            line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
+    # if it's the only assignment, remove the statement
+    if len(stmt.get_defined_names()) == 1:
+        line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
 
-        dct = _rename(inlines, replace_str)
-        # remove the empty line
-        new_lines = dct[script.path][2]
-        if line.strip():
-            new_lines[index] = line
-        else:
-            new_lines.pop(index)
+    dct = _rename(inlines, replace_str)
+    # remove the empty line
+    new_lines = dct[script.path][2]
+    if line.strip():
+        new_lines[index] = line
+    else:
+        new_lines.pop(index)
 
     return Refactoring(dct)

requirements.txt

@@ -1 +1 @@
-parso==0.1.0
+parso==0.1.1

setup.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 import ast
-
+import sys
 
 __AUTHOR__ = 'David Halter'
 __AUTHOR_EMAIL__ = 'davidhalter88@gmail.com'
@@ -11,10 +11,12 @@ __AUTHOR_EMAIL__ = 'davidhalter88@gmail.com'
 # Get the version from within jedi. It's defined in exactly one place now.
 with open('jedi/__init__.py') as f:
     tree = ast.parse(f.read())
-version = tree.body[1].value.s
+if sys.version_info > (3, 7):
+    version = tree.body[0].value.s
+else:
+    version = tree.body[1].value.s
 
 readme = open('README.rst').read() + '\n\n' + open('CHANGELOG.rst').read()
-packages = ['jedi', 'jedi.evaluate', 'jedi.evaluate.compiled', 'jedi.api']
 with open('requirements.txt') as f:
     install_requires = f.read().splitlines()
 
@@ -30,8 +32,9 @@ setup(name='jedi',
       license='MIT',
       keywords='python completion refactoring vim',
       long_description=readme,
-      packages=packages,
+      packages=find_packages(exclude=['test']),
       install_requires=install_requires,
+      extras_require={'dev': ['docopt']},
       package_data={'jedi': ['evaluate/compiled/fake/*.pym']},
       platforms=['any'],
       classifiers=[
@@ -47,6 +50,8 @@ setup(name='jedi',
           'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',

test/completion/async_.py

@@ -23,3 +23,14 @@ async def x2():
     async with open('asdf') as f:
         #? ['readlines']
         f.readlines
+
+class A():
+    @staticmethod
+    async def b(c=1, d=2):
+        return 1
+
+#! 9 ['def b']
+await A.b()
+
+#! 11 ['param d=2']
+await A.b(d=3)

test/completion/sys_path.py

@@ -7,8 +7,6 @@ sys.path.insert(0, '../../jedi')
 sys.path.append(dirname(os.path.abspath('thirdparty' + os.path.sep + 'asdf')))
 
 # modifications, that should fail:
-# because of sys module
-sys.path.append(sys.path[1] + '/thirdparty')
 # syntax err
 sys.path.append('a' +* '/thirdparty')
 
@@ -18,8 +16,9 @@ import evaluate
 #? ['evaluator_function_cache']
 evaluate.Evaluator_fu
 
-#? ['jedi_']
+# Those don't work because dirname and abspath are not properly understood.
+##? ['jedi_']
 import jedi_
 
-#? ['el']
+##? ['el']
 jedi_.el

test/completion/usages.py

@@ -188,7 +188,7 @@ class TestClass(Super):
         self.base_class
         #< (-20,13), (0,13)
         self.base_var
-        #< 
+        #< (0, 18),
         TestClass.base_var
 
 
@@ -242,7 +242,7 @@ def f(**kwargs):
 # No result
 # -----------------
 if isinstance(j, int):
-    #< 
+    #< (0, 4),
     j
 
 # -----------------
@@ -302,3 +302,15 @@ x = 3
 {x:1 for  x in something}
 #< 10 (0,1), (0,10)
 {x:1 for  x in something}
+
+def x():
+    zzz = 3
+    if UNDEFINED:
+        zzz = 5
+        if UNDEFINED2:
+            #< (3, 8), (4, 4), (0, 12), (-3, 8), (-5, 4)
+            zzz
+    else:
+        #< (0, 8), (1, 4), (-3, 12), (-6, 8), (-8, 4)
+        zzz
+    zzz

test/run.py

@@ -158,7 +158,7 @@ class IntegrationTestCase(object):
         return self.line_nr - 1
 
     def __repr__(self):
-        return '<%s: %s:%s:%s>' % (self.__class__.__name__, self.module_name,
+        return '<%s: %s:%s %r>' % (self.__class__.__name__, self.path,
                                    self.line_nr_test, self.line.rstrip())
 
     def script(self):

test/test_api/test_interpreter.py

@@ -4,10 +4,18 @@ Tests of ``jedi.api.Interpreter``.
 import pytest
 
 import jedi
-from jedi._compatibility import is_py33, exec_function, py_version
+from jedi._compatibility import is_py33, py_version
 from jedi.evaluate.compiled import mixed
 
 
+if py_version > 30:
+    def exec_(source, global_map):
+        exec(source, global_map)
+else:
+    eval(compile("""def exec_(source, global_map):
+                        exec source in global_map """, 'blub', 'exec'))
+
+
 class _GlobalNameSpace():
     class SideEffectContainer():
         pass
@@ -247,7 +255,7 @@ def test_completion_param_annotations():
     # Need to define this function not directly in Python. Otherwise Jedi is to
     # clever and uses the Python code instead of the signature object.
     code = 'def foo(a: 1, b: str, c: int = 1.0): pass'
-    exec_function(code, locals())
+    exec_(code, locals())
     script = jedi.Interpreter('foo', [locals()])
     c, = script.completions()
     a, b, c = c.params

test/test_api/test_usages.py

@@ -1,48 +1,6 @@
 import jedi
-import os.path
+
 
 def test_import_usage():
     s = jedi.Script("from .. import foo", line=1, column=18, path="foo.py")
     assert [usage.line for usage in s.usages()] == [1]
-
-
-def usages_with_additional_modules(script, additional_modules):
-    """
-    Stripped down version of `jedi.api.Script.usages` that can take an
-    explicit set of additional modules. For use with `test_cross_module_usages`.
-    """
-
-    definition_names = jedi.api.usages.resolve_potential_imports(script._evaluator,
-                                                        script._goto())
-    modules = set([d.get_root_context() for d in definition_names])
-    modules.add(script._get_module())
-    for additional_module in additional_modules:
-        modules.add(additional_module._name.get_root_context())
-    return jedi.api.usages.usages(script._evaluator, definition_names, modules)
-
-
-def test_cross_module_usages():
-    """
-    This tests finding of usages between different modules. In
-    `jedi.api.usages.compare_contexts`, this exercises the case where
-    `c1 != c2`. This tests whether `jedi` can find the usage of
-    `import_tree_for_usages.b.bar` in `import_tree_for_usages.a`
-    """
-
-    def usages_script():
-        source = 'import import_tree_for_usages.b; import_tree_for_usages.b.bar'
-        return jedi.api.Script(source=source, line=1, column=len(source),
-                               sys_path=[os.path.dirname(os.path.abspath(__file__))])
-
-    def module_script():
-        source = 'import import_tree_for_usages.a; import_tree_for_usages.a'
-        return jedi.api.Script(source=source, line=1, column=len(source),
-                               sys_path=[os.path.dirname(os.path.abspath(__file__))])
-
-    module = module_script().goto_definitions()[0]
-    module_definition = module._name.get_root_context()
-    usages_list = usages_with_additional_modules(usages_script(), set([module]))
-
-    assert any([elt for elt in usages_list if elt.module_name == 'a']), (
-        "Did not find cross-module usage of :func:`b.bar` in :mod:`a`. Usages list was: {}"
-        .format(usages_list))

test/test_evaluate/test_buildout_detection.py

@@ -1,23 +1,17 @@
 import os
 from textwrap import dedent
 
-import parso
-
-from jedi._compatibility import u
+from jedi import Script
 from jedi.evaluate.sys_path import (_get_parent_dir_with_file,
                                     _get_buildout_script_paths,
-                                    sys_path_with_modifications,
-                                    _check_module)
-from jedi.evaluate import Evaluator
-from jedi.evaluate.representation import ModuleContext
+                                    check_sys_path_modifications)
 
 from ..helpers import cwd_at
 
 
 def check_module_test(code):
-    grammar = parso.load_grammar()
-    module_context = ModuleContext(Evaluator(grammar), parso.parse(code), path=None)
-    return _check_module(module_context)
+    module_context = Script(code)._get_module()
+    return check_sys_path_modifications(module_context)
 
 
 @cwd_at('test/test_evaluate/buildout_project/src/proj_name')
@@ -38,25 +32,27 @@ def test_buildout_detection():
 
 
 def test_append_on_non_sys_path():
-    code = dedent(u("""
+    code = dedent("""
         class Dummy(object):
             path = []
 
         d = Dummy()
-        d.path.append('foo')"""))
+        d.path.append('foo')"""
+    )
 
     paths = check_module_test(code)
-    assert len(paths) > 0
+    assert not paths
     assert 'foo' not in paths
 
 
 def test_path_from_invalid_sys_path_assignment():
-    code = dedent(u("""
+    code = dedent("""
         import sys
-        sys.path = 'invalid'"""))
+        sys.path = 'invalid'"""
+    )
 
     paths = check_module_test(code)
-    assert len(paths) > 0
+    assert not paths
     assert 'invalid' not in paths
 
 
@@ -67,15 +63,12 @@ def test_sys_path_with_modifications():
     """)
 
     path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
-    grammar = parso.load_grammar()
-    module_node = parso.parse(code, path=path)
-    module_context = ModuleContext(Evaluator(grammar), module_node, path=path)
-    paths = sys_path_with_modifications(module_context.evaluator, module_context)
+    paths = Script(code, path=path)._evaluator.project.sys_path
     assert '/tmp/.buildout/eggs/important_package.egg' in paths
 
 
 def test_path_from_sys_path_assignment():
-    code = dedent(u("""
+    code = dedent("""
         #!/usr/bin/python
 
         import sys
@@ -89,7 +82,8 @@ def test_path_from_sys_path_assignment():
         import important_package
 
         if __name__ == '__main__':
-            sys.exit(important_package.main())"""))
+            sys.exit(important_package.main())"""
+    )
 
     paths = check_module_test(code)
     assert 1 not in paths

test/test_evaluate/test_compiled.py

@@ -3,15 +3,17 @@ from textwrap import dedent
 import parso
 
 from jedi._compatibility import builtins, is_py3
-from jedi.evaluate import compiled, instance
-from jedi.evaluate.representation import FunctionContext
+from jedi.evaluate import compiled
+from jedi.evaluate.context import instance
+from jedi.evaluate.context.function import FunctionContext
 from jedi.evaluate import Evaluator
+from jedi.evaluate.project import Project
 from jedi.parser_utils import clean_scope_docstring
 from jedi import Script
 
 
 def _evaluator():
-    return Evaluator(parso.load_grammar())
+    return Evaluator(parso.load_grammar(), Project())
 
 
 def test_simple():
@@ -95,4 +97,4 @@ def test_time_docstring():
 
 
 def test_dict_values():
-    assert Script('import sys/sys.modules["alshdb;lasdhf"]').goto_definitions()
+    assert Script('import sys\nsys.modules["alshdb;lasdhf"]').goto_definitions()

test/test_evaluate/test_sys_path.py

@@ -10,13 +10,13 @@ from jedi import Script
 
 def test_paths_from_assignment():
     def paths(src):
-        script = Script(src)
+        script = Script(src, path='/foo/bar.py')
         expr_stmt = script._get_module_node().children[0]
         return set(sys_path._paths_from_assignment(script._get_module(), expr_stmt))
 
-    assert paths('sys.path[0:0] = ["a"]') == set(['a'])
-    assert paths('sys.path = ["b", 1, x + 3, y, "c"]') == set(['b', 'c'])
-    assert paths('sys.path = a = ["a"]') == set(['a'])
+    assert paths('sys.path[0:0] = ["a"]') == set(['/foo/a'])
+    assert paths('sys.path = ["b", 1, x + 3, y, "c"]') == set(['/foo/b', '/foo/c'])
+    assert paths('sys.path = a = ["a"]') == set(['/foo/a'])
 
     # Fail for complicated examples.
     assert paths('sys.path, other = ["a"], 2') == set()

tox.ini

@@ -1,8 +1,8 @@
 [tox]
-envlist = py26, py27, py33, py34, py35, py36
+envlist = py27, py33, py34, py35, py36
 [testenv]
 deps =
-    pytest>=2.3.5
+    pytest>=2.3.5, < 3.3
     pytest-cache
 # docopt for sith doctests
     docopt