Move the evaluate package to inference

jedi/inference/compiled/__init__.py

@@ -0,0 +1,64 @@
+from jedi._compatibility import unicode
+from jedi.inference.compiled.context import CompiledObject, CompiledName, \
+    CompiledObjectFilter, CompiledContextName, create_from_access_path
+from jedi.inference.base_context import ContextWrapper, LazyContextWrapper
+
+
+def builtin_from_name(evaluator, string):
+    typing_builtins_module = evaluator.builtins_module
+    if string in ('None', 'True', 'False'):
+        builtins, = typing_builtins_module.non_stub_context_set
+        filter_ = next(builtins.get_filters())
+    else:
+        filter_ = next(typing_builtins_module.get_filters())
+    name, = filter_.get(string)
+    context, = name.infer()
+    return context
+
+
+class CompiledValue(LazyContextWrapper):
+    def __init__(self, compiled_obj):
+        self.evaluator = compiled_obj.evaluator
+        self._compiled_obj = compiled_obj
+
+    def __getattribute__(self, name):
+        if name in ('get_safe_value', 'execute_operation', 'access_handle',
+                    'negate', 'py__bool__', 'is_compiled'):
+            return getattr(self._compiled_obj, name)
+        return super(CompiledValue, self).__getattribute__(name)
+
+    def _get_wrapped_context(self):
+        instance, = builtin_from_name(
+            self.evaluator, self._compiled_obj.name.string_name).execute_with_values()
+        return instance
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._compiled_obj)
+
+
+def create_simple_object(evaluator, obj):
+    """
+    Only allows creations of objects that are easily picklable across Python
+    versions.
+    """
+    assert type(obj) in (int, float, str, bytes, unicode, slice, complex, bool), obj
+    compiled_obj = create_from_access_path(
+        evaluator,
+        evaluator.compiled_subprocess.create_simple_object(obj)
+    )
+    return CompiledValue(compiled_obj)
+
+
+def get_string_context_set(evaluator):
+    return builtin_from_name(evaluator, u'str').execute_with_values()
+
+
+def load_module(evaluator, dotted_name, **kwargs):
+    # Temporary, some tensorflow builtins cannot be loaded, so it's tried again
+    # and again and it's really slow.
+    if dotted_name.startswith('tensorflow.'):
+        return None
+    access_path = evaluator.compiled_subprocess.load_module(dotted_name=dotted_name, **kwargs)
+    if access_path is None:
+        return None
+    return create_from_access_path(evaluator, access_path)

jedi/inference/compiled/access.py

@@ -0,0 +1,497 @@
+from __future__ import print_function
+import inspect
+import types
+import sys
+import operator as op
+from collections import namedtuple
+
+from jedi._compatibility import unicode, is_py3, builtins, \
+    py_version, force_unicode
+from jedi.inference.compiled.getattr_static import getattr_static
+
+ALLOWED_GETITEM_TYPES = (str, list, tuple, unicode, bytes, bytearray, dict)
+
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace,
+        types.DynamicClassAttribute,
+    )
+
+
+# Those types don't exist in typing.
+MethodDescriptorType = type(str.replace)
+WrapperDescriptorType = type(set.__iter__)
+# `object.__subclasshook__` is an already executed descriptor.
+object_class_dict = type.__dict__["__dict__"].__get__(object)
+ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
+
+_sentinel = object()
+
+# Maps Python syntax to the operator module.
+COMPARISON_OPERATORS = {
+    '==': op.eq,
+    '!=': op.ne,
+    'is': op.is_,
+    'is not': op.is_not,
+    '<': op.lt,
+    '<=': op.le,
+    '>': op.gt,
+    '>=': op.ge,
+}
+
+_OPERATORS = {
+    '+': op.add,
+    '-': op.sub,
+}
+_OPERATORS.update(COMPARISON_OPERATORS)
+
+ALLOWED_DESCRIPTOR_ACCESS = (
+    types.FunctionType,
+    types.GetSetDescriptorType,
+    types.MemberDescriptorType,
+    MethodDescriptorType,
+    WrapperDescriptorType,
+    ClassMethodDescriptorType,
+    staticmethod,
+    classmethod,
+)
+
+
+def safe_getattr(obj, name, default=_sentinel):
+    try:
+        attr, is_get_descriptor = getattr_static(obj, name)
+    except AttributeError:
+        if default is _sentinel:
+            raise
+        return default
+    else:
+        if isinstance(attr, ALLOWED_DESCRIPTOR_ACCESS):
+            # In case of descriptors that have get methods we cannot return
+            # it's value, because that would mean code execution.
+            # Since it's an isinstance call, code execution is still possible,
+            # but this is not really a security feature, but much more of a
+            # safety feature. Code execution is basically always possible when
+            # a module is imported. This is here so people don't shoot
+            # themselves in the foot.
+            return getattr(obj, name)
+    return attr
+
+
+SignatureParam = namedtuple(
+    'SignatureParam',
+    'name has_default default default_string has_annotation annotation annotation_string kind_name'
+)
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent_context=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj)
+            try:
+                cache[key]
+                return cache[key][0]
+            except KeyError:
+                # TODO wuaaaarrghhhhhhhh
+                if attribute_name == 'mixed_cache':
+                    result = func(evaluator, obj, parent_context)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent_context
+                return result
+        return wrapper
+
+    return decorator
+
+
+def create_access(evaluator, obj):
+    return evaluator.compiled_subprocess.get_or_create_access_handle(obj)
+
+
+def load_module(evaluator, dotted_name, sys_path):
+    temp, sys.path = sys.path, sys_path
+    try:
+        __import__(dotted_name)
+    except ImportError:
+        # If a module is "corrupt" or not really a Python module or whatever.
+        print('Module %s not importable in path %s.' % (dotted_name, sys_path), file=sys.stderr)
+        return None
+    except Exception:
+        # Since __import__ pretty much makes code execution possible, just
+        # catch any error here and print it.
+        import traceback
+        print("Cannot import:\n%s" % traceback.format_exc(), file=sys.stderr)
+        return None
+    finally:
+        sys.path = temp
+
+    # Just access the cache after import, because of #59 as well as the very
+    # complicated import structure of Python.
+    module = sys.modules[dotted_name]
+    return create_access_path(evaluator, module)
+
+
+class AccessPath(object):
+    def __init__(self, accesses):
+        self.accesses = accesses
+
+    # Writing both of these methods here looks a bit ridiculous. However with
+    # the differences of Python 2/3 it's actually necessary, because we will
+    # otherwise have a accesses attribute that is bytes instead of unicode.
+    def __getstate__(self):
+        return self.accesses
+
+    def __setstate__(self, value):
+        self.accesses = value
+
+
+def create_access_path(evaluator, obj):
+    access = create_access(evaluator, obj)
+    return AccessPath(access.get_access_path_tuples())
+
+
+def _force_unicode_decorator(func):
+    return lambda *args, **kwargs: force_unicode(func(*args, **kwargs))
+
+
+def get_api_type(obj):
+    if inspect.isclass(obj):
+        return u'class'
+    elif inspect.ismodule(obj):
+        return u'module'
+    elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+            or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
+        return u'function'
+    # Everything else...
+    return u'instance'
+
+
+class DirectObjectAccess(object):
+    def __init__(self, evaluator, obj):
+        self._evaluator = evaluator
+        self._obj = obj
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.get_repr())
+
+    def _create_access(self, obj):
+        return create_access(self._evaluator, obj)
+
+    def _create_access_path(self, obj):
+        return create_access_path(self._evaluator, obj)
+
+    def py__bool__(self):
+        return bool(self._obj)
+
+    def py__file__(self):
+        try:
+            return self._obj.__file__
+        except AttributeError:
+            return None
+
+    def py__doc__(self):
+        return force_unicode(inspect.getdoc(self._obj)) or u''
+
+    def py__name__(self):
+        if not _is_class_instance(self._obj) or \
+                inspect.ismethoddescriptor(self._obj):  # slots
+            cls = self._obj
+        else:
+            try:
+                cls = self._obj.__class__
+            except AttributeError:
+                # happens with numpy.core.umath._UFUNC_API (you get it
+                # automatically by doing `import numpy`.
+                return None
+
+        try:
+            return force_unicode(cls.__name__)
+        except AttributeError:
+            return None
+
+    def py__mro__accesses(self):
+        return tuple(self._create_access_path(cls) for cls in self._obj.__mro__[1:])
+
+    def py__getitem__all_values(self):
+        if isinstance(self._obj, dict):
+            return [self._create_access_path(v) for v in self._obj.values()]
+        return self.py__iter__list()
+
+    def py__simple_getitem__(self, index):
+        if type(self._obj) not in ALLOWED_GETITEM_TYPES:
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return None
+
+        return self._create_access_path(self._obj[index])
+
+    def py__iter__list(self):
+        if not hasattr(self._obj, '__getitem__'):
+            return None
+
+        if type(self._obj) not in ALLOWED_GETITEM_TYPES:
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return []
+
+        lst = []
+        for i, part in enumerate(self._obj):
+            if i > 20:
+                # Should not go crazy with large iterators
+                break
+            lst.append(self._create_access_path(part))
+        return lst
+
+    def py__class__(self):
+        return self._create_access_path(self._obj.__class__)
+
+    def py__bases__(self):
+        return [self._create_access_path(base) for base in self._obj.__bases__]
+
+    def py__path__(self):
+        return self._obj.__path__
+
+    @_force_unicode_decorator
+    def get_repr(self):
+        builtins = 'builtins', '__builtin__'
+
+        if inspect.ismodule(self._obj):
+            return repr(self._obj)
+        # Try to avoid execution of the property.
+        if safe_getattr(self._obj, '__module__', default='') in builtins:
+            return repr(self._obj)
+
+        type_ = type(self._obj)
+        if type_ == type:
+            return type.__repr__(self._obj)
+
+        if safe_getattr(type_, '__module__', default='') in builtins:
+            # Allow direct execution of repr for builtins.
+            return repr(self._obj)
+        return object.__repr__(self._obj)
+
+    def is_class(self):
+        return inspect.isclass(self._obj)
+
+    def is_module(self):
+        return inspect.ismodule(self._obj)
+
+    def is_instance(self):
+        return _is_class_instance(self._obj)
+
+    def ismethoddescriptor(self):
+        return inspect.ismethoddescriptor(self._obj)
+
+    def get_qualified_names(self):
+        def try_to_get_name(obj):
+            return getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+
+        if self.is_module():
+            return ()
+        name = try_to_get_name(self._obj)
+        if name is None:
+            name = try_to_get_name(type(self._obj))
+            if name is None:
+                return ()
+        return tuple(name.split('.'))
+
+    def dir(self):
+        return list(map(force_unicode, dir(self._obj)))
+
+    def has_iter(self):
+        try:
+            iter(self._obj)
+            return True
+        except TypeError:
+            return False
+
+    def is_allowed_getattr(self, name):
+        # TODO this API is ugly.
+        try:
+            attr, is_get_descriptor = getattr_static(self._obj, name)
+        except AttributeError:
+            return False, False
+        else:
+            if is_get_descriptor and type(attr) not in ALLOWED_DESCRIPTOR_ACCESS:
+                # In case of descriptors that have get methods we cannot return
+                # it's value, because that would mean code execution.
+                return True, True
+        return True, False
+
+    def getattr_paths(self, name, default=_sentinel):
+        try:
+            return_obj = getattr(self._obj, name)
+        except Exception as e:
+            if default is _sentinel:
+                if isinstance(e, AttributeError):
+                    # Happens e.g. in properties of
+                    # PyQt4.QtGui.QStyleOptionComboBox.currentText
+                    # -> just set it to None
+                    raise
+                # Just in case anything happens, return an AttributeError. It
+                # should not crash.
+                raise AttributeError
+            return_obj = default
+        access = self._create_access(return_obj)
+        if inspect.ismodule(return_obj):
+            return [access]
+
+        module = inspect.getmodule(return_obj)
+        if module is None:
+            module = inspect.getmodule(type(return_obj))
+            if module is None:
+                module = builtins
+        return [self._create_access(module), access]
+
+    def get_safe_value(self):
+        if type(self._obj) in (bool, bytes, float, int, str, unicode, slice):
+            return self._obj
+        raise ValueError("Object is type %s and not simple" % type(self._obj))
+
+    def get_api_type(self):
+        return get_api_type(self._obj)
+
+    def get_access_path_tuples(self):
+        accesses = [create_access(self._evaluator, o) for o in self._get_objects_path()]
+        return [(access.py__name__(), access) for access in accesses]
+
+    def _get_objects_path(self):
+        def get():
+            obj = self._obj
+            yield obj
+            try:
+                obj = obj.__objclass__
+            except AttributeError:
+                pass
+            else:
+                yield obj
+
+            try:
+                # Returns a dotted string path.
+                imp_plz = obj.__module__
+            except AttributeError:
+                # Unfortunately in some cases like `int` there's no __module__
+                if not inspect.ismodule(obj):
+                    yield builtins
+            else:
+                if imp_plz is None:
+                    # Happens for example in `(_ for _ in []).send.__module__`.
+                    yield builtins
+                else:
+                    try:
+                        yield sys.modules[imp_plz]
+                    except KeyError:
+                        # __module__ can be something arbitrary that doesn't exist.
+                        yield builtins
+
+        return list(reversed(list(get())))
+
+    def execute_operation(self, other_access_handle, operator):
+        other_access = other_access_handle.access
+        op = _OPERATORS[operator]
+        return self._create_access_path(op(self._obj, other_access._obj))
+
+    def needs_type_completions(self):
+        return inspect.isclass(self._obj) and self._obj != type
+
+    def get_signature_params(self):
+        return [
+            SignatureParam(
+                name=p.name,
+                has_default=p.default is not p.empty,
+                default=self._create_access_path(p.default),
+                default_string=repr(p.default),
+                has_annotation=p.annotation is not p.empty,
+                annotation=self._create_access_path(p.annotation),
+                annotation_string=str(p.default),
+                kind_name=str(p.kind)
+            ) for p in self._get_signature().parameters.values()
+        ]
+
+    def _get_signature(self):
+        obj = self._obj
+        if py_version < 33:
+            raise ValueError("inspect.signature was introduced in 3.3")
+        if py_version == 34:
+            # In 3.4 inspect.signature are wrong for str and int. This has
+            # been fixed in 3.5. The signature of object is returned,
+            # because no signature was found for str. Here we imitate 3.5
+            # logic and just ignore the signature if the magic methods
+            # don't match object.
+            # 3.3 doesn't even have the logic and returns nothing for str
+            # and classes that inherit from object.
+            user_def = inspect._signature_get_user_defined_method
+            if (inspect.isclass(obj)
+                    and not user_def(type(obj), '__init__')
+                    and not user_def(type(obj), '__new__')
+                    and (obj.__init__ != object.__init__
+                         or obj.__new__ != object.__new__)):
+                raise ValueError
+
+        try:
+            return inspect.signature(obj)
+        except (RuntimeError, TypeError):
+            # Reading the code of the function in Python 3.6 implies there are
+            # at least these errors that might occur if something is wrong with
+            # the signature. In that case we just want a simple escape for now.
+            raise ValueError
+
+    def get_return_annotation(self):
+        try:
+            o = self._obj.__annotations__.get('return')
+        except AttributeError:
+            return None
+
+        if o is None:
+            return None
+
+        return self._create_access_path(o)
+
+    def negate(self):
+        return self._create_access_path(-self._obj)
+
+    def get_dir_infos(self):
+        """
+        Used to return a couple of infos that are needed when accessing the sub
+        objects of an objects
+        """
+        # TODO is_allowed_getattr might raise an AttributeError
+        tuples = dict(
+            (force_unicode(name), self.is_allowed_getattr(name))
+            for name in self.dir()
+        )
+        return self.needs_type_completions(), tuples
+
+
+def _is_class_instance(obj):
+    """Like inspect.* methods."""
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

jedi/inference/compiled/context.py

@@ -0,0 +1,541 @@
+"""
+Imitate the parser representation.
+"""
+import re
+from functools import partial
+
+from jedi import debug
+from jedi.inference.utils import to_list
+from jedi._compatibility import force_unicode, Parameter, cast_path
+from jedi.cache import underscore_memoization, memoize_method
+from jedi.inference.filters import AbstractFilter
+from jedi.inference.names import AbstractNameDefinition, ContextNameMixin, \
+    ParamNameInterface
+from jedi.inference.base_context import Context, ContextSet, NO_CONTEXTS
+from jedi.inference.lazy_context import LazyKnownContext
+from jedi.inference.compiled.access import _sentinel
+from jedi.inference.cache import evaluator_function_cache
+from jedi.inference.helpers import reraise_getitem_errors
+from jedi.inference.signature import BuiltinSignature
+
+
+class CheckAttribute(object):
+    """Raises an AttributeError if the attribute X isn't available."""
+    def __init__(self, check_name=None):
+        # Remove the py in front of e.g. py__call__.
+        self.check_name = check_name
+
+    def __call__(self, func):
+        self.func = func
+        if self.check_name is None:
+            self.check_name = force_unicode(func.__name__[2:])
+        return self
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        # This might raise an AttributeError. That's wanted.
+        instance.access_handle.getattr_paths(self.check_name)
+        return partial(self.func, instance)
+
+
+class CompiledObject(Context):
+    def __init__(self, evaluator, access_handle, parent_context=None):
+        super(CompiledObject, self).__init__(evaluator, parent_context)
+        self.access_handle = access_handle
+
+    def py__call__(self, arguments):
+        return_annotation = self.access_handle.get_return_annotation()
+        if return_annotation is not None:
+            # TODO the return annotation may also be a string.
+            return create_from_access_path(self.evaluator, return_annotation).execute_annotation()
+
+        try:
+            self.access_handle.getattr_paths(u'__call__')
+        except AttributeError:
+            return super(CompiledObject, self).py__call__(arguments)
+        else:
+            if self.access_handle.is_class():
+                from jedi.inference.context import CompiledInstance
+                return ContextSet([
+                    CompiledInstance(self.evaluator, self.parent_context, self, arguments)
+                ])
+            else:
+                return ContextSet(self._execute_function(arguments))
+
+    @CheckAttribute()
+    def py__class__(self):
+        return create_from_access_path(self.evaluator, self.access_handle.py__class__())
+
+    @CheckAttribute()
+    def py__mro__(self):
+        return (self,) + tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__mro__accesses()
+        )
+
+    @CheckAttribute()
+    def py__bases__(self):
+        return tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__bases__()
+        )
+
+    @CheckAttribute()
+    def py__path__(self):
+        return map(cast_path, self.access_handle.py__path__())
+
+    @property
+    def string_names(self):
+        # For modules
+        name = self.py__name__()
+        if name is None:
+            return ()
+        return tuple(name.split('.'))
+
+    def get_qualified_names(self):
+        return self.access_handle.get_qualified_names()
+
+    def py__bool__(self):
+        return self.access_handle.py__bool__()
+
+    def py__file__(self):
+        return cast_path(self.access_handle.py__file__())
+
+    def is_class(self):
+        return self.access_handle.is_class()
+
+    def is_module(self):
+        return self.access_handle.is_module()
+
+    def is_compiled(self):
+        return True
+
+    def is_stub(self):
+        return False
+
+    def is_instance(self):
+        return self.access_handle.is_instance()
+
+    def py__doc__(self):
+        return self.access_handle.py__doc__()
+
+    @to_list
+    def get_param_names(self):
+        try:
+            signature_params = self.access_handle.get_signature_params()
+        except ValueError:  # Has no signature
+            params_str, ret = self._parse_function_doc()
+            if not params_str:
+                tokens = []
+            else:
+                tokens = params_str.split(',')
+            if self.access_handle.ismethoddescriptor():
+                tokens.insert(0, 'self')
+            for p in tokens:
+                name, _, default = p.strip().partition('=')
+                yield UnresolvableParamName(self, name, default)
+        else:
+            for signature_param in signature_params:
+                yield SignatureParamName(self, signature_param)
+
+    def get_signatures(self):
+        _, return_string = self._parse_function_doc()
+        return [BuiltinSignature(self, return_string)]
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.access_handle.get_repr())
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        doc = self.py__doc__()
+        if doc is None:
+            return '', ''
+
+        return _parse_function_doc(doc)
+
+    @property
+    def api_type(self):
+        return self.access_handle.get_api_type()
+
+    @underscore_memoization
+    def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
+        # Ensures that a CompiledObject is returned that is not an instance (like list)
+        return self
+
+    def get_filters(self, search_global=False, is_instance=False,
+                    until_position=None, origin_scope=None):
+        yield self._ensure_one_filter(is_instance)
+
+    @memoize_method
+    def _ensure_one_filter(self, is_instance):
+        """
+        search_global shouldn't change the fact that there's one dict, this way
+        there's only one `object`.
+        """
+        return CompiledObjectFilter(self.evaluator, self, is_instance)
+
+    @CheckAttribute(u'__getitem__')
+    def py__simple_getitem__(self, index):
+        with reraise_getitem_errors(IndexError, KeyError, TypeError):
+            access = self.access_handle.py__simple_getitem__(index)
+        if access is None:
+            return NO_CONTEXTS
+
+        return ContextSet([create_from_access_path(self.evaluator, access)])
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        all_access_paths = self.access_handle.py__getitem__all_values()
+        if all_access_paths is None:
+            # This means basically that no __getitem__ has been defined on this
+            # object.
+            return super(CompiledObject, self).py__getitem__(index_context_set, contextualized_node)
+        return ContextSet(
+            create_from_access_path(self.evaluator, access)
+            for access in all_access_paths
+        )
+
+    def py__iter__(self, contextualized_node=None):
+        # Python iterators are a bit strange, because there's no need for
+        # the __iter__ function as long as __getitem__ is defined (it will
+        # just start with __getitem__(0). This is especially true for
+        # Python 2 strings, where `str.__iter__` is not even defined.
+        if not self.access_handle.has_iter():
+            for x in super(CompiledObject, self).py__iter__(contextualized_node):
+                yield x
+
+        access_path_list = self.access_handle.py__iter__list()
+        if access_path_list is None:
+            # There is no __iter__ method on this object.
+            return
+
+        for access in access_path_list:
+            yield LazyKnownContext(create_from_access_path(self.evaluator, access))
+
+    def py__name__(self):
+        return self.access_handle.py__name__()
+
+    @property
+    def name(self):
+        name = self.py__name__()
+        if name is None:
+            name = self.access_handle.get_repr()
+        return CompiledContextName(self, name)
+
+    def _execute_function(self, params):
+        from jedi.inference import docstrings
+        from jedi.inference.compiled import builtin_from_name
+        if self.api_type != 'function':
+            return
+
+        for name in self._parse_function_doc()[1].split():
+            try:
+                # TODO wtf is this? this is exactly the same as the thing
+                # below. It uses getattr as well.
+                self.evaluator.builtins_module.access_handle.getattr_paths(name)
+            except AttributeError:
+                continue
+            else:
+                bltn_obj = builtin_from_name(self.evaluator, name)
+                for result in self.evaluator.execute(bltn_obj, params):
+                    yield result
+        for type_ in docstrings.infer_return_types(self):
+            yield type_
+
+    def get_safe_value(self, default=_sentinel):
+        try:
+            return self.access_handle.get_safe_value()
+        except ValueError:
+            if default == _sentinel:
+                raise
+            return default
+
+    def execute_operation(self, other, operator):
+        return create_from_access_path(
+            self.evaluator,
+            self.access_handle.execute_operation(other.access_handle, operator)
+        )
+
+    def negate(self):
+        return create_from_access_path(self.evaluator, self.access_handle.negate())
+
+    def get_metaclasses(self):
+        return NO_CONTEXTS
+
+
+class CompiledName(AbstractNameDefinition):
+    def __init__(self, evaluator, parent_context, name):
+        self._evaluator = evaluator
+        self.parent_context = parent_context
+        self.string_name = name
+
+    def _get_qualified_names(self):
+        parent_qualified_names = self.parent_context.get_qualified_names()
+        return parent_qualified_names + (self.string_name,)
+
+    def __repr__(self):
+        try:
+            name = self.parent_context.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (self.__class__.__name__, name, self.string_name)
+
+    @property
+    def api_type(self):
+        api = self.infer()
+        # If we can't find the type, assume it is an instance variable
+        if not api:
+            return "instance"
+        return next(iter(api)).api_type
+
+    @underscore_memoization
+    def infer(self):
+        return ContextSet([_create_from_name(
+            self._evaluator, self.parent_context, self.string_name
+        )])
+
+
+class SignatureParamName(ParamNameInterface, AbstractNameDefinition):
+    def __init__(self, compiled_obj, signature_param):
+        self.parent_context = compiled_obj.parent_context
+        self._signature_param = signature_param
+
+    @property
+    def string_name(self):
+        return self._signature_param.name
+
+    def to_string(self):
+        s = self._kind_string() + self.string_name
+        if self._signature_param.has_annotation:
+            s += ': ' + self._signature_param.annotation_string
+        if self._signature_param.has_default:
+            s += '=' + self._signature_param.default_string
+        return s
+
+    def get_kind(self):
+        return getattr(Parameter, self._signature_param.kind_name)
+
+    def infer(self):
+        p = self._signature_param
+        evaluator = self.parent_context.evaluator
+        contexts = NO_CONTEXTS
+        if p.has_default:
+            contexts = ContextSet([create_from_access_path(evaluator, p.default)])
+        if p.has_annotation:
+            annotation = create_from_access_path(evaluator, p.annotation)
+            contexts |= annotation.execute_with_values()
+        return contexts
+
+
+class UnresolvableParamName(ParamNameInterface, AbstractNameDefinition):
+    def __init__(self, compiled_obj, name, default):
+        self.parent_context = compiled_obj.parent_context
+        self.string_name = name
+        self._default = default
+
+    def get_kind(self):
+        return Parameter.POSITIONAL_ONLY
+
+    def to_string(self):
+        string = self.string_name
+        if self._default:
+            string += '=' + self._default
+        return string
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
+    def __init__(self, context, name):
+        self.string_name = name
+        self._context = context
+        self.parent_context = context.parent_context
+
+
+class EmptyCompiledName(AbstractNameDefinition):
+    """
+    Accessing some names will raise an exception. To avoid not having any
+    completions, just give Jedi the option to return this object. It infers to
+    nothing.
+    """
+    def __init__(self, evaluator, name):
+        self.parent_context = evaluator.builtins_module
+        self.string_name = name
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class CompiledObjectFilter(AbstractFilter):
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_object, is_instance=False):
+        self._evaluator = evaluator
+        self.compiled_object = compiled_object
+        self.is_instance = is_instance
+
+    def get(self, name):
+        return self._get(
+            name,
+            lambda: self.compiled_object.access_handle.is_allowed_getattr(name),
+            lambda: self.compiled_object.access_handle.dir(),
+            check_has_attribute=True
+        )
+
+    def _get(self, name, allowed_getattr_callback, dir_callback, check_has_attribute=False):
+        """
+        To remove quite a few access calls we introduced the callback here.
+        """
+        has_attribute, is_descriptor = allowed_getattr_callback()
+        if check_has_attribute and not has_attribute:
+            return []
+
+        # Always use unicode objects in Python 2 from here.
+        name = force_unicode(name)
+
+        if (is_descriptor and not self._evaluator.allow_descriptor_getattr) or not has_attribute:
+            return [self._get_cached_name(name, is_empty=True)]
+
+        if self.is_instance and name not in dir_callback():
+            return []
+        return [self._get_cached_name(name)]
+
+    @memoize_method
+    def _get_cached_name(self, name, is_empty=False):
+        if is_empty:
+            return EmptyCompiledName(self._evaluator, name)
+        else:
+            return self._create_name(name)
+
+    def values(self):
+        from jedi.inference.compiled import builtin_from_name
+        names = []
+        needs_type_completions, dir_infos = self.compiled_object.access_handle.get_dir_infos()
+        for name in dir_infos:
+            names += self._get(
+                name,
+                lambda: dir_infos[name],
+                lambda: dir_infos.keys(),
+            )
+
+        # ``dir`` doesn't include the type names.
+        if not self.is_instance and needs_type_completions:
+            for filter in builtin_from_name(self._evaluator, u'type').get_filters():
+                names += filter.values()
+        return names
+
+    def _create_name(self, name):
+        return self.name_class(self._evaluator, self.compiled_object, name)
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self.compiled_object)
+
+
+docstr_defaults = {
+    'floating point number': u'float',
+    'character': u'str',
+    'integer': u'int',
+    'dictionary': u'dict',
+    'string': u'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    doc = force_unicode(doc)
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = u''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search(u'-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = u''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+def _create_from_name(evaluator, compiled_object, name):
+    access_paths = compiled_object.access_handle.getattr_paths(name, default=None)
+    parent_context = compiled_object
+    if parent_context.is_class():
+        parent_context = parent_context.parent_context
+
+    context = None
+    for access_path in access_paths:
+        context = create_cached_compiled_object(
+            evaluator, access_path, parent_context=context
+        )
+    return context
+
+
+def _normalize_create_args(func):
+    """The cache doesn't care about keyword vs. normal args."""
+    def wrapper(evaluator, obj, parent_context=None):
+        return func(evaluator, obj, parent_context)
+    return wrapper
+
+
+def create_from_access_path(evaluator, access_path):
+    parent_context = None
+    for name, access in access_path.accesses:
+        parent_context = create_cached_compiled_object(evaluator, access, parent_context)
+    return parent_context
+
+
+@_normalize_create_args
+@evaluator_function_cache()
+def create_cached_compiled_object(evaluator, access_handle, parent_context):
+    return CompiledObject(evaluator, access_handle, parent_context)

jedi/inference/compiled/getattr_static.py

@@ -0,0 +1,176 @@
+"""
+A static version of getattr.
+This is a backport of the Python 3 code with a little bit of additional
+information returned to enable Jedi to make decisions.
+"""
+
+import types
+
+from jedi._compatibility import py_version
+
+_sentinel = object()
+
+
+def _check_instance(obj, attr):
+    instance_dict = {}
+    try:
+        instance_dict = object.__getattribute__(obj, "__dict__")
+    except AttributeError:
+        pass
+    return dict.get(instance_dict, attr, _sentinel)
+
+
+def _check_class(klass, attr):
+    for entry in _static_getmro(klass):
+        if _shadowed_dict(type(entry)) is _sentinel:
+            try:
+                return entry.__dict__[attr]
+            except KeyError:
+                pass
+    return _sentinel
+
+
+def _is_type(obj):
+    try:
+        _static_getmro(obj)
+    except TypeError:
+        return False
+    return True
+
+
+def _shadowed_dict_newstyle(klass):
+    dict_attr = type.__dict__["__dict__"]
+    for entry in _static_getmro(klass):
+        try:
+            class_dict = dict_attr.__get__(entry)["__dict__"]
+        except KeyError:
+            pass
+        else:
+            if not (type(class_dict) is types.GetSetDescriptorType and
+                    class_dict.__name__ == "__dict__" and
+                    class_dict.__objclass__ is entry):
+                return class_dict
+    return _sentinel
+
+
+def _static_getmro_newstyle(klass):
+    return type.__dict__['__mro__'].__get__(klass)
+
+
+if py_version >= 30:
+    _shadowed_dict = _shadowed_dict_newstyle
+    _get_type = type
+    _static_getmro = _static_getmro_newstyle
+else:
+    def _shadowed_dict(klass):
+        """
+        In Python 2 __dict__ is not overwritable:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            TypeError: __dict__ must be a dictionary object
+
+        It applies to both newstyle and oldstyle classes:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            AttributeError: attribute '__dict__' of 'type' objects is not writable
+
+        It also applies to instances of those objects. However to keep things
+        straight forward, newstyle classes always use the complicated way of
+        accessing it while oldstyle classes just use getattr.
+        """
+        if type(klass) is _oldstyle_class_type:
+            return getattr(klass, '__dict__', _sentinel)
+        return _shadowed_dict_newstyle(klass)
+
+    class _OldStyleClass:
+        pass
+
+    _oldstyle_instance_type = type(_OldStyleClass())
+    _oldstyle_class_type = type(_OldStyleClass)
+
+    def _get_type(obj):
+        type_ = object.__getattribute__(obj, '__class__')
+        if type_ is _oldstyle_instance_type:
+            # Somehow for old style classes we need to access it directly.
+            return obj.__class__
+        return type_
+
+    def _static_getmro(klass):
+        if type(klass) is _oldstyle_class_type:
+            def oldstyle_mro(klass):
+                """
+                Oldstyle mro is a really simplistic way of look up mro:
+                https://stackoverflow.com/questions/54867/what-is-the-difference-between-old-style-and-new-style-classes-in-python
+                """
+                yield klass
+                for base in klass.__bases__:
+                    for yield_from in oldstyle_mro(base):
+                        yield yield_from
+
+            return oldstyle_mro(klass)
+
+        return _static_getmro_newstyle(klass)
+
+
+def _safe_hasattr(obj, name):
+    return _check_class(_get_type(obj), name) is not _sentinel
+
+
+def _safe_is_data_descriptor(obj):
+    return _safe_hasattr(obj, '__set__') or _safe_hasattr(obj, '__delete__')
+
+
+def getattr_static(obj, attr, default=_sentinel):
+    """Retrieve attributes without triggering dynamic lookup via the
+       descriptor protocol,  __getattr__ or __getattribute__.
+
+       Note: this function may not be able to retrieve all attributes
+       that getattr can fetch (like dynamically created attributes)
+       and may find attributes that getattr can't (like descriptors
+       that raise AttributeError). It can also return descriptor objects
+       instead of instance members in some cases. See the
+       documentation for details.
+
+       Returns a tuple `(attr, is_get_descriptor)`. is_get_descripter means that
+       the attribute is a descriptor that has a `__get__` attribute.
+    """
+    instance_result = _sentinel
+    if not _is_type(obj):
+        klass = _get_type(obj)
+        dict_attr = _shadowed_dict(klass)
+        if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType):
+            instance_result = _check_instance(obj, attr)
+    else:
+        klass = obj
+
+    klass_result = _check_class(klass, attr)
+
+    if instance_result is not _sentinel and klass_result is not _sentinel:
+        if _safe_hasattr(klass_result, '__get__') \
+                and _safe_is_data_descriptor(klass_result):
+            # A get/set descriptor has priority over everything.
+            return klass_result, True
+
+    if instance_result is not _sentinel:
+        return instance_result, False
+    if klass_result is not _sentinel:
+        return klass_result, _safe_hasattr(klass_result, '__get__')
+
+    if obj is klass:
+        # for types we check the metaclass too
+        for entry in _static_getmro(type(klass)):
+            if _shadowed_dict(type(entry)) is _sentinel:
+                try:
+                    return entry.__dict__[attr], False
+                except KeyError:
+                    pass
+    if default is not _sentinel:
+        return default, False
+    raise AttributeError(attr)

jedi/inference/compiled/mixed.py

@@ -0,0 +1,291 @@
+"""
+Used only for REPL Completion.
+"""
+
+import inspect
+import os
+import sys
+
+from jedi.parser_utils import get_cached_code_lines
+
+from jedi import settings
+from jedi.inference import compiled
+from jedi.cache import underscore_memoization
+from jedi.file_io import FileIO
+from jedi.inference.base_context import ContextSet, ContextWrapper
+from jedi.inference.helpers import SimpleGetItemNotFound
+from jedi.inference.context import ModuleContext
+from jedi.inference.cache import evaluator_function_cache
+from jedi.inference.compiled.getattr_static import getattr_static
+from jedi.inference.compiled.access import compiled_objects_cache, \
+    ALLOWED_GETITEM_TYPES, get_api_type
+from jedi.inference.compiled.context import create_cached_compiled_object
+from jedi.inference.gradual.conversion import to_stub
+
+_sentinel = object()
+
+
+class MixedObject(ContextWrapper):
+    """
+    A ``MixedObject`` is used in two ways:
+
+    1. It uses the default logic of ``parser.python.tree`` objects,
+    2. except for getattr calls. The names dicts are generated in a fashion
+       like ``CompiledObject``.
+
+    This combined logic makes it possible to provide more powerful REPL
+    completion. It allows side effects that are not noticable with the default
+    parser structure to still be completeable.
+
+    The biggest difference from CompiledObject to MixedObject is that we are
+    generally dealing with Python code and not with C code. This will generate
+    fewer special cases, because we in Python you don't have the same freedoms
+    to modify the runtime.
+    """
+    def __init__(self, compiled_object, tree_context):
+        super(MixedObject, self).__init__(tree_context)
+        self.compiled_object = compiled_object
+        self.access_handle = compiled_object.access_handle
+
+    def get_filters(self, *args, **kwargs):
+        yield MixedObjectFilter(self.evaluator, self)
+
+    def get_signatures(self):
+        # Prefer `inspect.signature` over somehow analyzing Python code. It
+        # should be very precise, especially for stuff like `partial`.
+        return self.compiled_object.get_signatures()
+
+    def py__call__(self, arguments):
+        return (to_stub(self._wrapped_context) or self._wrapped_context).py__call__(arguments)
+
+    def get_safe_value(self, default=_sentinel):
+        if default is _sentinel:
+            return self.compiled_object.get_safe_value()
+        else:
+            return self.compiled_object.get_safe_value(default)
+
+    def py__simple_getitem__(self, index):
+        python_object = self.compiled_object.access_handle.access._obj
+        if type(python_object) in ALLOWED_GETITEM_TYPES:
+            return self.compiled_object.py__simple_getitem__(index)
+        raise SimpleGetItemNotFound
+
+    def __repr__(self):
+        return '<%s: %s>' % (
+            type(self).__name__,
+            self.access_handle.get_repr()
+        )
+
+
+class MixedName(compiled.CompiledName):
+    """
+    The ``CompiledName._compiled_object`` is our MixedObject.
+    """
+    @property
+    def start_pos(self):
+        contexts = list(self.infer())
+        if not contexts:
+            # This means a start_pos that doesn't exist (compiled objects).
+            return 0, 0
+        return contexts[0].name.start_pos
+
+    @start_pos.setter
+    def start_pos(self, value):
+        # Ignore the __init__'s start_pos setter call.
+        pass
+
+    @underscore_memoization
+    def infer(self):
+        # TODO use logic from compiled.CompiledObjectFilter
+        access_paths = self.parent_context.access_handle.getattr_paths(
+            self.string_name,
+            default=None
+        )
+        assert len(access_paths)
+        contexts = [None]
+        for access in access_paths:
+            contexts = ContextSet.from_sets(
+                _create(self._evaluator, access, parent_context=c)
+                if c is None or isinstance(c, MixedObject)
+                else ContextSet({create_cached_compiled_object(c.evaluator, access, c)})
+                for c in contexts
+            )
+        return contexts
+
+    @property
+    def api_type(self):
+        return next(iter(self.infer())).api_type
+
+
+class MixedObjectFilter(compiled.CompiledObjectFilter):
+    name_class = MixedName
+
+
+@evaluator_function_cache()
+def _load_module(evaluator, path):
+    module_node = evaluator.parse(
+        path=path,
+        cache=True,
+        diff_cache=settings.fast_parser,
+        cache_path=settings.cache_directory
+    ).get_root_node()
+    # python_module = inspect.getmodule(python_object)
+    # TODO we should actually make something like this possible.
+    #evaluator.modules[python_module.__name__] = module_node
+    return module_node
+
+
+def _get_object_to_check(python_object):
+    """Check if inspect.getfile has a chance to find the source."""
+    if sys.version_info[0] > 2:
+        python_object = inspect.unwrap(python_object)
+
+    if (inspect.ismodule(python_object) or
+            inspect.isclass(python_object) or
+            inspect.ismethod(python_object) or
+            inspect.isfunction(python_object) or
+            inspect.istraceback(python_object) or
+            inspect.isframe(python_object) or
+            inspect.iscode(python_object)):
+        return python_object
+
+    try:
+        return python_object.__class__
+    except AttributeError:
+        raise TypeError  # Prevents computation of `repr` within inspect.
+
+
+def _find_syntax_node_name(evaluator, python_object):
+    original_object = python_object
+    try:
+        python_object = _get_object_to_check(python_object)
+        path = inspect.getsourcefile(python_object)
+    except TypeError:
+        # The type might not be known (e.g. class_with_dict.__weakref__)
+        return None
+    if path is None or not os.path.exists(path):
+        # The path might not exist or be e.g. <stdin>.
+        return None
+
+    file_io = FileIO(path)
+    module_node = _load_module(evaluator, path)
+
+    if inspect.ismodule(python_object):
+        # We don't need to check names for modules, because there's not really
+        # a way to write a module in a module in Python (and also __name__ can
+        # be something like ``email.utils``).
+        code_lines = get_cached_code_lines(evaluator.grammar, path)
+        return module_node, module_node, file_io, code_lines
+
+    try:
+        name_str = python_object.__name__
+    except AttributeError:
+        # Stuff like python_function.__code__.
+        return None
+
+    if name_str == '<lambda>':
+        return None  # It's too hard to find lambdas.
+
+    # Doesn't always work (e.g. os.stat_result)
+    names = module_node.get_used_names().get(name_str, [])
+    # Only functions and classes are relevant. If a name e.g. points to an
+    # import, it's probably a builtin (like collections.deque) and needs to be
+    # ignored.
+    names = [
+        n for n in names
+        if n.parent.type in ('funcdef', 'classdef') and n.parent.name == n
+    ]
+    if not names:
+        return None
+
+    try:
+        code = python_object.__code__
+        # By using the line number of a code object we make the lookup in a
+        # file pretty easy. There's still a possibility of people defining
+        # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
+        # do so we just don't care.
+        line_nr = code.co_firstlineno
+    except AttributeError:
+        pass
+    else:
+        line_names = [name for name in names if name.start_pos[0] == line_nr]
+        # There's a chance that the object is not available anymore, because
+        # the code has changed in the background.
+        if line_names:
+            names = line_names
+
+    code_lines = get_cached_code_lines(evaluator.grammar, path)
+    # It's really hard to actually get the right definition, here as a last
+    # resort we just return the last one. This chance might lead to odd
+    # completions at some points but will lead to mostly correct type
+    # inference, because people tend to define a public name in a module only
+    # once.
+    tree_node = names[-1].parent
+    if tree_node.type == 'funcdef' and get_api_type(original_object) == 'instance':
+        # If an instance is given and we're landing on a function (e.g.
+        # partial in 3.5), something is completely wrong and we should not
+        # return that.
+        return None
+    return module_node, tree_node, file_io, code_lines
+
+
+@compiled_objects_cache('mixed_cache')
+def _create(evaluator, access_handle, parent_context, *args):
+    compiled_object = create_cached_compiled_object(
+        evaluator,
+        access_handle,
+        parent_context=parent_context and parent_context.compiled_object
+    )
+
+    # TODO accessing this is bad, but it probably doesn't matter that much,
+    # because we're working with interpreteters only here.
+    python_object = access_handle.access._obj
+    result = _find_syntax_node_name(evaluator, python_object)
+    if result is None:
+        # TODO Care about generics from stuff like `[1]` and don't return like this.
+        if type(python_object) in (dict, list, tuple):
+            return ContextSet({compiled_object})
+
+        tree_contexts = to_stub(compiled_object)
+        if not tree_contexts:
+            return ContextSet({compiled_object})
+    else:
+        module_node, tree_node, file_io, code_lines = result
+
+        if parent_context is None:
+            # TODO this __name__ is probably wrong.
+            name = compiled_object.get_root_context().py__name__()
+            string_names = tuple(name.split('.'))
+            module_context = ModuleContext(
+                evaluator, module_node,
+                file_io=file_io,
+                string_names=string_names,
+                code_lines=code_lines,
+                is_package=hasattr(compiled_object, 'py__path__'),
+            )
+            if name is not None:
+                evaluator.module_cache.add(string_names, ContextSet([module_context]))
+        else:
+            if parent_context.tree_node.get_root_node() != module_node:
+                # This happens e.g. when __module__ is wrong, or when using
+                # TypeVar('foo'), where Jedi uses 'foo' as the name and
+                # Python's TypeVar('foo').__module__ will be typing.
+                return ContextSet({compiled_object})
+            module_context = parent_context.get_root_context()
+
+        tree_contexts = ContextSet({
+            module_context.create_context(
+                tree_node,
+                node_is_context=True,
+                node_is_object=True
+            )
+        })
+        if tree_node.type == 'classdef':
+            if not access_handle.is_class():
+                # Is an instance, not a class.
+                tree_contexts = tree_contexts.execute_with_values()
+
+    return ContextSet(
+        MixedObject(compiled_object, tree_context=tree_context)
+        for tree_context in tree_contexts
+    )

jedi/inference/compiled/subprocess/__init__.py

@@ -0,0 +1,406 @@
+"""
+Makes it possible to do the compiled analysis in a subprocess. This has two
+goals:
+
+1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
+   be ignored and dealt with.
+2. Make it possible to handle different Python versions as well as virtualenvs.
+"""
+
+import os
+import sys
+import subprocess
+import socket
+import errno
+import traceback
+from functools import partial
+from threading import Thread
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty  # python 2.7
+
+from jedi._compatibility import queue, is_py3, force_unicode, \
+    pickle_dump, pickle_load, GeneralizedPopen, weakref
+from jedi import debug
+from jedi.cache import memoize_method
+from jedi.inference.compiled.subprocess import functions
+from jedi.inference.compiled.access import DirectObjectAccess, AccessPath, \
+    SignatureParam
+from jedi.api.exceptions import InternalError
+
+
+_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
+
+
+def _enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+
+
+def _add_stderr_to_debug(stderr_queue):
+    while True:
+        # Try to do some error reporting from the subprocess and print its
+        # stderr contents.
+        try:
+            line = stderr_queue.get_nowait()
+            line = line.decode('utf-8', 'replace')
+            debug.warning('stderr output: %s' % line.rstrip('\n'))
+        except Empty:
+            break
+
+
+def _get_function(name):
+    return getattr(functions, name)
+
+
+def _cleanup_process(process, thread):
+    try:
+        process.kill()
+        process.wait()
+    except OSError:
+        # Raised if the process is already killed.
+        pass
+    thread.join()
+    for stream in [process.stdin, process.stdout, process.stderr]:
+        try:
+            stream.close()
+        except OSError:
+            # Raised if the stream is broken.
+            pass
+
+
+class _EvaluatorProcess(object):
+    def __init__(self, evaluator):
+        self._evaluator_weakref = weakref.ref(evaluator)
+        self._evaluator_id = id(evaluator)
+        self._handles = {}
+
+    def get_or_create_access_handle(self, obj):
+        id_ = id(obj)
+        try:
+            return self.get_access_handle(id_)
+        except KeyError:
+            access = DirectObjectAccess(self._evaluator_weakref(), obj)
+            handle = AccessHandle(self, access, id_)
+            self.set_access_handle(handle)
+            return handle
+
+    def get_access_handle(self, id_):
+        return self._handles[id_]
+
+    def set_access_handle(self, handle):
+        self._handles[handle.id] = handle
+
+
+class EvaluatorSameProcess(_EvaluatorProcess):
+    """
+    Basically just an easy access to functions.py. It has the same API
+    as EvaluatorSubprocess and does the same thing without using a subprocess.
+    This is necessary for the Interpreter process.
+    """
+    def __getattr__(self, name):
+        return partial(_get_function(name), self._evaluator_weakref())
+
+
+class EvaluatorSubprocess(_EvaluatorProcess):
+    def __init__(self, evaluator, compiled_subprocess):
+        super(EvaluatorSubprocess, self).__init__(evaluator)
+        self._used = False
+        self._compiled_subprocess = compiled_subprocess
+
+    def __getattr__(self, name):
+        func = _get_function(name)
+
+        def wrapper(*args, **kwargs):
+            self._used = True
+
+            result = self._compiled_subprocess.run(
+                self._evaluator_weakref(),
+                func,
+                args=args,
+                kwargs=kwargs,
+            )
+            # IMO it should be possible to create a hook in pickle.load to
+            # mess with the loaded objects. However it's extremely complicated
+            # to work around this so just do it with this call. ~ dave
+            return self._convert_access_handles(result)
+
+        return wrapper
+
+    def _convert_access_handles(self, obj):
+        if isinstance(obj, SignatureParam):
+            return SignatureParam(*self._convert_access_handles(tuple(obj)))
+        elif isinstance(obj, tuple):
+            return tuple(self._convert_access_handles(o) for o in obj)
+        elif isinstance(obj, list):
+            return [self._convert_access_handles(o) for o in obj]
+        elif isinstance(obj, AccessHandle):
+            try:
+                # Rewrite the access handle to one we're already having.
+                obj = self.get_access_handle(obj.id)
+            except KeyError:
+                obj.add_subprocess(self)
+                self.set_access_handle(obj)
+        elif isinstance(obj, AccessPath):
+            return AccessPath(self._convert_access_handles(obj.accesses))
+        return obj
+
+    def __del__(self):
+        if self._used and not self._compiled_subprocess.is_crashed:
+            self._compiled_subprocess.delete_evaluator(self._evaluator_id)
+
+
+class CompiledSubprocess(object):
+    is_crashed = False
+    # Start with 2, gets set after _get_info.
+    _pickle_protocol = 2
+
+    def __init__(self, executable):
+        self._executable = executable
+        self._evaluator_deletion_queue = queue.deque()
+        self._cleanup_callable = lambda: None
+
+    def __repr__(self):
+        pid = os.getpid()
+        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+            self.__class__.__name__,
+            self._executable,
+            self._pickle_protocol,
+            self.is_crashed,
+            pid,
+        )
+
+    @memoize_method
+    def _get_process(self):
+        debug.dbg('Start environment subprocess %s', self._executable)
+        parso_path = sys.modules['parso'].__file__
+        args = (
+            self._executable,
+            _MAIN_PATH,
+            os.path.dirname(os.path.dirname(parso_path)),
+            '.'.join(str(x) for x in sys.version_info[:3]),
+        )
+        process = GeneralizedPopen(
+            args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            # Use system default buffering on Python 2 to improve performance
+            # (this is already the case on Python 3).
+            bufsize=-1
+        )
+        self._stderr_queue = Queue()
+        self._stderr_thread = t = Thread(
+            target=_enqueue_output,
+            args=(process.stderr, self._stderr_queue)
+        )
+        t.daemon = True
+        t.start()
+        # Ensure the subprocess is properly cleaned up when the object
+        # is garbage collected.
+        self._cleanup_callable = weakref.finalize(self,
+                                                  _cleanup_process,
+                                                  process,
+                                                  t)
+        return process
+
+    def run(self, evaluator, function, args=(), kwargs={}):
+        # Delete old evaluators.
+        while True:
+            try:
+                evaluator_id = self._evaluator_deletion_queue.pop()
+            except IndexError:
+                break
+            else:
+                self._send(evaluator_id, None)
+
+        assert callable(function)
+        return self._send(id(evaluator), function, args, kwargs)
+
+    def get_sys_path(self):
+        return self._send(None, functions.get_sys_path, (), {})
+
+    def _kill(self):
+        self.is_crashed = True
+        self._cleanup_callable()
+
+    def _send(self, evaluator_id, function, args=(), kwargs={}):
+        if self.is_crashed:
+            raise InternalError("The subprocess %s has crashed." % self._executable)
+
+        if not is_py3:
+            # Python 2 compatibility
+            kwargs = {force_unicode(key): value for key, value in kwargs.items()}
+
+        data = evaluator_id, function, args, kwargs
+        try:
+            pickle_dump(data, self._get_process().stdin, self._pickle_protocol)
+        except (socket.error, IOError) as e:
+            # Once Python2 will be removed we can just use `BrokenPipeError`.
+            # Also, somehow in windows it returns EINVAL instead of EPIPE if
+            # the subprocess dies.
+            if e.errno not in (errno.EPIPE, errno.EINVAL):
+                # Not a broken pipe
+                raise
+            self._kill()
+            raise InternalError("The subprocess %s was killed. Maybe out of memory?"
+                                % self._executable)
+
+        try:
+            is_exception, traceback, result = pickle_load(self._get_process().stdout)
+        except EOFError as eof_error:
+            try:
+                stderr = self._get_process().stderr.read().decode('utf-8', 'replace')
+            except Exception as exc:
+                stderr = '<empty/not available (%r)>' % exc
+            self._kill()
+            _add_stderr_to_debug(self._stderr_queue)
+            raise InternalError(
+                "The subprocess %s has crashed (%r, stderr=%s)." % (
+                    self._executable,
+                    eof_error,
+                    stderr,
+                ))
+
+        _add_stderr_to_debug(self._stderr_queue)
+
+        if is_exception:
+            # Replace the attribute error message with a the traceback. It's
+            # way more informative.
+            result.args = (traceback,)
+            raise result
+        return result
+
+    def delete_evaluator(self, evaluator_id):
+        """
+        Currently we are not deleting evalutors instantly. They only get
+        deleted once the subprocess is used again. It would probably a better
+        solution to move all of this into a thread. However, the memory usage
+        of a single evaluator shouldn't be that high.
+        """
+        # With an argument - the evaluator gets deleted.
+        self._evaluator_deletion_queue.append(evaluator_id)
+
+
+class Listener(object):
+    def __init__(self, pickle_protocol):
+        self._evaluators = {}
+        # TODO refactor so we don't need to process anymore just handle
+        # controlling.
+        self._process = _EvaluatorProcess(Listener)
+        self._pickle_protocol = pickle_protocol
+
+    def _get_evaluator(self, function, evaluator_id):
+        from jedi.inference import Evaluator
+
+        try:
+            evaluator = self._evaluators[evaluator_id]
+        except KeyError:
+            from jedi.api.environment import InterpreterEnvironment
+            evaluator = Evaluator(
+                # The project is not actually needed. Nothing should need to
+                # access it.
+                project=None,
+                environment=InterpreterEnvironment()
+            )
+            self._evaluators[evaluator_id] = evaluator
+        return evaluator
+
+    def _run(self, evaluator_id, function, args, kwargs):
+        if evaluator_id is None:
+            return function(*args, **kwargs)
+        elif function is None:
+            del self._evaluators[evaluator_id]
+        else:
+            evaluator = self._get_evaluator(function, evaluator_id)
+
+            # Exchange all handles
+            args = list(args)
+            for i, arg in enumerate(args):
+                if isinstance(arg, AccessHandle):
+                    args[i] = evaluator.compiled_subprocess.get_access_handle(arg.id)
+            for key, value in kwargs.items():
+                if isinstance(value, AccessHandle):
+                    kwargs[key] = evaluator.compiled_subprocess.get_access_handle(value.id)
+
+            return function(evaluator, *args, **kwargs)
+
+    def listen(self):
+        stdout = sys.stdout
+        # Mute stdout. Nobody should actually be able to write to it,
+        # because stdout is used for IPC.
+        sys.stdout = open(os.devnull, 'w')
+        stdin = sys.stdin
+        if sys.version_info[0] > 2:
+            stdout = stdout.buffer
+            stdin = stdin.buffer
+        # Python 2 opens streams in text mode on Windows. Set stdout and stdin
+        # to binary mode.
+        elif sys.platform == 'win32':
+            import msvcrt
+            msvcrt.setmode(stdout.fileno(), os.O_BINARY)
+            msvcrt.setmode(stdin.fileno(), os.O_BINARY)
+
+        while True:
+            try:
+                payload = pickle_load(stdin)
+            except EOFError:
+                # It looks like the parent process closed.
+                # Don't make a big fuss here and just exit.
+                exit(0)
+            try:
+                result = False, None, self._run(*payload)
+            except Exception as e:
+                result = True, traceback.format_exc(), e
+
+            pickle_dump(result, stdout, self._pickle_protocol)
+
+
+class AccessHandle(object):
+    def __init__(self, subprocess, access, id_):
+        self.access = access
+        self._subprocess = subprocess
+        self.id = id_
+
+    def add_subprocess(self, subprocess):
+        self._subprocess = subprocess
+
+    def __repr__(self):
+        try:
+            detail = self.access
+        except AttributeError:
+            detail = '#' + str(self.id)
+        return '<%s of %s>' % (self.__class__.__name__, detail)
+
+    def __getstate__(self):
+        return self.id
+
+    def __setstate__(self, state):
+        self.id = state
+
+    def __getattr__(self, name):
+        if name in ('id', 'access') or name.startswith('_'):
+            raise AttributeError("Something went wrong with unpickling")
+
+        #if not is_py3: print >> sys.stderr, name
+        #print('getattr', name, file=sys.stderr)
+        return partial(self._workaround, force_unicode(name))
+
+    def _workaround(self, name, *args, **kwargs):
+        """
+        TODO Currently we're passing slice objects around. This should not
+        happen. They are also the only unhashable objects that we're passing
+        around.
+        """
+        if args and isinstance(args[0], slice):
+            return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+        return self._cached_results(name, *args, **kwargs)
+
+    @memoize_method
+    def _cached_results(self, name, *args, **kwargs):
+        #if type(self._subprocess) == EvaluatorSubprocess:
+            #print(name, args, kwargs,
+                #self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+            #)
+        return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)

jedi/inference/compiled/subprocess/__main__.py

@@ -0,0 +1,55 @@
+import os
+import sys
+
+
+def _get_paths():
+    # Get the path to jedi.
+    _d = os.path.dirname
+    _jedi_path = _d(_d(_d(_d(_d(__file__)))))
+    _parso_path = sys.argv[1]
+    # The paths are the directory that jedi and parso lie in.
+    return {'jedi': _jedi_path, 'parso': _parso_path}
+
+
+# Remove the first entry, because it's simply a directory entry that equals
+# this directory.
+del sys.path[0]
+
+if sys.version_info > (3, 4):
+    from importlib.machinery import PathFinder
+
+    class _ExactImporter(object):
+        def __init__(self, path_dct):
+            self._path_dct = path_dct
+
+        def find_module(self, fullname, path=None):
+            if path is None and fullname in self._path_dct:
+                p = self._path_dct[fullname]
+                loader = PathFinder.find_module(fullname, path=[p])
+                return loader
+            return None
+
+    # Try to import jedi/parso.
+    sys.meta_path.insert(0, _ExactImporter(_get_paths()))
+    from jedi.inference.compiled import subprocess  # NOQA
+    sys.meta_path.pop(0)
+else:
+    import imp
+
+    def load(name):
+        paths = list(_get_paths().values())
+        fp, pathname, description = imp.find_module(name, paths)
+        return imp.load_module(name, fp, pathname, description)
+
+    load('parso')
+    load('jedi')
+    from jedi.inference.compiled import subprocess  # NOQA
+
+from jedi._compatibility import highest_pickle_protocol  # noqa: E402
+
+
+# Retrieve the pickle protocol.
+host_sys_version = [int(x) for x in sys.argv[2].split('.')]
+pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
+# And finally start the client.
+subprocess.Listener(pickle_protocol=pickle_protocol).listen()

jedi/inference/compiled/subprocess/functions.py

@@ -0,0 +1,86 @@
+from __future__ import print_function
+import sys
+import os
+
+from jedi._compatibility import find_module, cast_path, force_unicode, \
+    iter_modules, all_suffixes
+from jedi.inference.compiled import access
+from jedi import parser_utils
+
+
+def get_sys_path():
+    return list(map(cast_path, sys.path))
+
+
+def load_module(evaluator, **kwargs):
+    return access.load_module(evaluator, **kwargs)
+
+
+def get_compiled_method_return(evaluator, id, attribute, *args, **kwargs):
+    handle = evaluator.compiled_subprocess.get_access_handle(id)
+    return getattr(handle.access, attribute)(*args, **kwargs)
+
+
+def create_simple_object(evaluator, obj):
+    return access.create_access_path(evaluator, obj)
+
+
+def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
+    """
+    Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
+    """
+    if sys_path is not None:
+        sys.path, temp = sys_path, sys.path
+    try:
+        return find_module(full_name=full_name, **kwargs)
+    except ImportError:
+        return None, None
+    finally:
+        if sys_path is not None:
+            sys.path = temp
+
+
+def list_module_names(evaluator, search_path):
+    return [
+        force_unicode(name)
+        for module_loader, name, is_pkg in iter_modules(search_path)
+    ]
+
+
+def get_builtin_module_names(evaluator):
+    return list(map(force_unicode, sys.builtin_module_names))
+
+
+def _test_raise_error(evaluator, exception_type):
+    """
+    Raise an error to simulate certain problems for unit tests.
+    """
+    raise exception_type
+
+
+def _test_print(evaluator, stderr=None, stdout=None):
+    """
+    Force some prints in the subprocesses. This exists for unit tests.
+    """
+    if stderr is not None:
+        print(stderr, file=sys.stderr)
+        sys.stderr.flush()
+    if stdout is not None:
+        print(stdout)
+        sys.stdout.flush()
+
+
+def _get_init_path(directory_path):
+    """
+    The __init__ file can be searched in a directory. If found return it, else
+    None.
+    """
+    for suffix in all_suffixes():
+        path = os.path.join(directory_path, '__init__' + suffix)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def safe_literal_eval(evaluator, value):
+    return parser_utils.safe_literal_eval(value)