Merge remote-tracking branch 'upstream/dev' into bugfix/performances_degradation

# Conflicts:
#	jedi/evaluate/compiled/fake.py
This commit is contained in:
ColinDuquesnoy
2016-07-21 10:41:11 +02:00
124 changed files with 6536 additions and 2897 deletions
+169 -161
View File
@@ -41,34 +41,30 @@ class CompiledObject(Base):
path = None # modules have this attribute - set it to None.
used_names = {} # To be consistent with modules.
def __init__(self, obj, parent=None):
def __init__(self, evaluator, obj, parent=None):
self._evaluator = evaluator
self.obj = obj
self.parent = parent
@property
def py__call__(self):
def actual(evaluator, params):
if inspect.isclass(self.obj):
from jedi.evaluate.representation import Instance
return [Instance(evaluator, self, params)]
else:
return list(self._execute_function(evaluator, params))
# Might raise an AttributeError, which is intentional.
self.obj.__call__
return actual
@CheckAttribute
def py__call__(self, params):
if inspect.isclass(self.obj):
from jedi.evaluate.representation import Instance
return set([Instance(self._evaluator, self, params)])
else:
return set(self._execute_function(params))
@CheckAttribute
def py__class__(self, evaluator):
return CompiledObject(self.obj.__class__, parent=self.parent)
def py__class__(self):
return create(self._evaluator, self.obj.__class__)
@CheckAttribute
def py__mro__(self, evaluator):
return tuple(create(evaluator, cls, self.parent) for cls in self.obj.__mro__)
def py__mro__(self):
return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__)
@CheckAttribute
def py__bases__(self, evaluator):
return tuple(create(evaluator, cls) for cls in self.obj.__bases__)
def py__bases__(self):
return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__)
def py__bool__(self):
return bool(self.obj)
@@ -87,7 +83,7 @@ class CompiledObject(Base):
def params(self):
params_str, ret = self._parse_function_doc()
tokens = params_str.split(',')
if inspect.ismethoddescriptor(self._cls().obj):
if inspect.ismethoddescriptor(self.obj):
tokens.insert(0, 'self')
params = []
for p in tokens:
@@ -108,22 +104,21 @@ class CompiledObject(Base):
return _parse_function_doc(self.doc)
def api_type(self):
if fake.is_class_instance(self.obj):
return 'instance'
cls = self._cls().obj
if inspect.isclass(cls):
obj = self.obj
if inspect.isclass(obj):
return 'class'
elif inspect.ismodule(cls):
elif inspect.ismodule(obj):
return 'module'
elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \
or inspect.ismethoddescriptor(cls):
elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
return 'function'
# Everything else...
return 'instance'
@property
def type(self):
"""Imitate the tree.Node.type values."""
cls = self._cls().obj
cls = self._get_class()
if inspect.isclass(cls):
return 'classdef'
elif inspect.ismodule(cls):
@@ -134,17 +129,24 @@ class CompiledObject(Base):
@underscore_memoization
def _cls(self):
"""
We used to limit the lookups for instantiated objects like list(), but
this is not the case anymore. Python itself
"""
# Ensures that a CompiledObject is returned that is not an instance (like list)
if fake.is_class_instance(self.obj):
try:
c = self.obj.__class__
except AttributeError:
# happens with numpy.core.umath._UFUNC_API (you get it
# automatically by doing `import numpy`.
c = type(None)
return CompiledObject(c, self.parent)
return self
def _get_class(self):
if not fake.is_class_instance(self.obj):
return self.obj
try:
return self.obj.__class__
except AttributeError:
# happens with numpy.core.umath._UFUNC_API (you get it
# automatically by doing `import numpy`.
return type
@property
def names_dict(self):
# For compatibility with `representation.Class`.
@@ -159,64 +161,55 @@ class CompiledObject(Base):
search_global shouldn't change the fact that there's one dict, this way
there's only one `object`.
"""
return [LazyNamesDict(self._cls(), is_instance)]
return [LazyNamesDict(self._evaluator, self, is_instance)]
def get_subscope_by_name(self, name):
if name in dir(self._cls().obj):
return CompiledName(self._cls(), name).parent
if name in dir(self.obj):
return CompiledName(self._evaluator, self, name).parent
else:
raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
def get_index_types(self, evaluator, index_array=()):
# If the object doesn't have `__getitem__`, just raise the
# AttributeError.
if not hasattr(self.obj, '__getitem__'):
debug.warning('Tried to call __getitem__ on non-iterable.')
return []
@CheckAttribute
def py__getitem__(self, index):
if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
# Get rid of side effects, we won't call custom `__getitem__`s.
return []
return set()
result = []
from jedi.evaluate.iterable import create_indexes_or_slices
for typ in create_indexes_or_slices(evaluator, index_array):
index = None
try:
index = typ.obj
new = self.obj[index]
except (KeyError, IndexError, TypeError, AttributeError):
# Just try, we don't care if it fails, except for slices.
if isinstance(index, slice):
result.append(self)
else:
result.append(CompiledObject(new))
if not result:
try:
for obj in self.obj:
result.append(CompiledObject(obj))
except TypeError:
pass # self.obj maynot have an __iter__ method.
return result
return set([create(self._evaluator, self.obj[index])])
@CheckAttribute
def py__iter__(self):
if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
# Get rid of side effects, we won't call custom `__getitem__`s.
return
for part in self.obj:
yield set([create(self._evaluator, part)])
@property
def name(self):
# might not exist sometimes (raises AttributeError)
return FakeName(self._cls().obj.__name__, self)
try:
name = self._get_class().__name__
except AttributeError:
name = repr(self.obj)
return FakeName(name, self)
def _execute_function(self, evaluator, params):
def _execute_function(self, params):
if self.type != 'funcdef':
return
for name in self._parse_function_doc()[1].split():
try:
bltn_obj = _create_from_name(builtin, builtin, name)
bltn_obj = getattr(_builtins, name)
except AttributeError:
continue
else:
if isinstance(bltn_obj, CompiledObject) and bltn_obj.obj is None:
# We want everything except None.
if bltn_obj is None:
# We want to evaluate everything except None.
# TODO do we?
continue
for result in evaluator.execute(bltn_obj, params):
bltn_obj = create(self._evaluator, bltn_obj)
for result in self._evaluator.execute(bltn_obj, params):
yield result
@property
@@ -228,7 +221,7 @@ class CompiledObject(Base):
"""
module = self.get_parent_until()
faked_subscopes = []
for name in dir(self._cls().obj):
for name in dir(self.obj):
f = fake.get_faked(module.obj, self.obj, name)
if f:
f.parent = self
@@ -245,11 +238,42 @@ class CompiledObject(Base):
return [] # Builtins don't have imports
class CompiledName(FakeName):
def __init__(self, evaluator, compiled_obj, name):
super(CompiledName, self).__init__(name)
self._evaluator = evaluator
self._compiled_obj = compiled_obj
self.name = name
def __repr__(self):
try:
name = self._compiled_obj.name # __name__ is not defined all the time
except AttributeError:
name = None
return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
def is_definition(self):
return True
@property
@underscore_memoization
def parent(self):
module = self._compiled_obj.get_parent_until()
return _create_from_name(self._evaluator, module, self._compiled_obj, self.name)
@parent.setter
def parent(self, value):
pass # Just ignore this, FakeName tries to overwrite the parent attribute.
class LazyNamesDict(object):
"""
A names_dict instance for compiled objects, resembles the parser.tree.
"""
def __init__(self, compiled_obj, is_instance):
name_class = CompiledName
def __init__(self, evaluator, compiled_obj, is_instance=False):
self._evaluator = evaluator
self._compiled_obj = compiled_obj
self._is_instance = is_instance
@@ -262,7 +286,12 @@ class LazyNamesDict(object):
getattr(self._compiled_obj.obj, name)
except AttributeError:
raise KeyError('%s in %s not found.' % (name, self._compiled_obj))
return [CompiledName(self._compiled_obj, name)]
except Exception:
# This is a bit ugly. We're basically returning this to make
# lookups possible without having the actual attribute. However
# this makes proper completion possible.
return [FakeName(name, create(self._evaluator, None), is_definition=True)]
return [self.name_class(self._evaluator, self._compiled_obj, name)]
def values(self):
obj = self._compiled_obj.obj
@@ -275,39 +304,13 @@ class LazyNamesDict(object):
# The dir function can be wrong.
pass
# dir doesn't include the type names.
if not inspect.ismodule(obj) and obj != type and not self._is_instance:
values += _type_names_dict.values()
is_instance = self._is_instance or fake.is_class_instance(obj)
# ``dir`` doesn't include the type names.
if not inspect.ismodule(obj) and obj != type and not is_instance:
values += create(self._evaluator, type).names_dict.values()
return values
class CompiledName(FakeName):
def __init__(self, obj, name):
super(CompiledName, self).__init__(name)
self._obj = obj
self.name = name
def __repr__(self):
try:
name = self._obj.name # __name__ is not defined all the time
except AttributeError:
name = None
return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
def is_definition(self):
return True
@property
@underscore_memoization
def parent(self):
module = self._obj.get_parent_until()
return _create_from_name(module, self._obj, self.name)
@parent.setter
def parent(self, value):
pass # Just ignore this, FakeName tries to overwrite the parent attribute.
def dotted_from_fs_path(fs_path, sys_path):
"""
Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`. I.e.
@@ -369,7 +372,7 @@ def load_module(evaluator, path=None, name=None):
# complicated import structure of Python.
module = sys.modules[dotted_path]
return CompiledObject(module)
return create(evaluator, module)
docstr_defaults = {
@@ -441,19 +444,7 @@ def _parse_function_doc(doc):
return param_str, ret
class Builtin(CompiledObject):
@memoize_method
def get_by_name(self, name):
return self.names_dict[name][0].parent
def _a_generator(foo):
"""Used to have an object to return for generators."""
yield 42
yield foo
def _create_from_name(module, parent, name):
def _create_from_name(evaluator, module, parent, name):
faked = fake.get_faked(module.obj, parent.obj, name)
# only functions are necessary.
if faked is not None:
@@ -467,61 +458,78 @@ def _create_from_name(module, parent, name):
# PyQt4.QtGui.QStyleOptionComboBox.currentText
# -> just set it to None
obj = None
return CompiledObject(obj, parent)
return create(evaluator, obj, parent)
builtin = Builtin(_builtins)
magic_function_class = CompiledObject(type(load_module), parent=builtin)
generator_obj = CompiledObject(_a_generator(1.0))
_type_names_dict = builtin.get_by_name('type').names_dict
none_obj = builtin.get_by_name('None')
false_obj = builtin.get_by_name('False')
true_obj = builtin.get_by_name('True')
object_obj = builtin.get_by_name('object')
def builtin_from_name(evaluator, string):
bltn_obj = getattr(_builtins, string)
return create(evaluator, bltn_obj)
def keyword_from_value(obj):
if obj is None:
return none_obj
elif obj is False:
return false_obj
elif obj is True:
return true_obj
else:
raise NotImplementedError
def _a_generator(foo):
"""Used to have an object to return for generators."""
yield 42
yield foo
def compiled_objects_cache(func):
def wrapper(evaluator, obj, parent=builtin, module=None):
# Do a very cheap form of caching here.
key = id(obj), id(parent), id(module)
try:
return evaluator.compiled_cache[key][0]
except KeyError:
result = func(evaluator, obj, parent, module)
# Need to cache all of them, otherwise the id could be overwritten.
evaluator.compiled_cache[key] = result, obj, parent, module
return result
return wrapper
_SPECIAL_OBJECTS = {
'FUNCTION_CLASS': type(load_module),
'MODULE_CLASS': type(os),
'GENERATOR_OBJECT': _a_generator(1.0),
'BUILTINS': _builtins,
}
@compiled_objects_cache
def create(evaluator, obj, parent=builtin, module=None):
def get_special_object(evaluator, identifier):
obj = _SPECIAL_OBJECTS[identifier]
return create(evaluator, obj, parent=create(evaluator, _builtins))
def compiled_objects_cache(attribute_name):
def decorator(func):
"""
This decorator caches just the ids, oopposed to caching the object itself.
Caching the id has the advantage that an object doesn't need to be
hashable.
"""
def wrapper(evaluator, obj, parent=None, module=None):
cache = getattr(evaluator, attribute_name)
# Do a very cheap form of caching here.
key = id(obj), id(parent)
try:
return cache[key][0]
except KeyError:
# TODO this whole decorator looks way too ugly and this if
# doesn't make it better. Find a more generic solution.
if parent or module:
result = func(evaluator, obj, parent, module)
else:
result = func(evaluator, obj)
# Need to cache all of them, otherwise the id could be overwritten.
cache[key] = result, obj, parent, module
return result
return wrapper
return decorator
@compiled_objects_cache('compiled_cache')
def create(evaluator, obj, parent=None, module=None):
"""
A very weird interface class to this module. The more options provided the
more acurate loading compiled objects is.
"""
if inspect.ismodule(obj):
if parent is not None:
# Modules don't have parents, be careful with caching: recurse.
return create(evaluator, obj)
else:
if parent is None and obj != _builtins:
return create(evaluator, obj, create(evaluator, _builtins))
if not inspect.ismodule(obj):
faked = fake.get_faked(module and module.obj, obj)
if faked is not None:
faked.parent = parent
return faked
try:
if parent == builtin and obj.__module__ in ('builtins', '__builtin__'):
return builtin.get_by_name(obj.__name__)
except AttributeError:
pass
return CompiledObject(obj, parent)
return CompiledObject(evaluator, obj, parent)
+64 -17
View File
@@ -6,16 +6,44 @@ mixing in Python code, the autocompletion should work much better for builtins.
import os
import inspect
import types
from jedi._compatibility import is_py3, builtins, unicode
from jedi._compatibility import is_py3, builtins, unicode, is_py34
from jedi.cache import memoize_function
from jedi.parser import Parser, load_grammar
from jedi.parser import ParserWithRecovery, load_grammar
from jedi.parser import tree as pt
from jedi.evaluate.helpers import FakeName
modules = {}
MethodDescriptorType = type(str.replace)
# These are not considered classes and access is granted even though they have
# a __class__ attribute.
NOT_CLASS_TYPES = (
types.BuiltinFunctionType,
types.CodeType,
types.FrameType,
types.FunctionType,
types.GeneratorType,
types.GetSetDescriptorType,
types.LambdaType,
types.MemberDescriptorType,
types.MethodType,
types.ModuleType,
types.TracebackType,
MethodDescriptorType
)
if is_py3:
NOT_CLASS_TYPES += (
types.MappingProxyType,
types.SimpleNamespace
)
if is_py34:
NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
def _load_faked_module(module):
module_name = module.__name__
if module_name == '__builtin__' and not is_py3:
@@ -31,8 +59,8 @@ def _load_faked_module(module):
except IOError:
modules[module_name] = None
return
grammar = load_grammar('grammar3.4')
module = Parser(grammar, unicode(source), module_name).module
grammar = load_grammar(version='3.4')
module = ParserWithRecovery(grammar, unicode(source), module_name).module
modules[module_name] = module
if module_name == 'builtins' and not is_py3:
@@ -65,7 +93,15 @@ def get_module(obj):
# Unfortunately in some cases like `int` there's no __module__
return builtins
else:
return __import__(imp_plz)
if imp_plz is None:
# Happens for example in `(_ for _ in []).send.__module__`.
return builtins
else:
try:
return __import__(imp_plz)
except ImportError:
# __module__ can be something arbitrary that doesn't exist.
return builtins
def _faked(module, obj, name):
@@ -75,7 +111,7 @@ def _faked(module, obj, name):
faked_mod = _load_faked_module(module)
if faked_mod is None:
return
return None
# Having the module as a `parser.representation.module`, we need to scan
# for methods.
@@ -84,23 +120,32 @@ def _faked(module, obj, name):
return search_scope(faked_mod, obj.__name__)
elif not inspect.isclass(obj):
# object is a method or descriptor
cls = search_scope(faked_mod, obj.__objclass__.__name__)
if cls is None:
return
return search_scope(cls, obj.__name__)
try:
objclass = obj.__objclass__
except AttributeError:
return None
else:
cls = search_scope(faked_mod, objclass.__name__)
if cls is None:
return None
return search_scope(cls, obj.__name__)
else:
if obj == module:
return search_scope(faked_mod, name)
else:
cls = search_scope(faked_mod, obj.__name__)
try:
cls_name = obj.__name__
except AttributeError:
return None
cls = search_scope(faked_mod, cls_name)
if cls is None:
return
return None
return search_scope(cls, name)
@memoize_function
def get_faked(module, obj, name=None):
obj = obj.__class__ if is_class_instance(obj) else obj
obj = type(obj) if is_class_instance(obj) else obj
result = _faked(module, obj, name)
if result is None or isinstance(result, pt.Class):
# We're not interested in classes. What we want is functions.
@@ -119,7 +164,9 @@ def get_faked(module, obj, name=None):
def is_class_instance(obj):
"""Like inspect.* methods."""
return not (inspect.isclass(obj) or inspect.ismodule(obj)
or inspect.isbuiltin(obj) or inspect.ismethod(obj)
or inspect.ismethoddescriptor(obj) or inspect.iscode(obj)
or inspect.isgenerator(obj))
try:
cls = obj.__class__
except AttributeError:
return False
else:
return cls != type and not issubclass(cls, NOT_CLASS_TYPES)
+19 -1
View File
@@ -124,7 +124,7 @@ class list():
return self.__iterable[y]
def pop(self):
return self.__iterable[-1]
return self.__iterable[int()]
class tuple():
@@ -202,11 +202,29 @@ class dict():
except KeyError:
return d
def values(self):
return self.__elements.values()
def setdefault(self, k, d):
# TODO maybe also return the content
return d
class enumerate():
def __init__(self, sequence, start=0):
self.__sequence = sequence
def __iter__(self):
for i in self.__sequence:
yield 1, i
def __next__(self):
return next(self.__iter__())
def next(self):
return next(self.__iter__())
class reversed():
def __init__(self, sequence):
self.__sequence = sequence
+158
View File
@@ -0,0 +1,158 @@
"""
Used only for REPL Completion.
"""
import inspect
import os
from jedi import common
from jedi.parser.fast import FastParser
from jedi.evaluate import compiled
from jedi.cache import underscore_memoization
class MixedObject(object):
"""
A ``MixedObject`` is used in two ways:
1. It uses the default logic of ``parser.tree`` objects,
2. except for getattr calls. The names dicts are generated in a fashion
like ``CompiledObject``.
This combined logic makes it possible to provide more powerful REPL
completion. It allows side effects that are not noticable with the default
parser structure to still be completeable.
The biggest difference from CompiledObject to MixedObject is that we are
generally dealing with Python code and not with C code. This will generate
fewer special cases, because we in Python you don't have the same freedoms
to modify the runtime.
"""
def __init__(self, evaluator, obj, node_name):
self._evaluator = evaluator
self.obj = obj
self.node_name = node_name
self.definition = node_name.get_definition()
@property
def names_dict(self):
return LazyMixedNamesDict(self._evaluator, self)
def names_dicts(self, search_global):
# TODO is this needed?
assert search_global is False
return [self.names_dict]
def api_type(self):
mappings = {
'expr_stmt': 'statement',
'classdef': 'class',
'funcdef': 'function',
'file_input': 'module',
}
return mappings[self.definition.type]
def __repr__(self):
return '<%s: %s>' % (type(self).__name__, repr(self.obj))
def __getattr__(self, name):
return getattr(self.definition, name)
class MixedName(compiled.CompiledName):
"""
The ``CompiledName._compiled_object`` is our MixedObject.
"""
@property
@underscore_memoization
def parent(self):
return create(self._evaluator, getattr(self._compiled_obj.obj, self.name))
@parent.setter
def parent(self, value):
pass # Just ignore this, Name tries to overwrite the parent attribute.
@property
def start_pos(self):
if isinstance(self.parent, MixedObject):
return self.parent.node_name.start_pos
# This means a start_pos that doesn't exist (compiled objects).
return (0, 0)
class LazyMixedNamesDict(compiled.LazyNamesDict):
name_class = MixedName
def parse(grammar, path):
with open(path) as f:
source = f.read()
source = common.source_to_unicode(source)
return FastParser(grammar, source, path)
def _load_module(evaluator, path, python_object):
module = parse(evaluator.grammar, path).module
python_module = inspect.getmodule(python_object)
evaluator.modules[python_module.__name__] = module
return module
def find_syntax_node_name(evaluator, python_object):
try:
path = inspect.getsourcefile(python_object)
except TypeError:
# The type might not be known (e.g. class_with_dict.__weakref__)
return None
if path is None or not os.path.exists(path):
# The path might not exist or be e.g. <stdin>.
return None
module = _load_module(evaluator, path, python_object)
if inspect.ismodule(python_object):
# We don't need to check names for modules, because there's not really
# a way to write a module in a module in Python (and also __name__ can
# be something like ``email.utils``).
return module
try:
names = module.used_names[python_object.__name__]
except NameError:
return None
names = [n for n in names if n.is_definition()]
try:
code = python_object.__code__
# By using the line number of a code object we make the lookup in a
# file pretty easy. There's still a possibility of people defining
# stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
# do so we just don't care.
line_nr = code.co_firstlineno
except AttributeError:
pass
else:
line_names = [name for name in names if name.start_pos[0] == line_nr]
# There's a chance that the object is not available anymore, because
# the code has changed in the background.
if line_names:
return line_names[-1]
# It's really hard to actually get the right definition, here as a last
# resort we just return the last one. This chance might lead to odd
# completions at some points but will lead to mostly correct type
# inference, because people tend to define a public name in a module only
# once.
return names[-1]
@compiled.compiled_objects_cache('mixed_cache')
def create(evaluator, obj):
name = find_syntax_node_name(evaluator, obj)
if name is None:
return compiled.create(evaluator, obj)
else:
return MixedObject(evaluator, obj, name)