VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

integrate memoize caches into evaluator

Browse Source
This commit is contained in:
Dave Halter
2013-12-24 18:46:40 +01:00
parent 066056fb2b
commit ab2c6bf45d
8 changed files with 83 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
jedi/cache.py
View File

@@ -5,9 +5,6 @@ available:
- module caching (`load_module` and `save_module`), which uses pickle and is
really important to assure low load times of modules like ``numpy``.
- the popular ``memoize_default`` works like a typical memoize and returns the
default otherwise.
- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
- ``time_cache`` can be used to cache something for just a limited time span,
which can be useful if there's user interaction and the user cannot react
faster than a certain time.
@@ -34,9 +31,6 @@ from jedi import settings
from jedi import common
from jedi import debug
# memoize caches will be deleted after every action
memoize_caches = []
time_caches = []
star_import_cache = {}
@@ -60,12 +54,7 @@ def clear_caches(delete_all=False):
:param delete_all: Deletes also the cache that is normally not deleted,
like parser cache, which is important for faster parsing.
"""
global memoize_caches, time_caches
# memorize_caches must never be deleted, because the dicts will get lost in
# the wrappers.
for m in memoize_caches:
m.clear()
global time_caches
if delete_all:
time_caches = []
@@ -81,41 +70,6 @@ def clear_caches(delete_all=False):
del tc[key]
def memoize_default(default, cache=memoize_caches):
""" This is a typical memoization decorator, BUT there is one difference:
To prevent recursion it sets defaults.
Preventing recursion is in this case the much bigger use than speed. I
don't think, that there is a big speed difference, but there are many cases
where recursion could happen (think about a = b; b = a).
"""
def func(function):
memo = {}
cache.append(memo)
def wrapper(*args, **kwargs):
key = (args, frozenset(kwargs.items()))
if key in memo:
return memo[key]
else:
memo[key] = default
rv = function(*args, **kwargs)
memo[key] = rv
return rv
return wrapper
return func
class CachedMetaClass(type):
""" This is basically almost the same than the decorator above, it just
caches class initializations. I haven't found any other way, so I do it
with meta classes.
"""
@memoize_default(None)
def __call__(self, *args, **kwargs):
return super(CachedMetaClass, self).__call__(*args, **kwargs)
def time_cache(time_add_setting):
""" This decorator works as follows: Call it with a setting and after that
use the function with a callable that returns the key.

4
jedi/docstrings.py
View File

@@ -16,7 +16,7 @@ annotations.
import re
from jedi import cache
from jedi.evaluate.cache import memoize_default
from jedi.parser import Parser
DOCSTRING_PARAM_PATTERNS = [
@@ -32,7 +32,7 @@ DOCSTRING_RETURN_PATTERNS = [
REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
@cache.memoize_default(None)
@memoize_default(None)
def follow_param(param):
func = param.parent_function
# print func, param, param.parent_function

5
jedi/dynamic.py
View File

@@ -63,6 +63,7 @@ from jedi import common
from jedi import debug
from jedi.parser import fast as fast_parser
from jedi.evaluate import imports
from jedi.evaluate.cache import memoize_default
# This is something like the sys.path, but only for searching params. It means
# that this is the order in which Jedi searches params.
@@ -140,7 +141,7 @@ class ParamListener(object):
self.param_possibilities.append(params)
@cache.memoize_default([])
@memoize_default([])
def search_params(param):
"""
This is a dynamic search for params. If you try to complete a type:
@@ -299,7 +300,7 @@ def _scan_statement(stmt, search_name, assignment_details=False):
return result
@cache.memoize_default([])
@memoize_default([])
def _check_array_additions(compare_array, module, is_list):
"""
Checks if a `pr.Array` has "add" statements:

6
jedi/evaluate/__init__.py
View File

@@ -75,13 +75,13 @@ import itertools
from jedi._compatibility import next, hasattr, is_py3k, unicode, reraise, u
from jedi import common
from jedi import cache
from jedi.parser import representation as pr
from jedi import debug
from jedi.evaluate import representation as er
from jedi.evaluate import builtin
from jedi.evaluate import imports
from jedi.evaluate import recursion
from jedi.evaluate.cache import memoize_default
from jedi import docstrings
from jedi import dynamic
@@ -117,7 +117,7 @@ def get_defined_names_for_position(scope, position=None, start_scope=None):
class Evaluator(object):
def __init__(self):
self.cache = None
self.memoize_cache = {}
def get_names_of_scope(self, scope, position=None, star_search=True,
include_builtin=True):
@@ -474,7 +474,7 @@ class Evaluator(object):
return filter_name(scope_generator)
return descriptor_check(remove_statements(filter_name(scope_generator)))
@cache.memoize_default(default=())
@memoize_default(default=(), cache_is_in_self=True)
@recursion.RecursionDecorator
def follow_statement(self, stmt, seek_name=None):
"""

48
jedi/evaluate/cache.py Normal file
View File

@@ -0,0 +1,48 @@
"""
- the popular ``memoize_default`` works like a typical memoize and returns the
default otherwise.
- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
"""
def memoize_default(default, cache_is_in_self=False):
""" This is a typical memoization decorator, BUT there is one difference:
To prevent recursion it sets defaults.
Preventing recursion is in this case the much bigger use than speed. I
don't think, that there is a big speed difference, but there are many cases
where recursion could happen (think about a = b; b = a).
"""
def func(function):
def wrapper(obj, *args, **kwargs):
if cache_is_in_self:
cache = obj.memoize_cache
else:
cache = obj._evaluator.memoize_cache
try:
memo = cache[function]
except KeyError:
memo = {}
cache[function] = function
key = (args, frozenset(kwargs.items()))
if key in memo:
return memo[key]
else:
memo[key] = default
rv = function(obj, *args, **kwargs)
memo[key] = rv
return rv
return wrapper
return func
class CachedMetaClass(type):
""" This is basically almost the same than the decorator above, it just
caches class initializations. I haven't found any other way, so I do it
with meta classes.
"""
@memoize_default(None)
def __call__(self, *args, **kwargs):
return super(CachedMetaClass, self).__call__(*args, **kwargs)

2
jedi/evaluate/imports.py
View File

@@ -200,7 +200,7 @@ class ImportPath(pr.Base):
return []
scopes = [scope]
scopes += remove_star_imports(scope)
scopes += remove_star_imports(self._evaluator, scope)
# follow the rest of the import (not FS -> classes, functions)
if len(rest) > 1 or rest and self.is_like_search:

48
jedi/evaluate/representation.py
View File

@@ -16,16 +16,16 @@ import itertools
from jedi._compatibility import use_metaclass, next, hasattr, unicode
from jedi.parser import representation as pr
from jedi import cache
from jedi import helpers
from jedi import debug
from jedi import common
from jedi.evaluate import imports
from jedi.evaluate import builtin
from jedi.evaluate import recursion
from jedi.evaluate.cache import memoize_default, CachedMetaClass
from jedi.evaluate.interfaces import Iterable
from jedi import docstrings
from jedi import dynamic
from jedi.evaluate.interfaces import Iterable
class Executable(pr.IsScope):
@@ -53,7 +53,7 @@ class Executable(pr.IsScope):
return self.base
class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
class Instance(use_metaclass(CachedMetaClass, Executable)):
"""
This class is used to evaluate instances.
"""
@@ -72,7 +72,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
# (No var_args) used.
self.is_generated = False
@cache.memoize_default(None)
@memoize_default(None)
def _get_method_execution(self, func):
func = InstanceElement(self, func, True)
return Execution(func, self.var_args)
@@ -87,7 +87,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
except IndexError:
return None
@cache.memoize_default([])
@memoize_default([])
def _get_self_attributes(self):
def add_self_dot_name(name):
"""
@@ -144,7 +144,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
args = [obj, obj.base] if isinstance(obj, Instance) else [None, obj]
return self.execute_subscope_by_name('__get__', args)
@cache.memoize_default([])
@memoize_default([])
def get_defined_names(self):
"""
Get the instance vars of a class. This includes the vars of all
@@ -190,7 +190,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
(type(self).__name__, self.base, len(self.var_args or []))
class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)):
"""
InstanceElement is a wrapper for any object, that is used as an instance
variable (e.g. self.variable or class methods).
@@ -205,7 +205,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
self.is_class_var = is_class_var
@property
@cache.memoize_default(None)
@memoize_default(None)
def parent(self):
par = self.var.parent
if isinstance(par, Class) and par == self.instance.base \
@@ -246,7 +246,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
return "<%s of %s>" % (type(self).__name__, self.var)
class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
class Class(use_metaclass(CachedMetaClass, pr.IsScope)):
"""
This class is not only important to extend `pr.Class`, it is also a
important for descriptors (if the descriptor methods are evaluated or not).
@@ -255,7 +255,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
self._evaluator = evaluator
self.base = base
@cache.memoize_default(default=())
@memoize_default(default=())
def get_super_classes(self):
supers = []
# TODO care for mro stuff (multiple super classes).
@@ -271,7 +271,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
supers += self._evaluator.find_name(builtin.Builtin.scope, 'object')
return supers
@cache.memoize_default(default=())
@memoize_default(default=())
def instance_names(self):
def in_iterable(name, iterable):
""" checks if the name is in the variable 'iterable'. """
@@ -293,7 +293,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
result += super_result
return result
@cache.memoize_default(default=())
@memoize_default(default=())
def get_defined_names(self):
result = self.instance_names()
type_cls = self._evaluator.find_name(builtin.Builtin.scope, 'type')[0]
@@ -320,7 +320,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
return "<e%s of %s>" % (type(self).__name__, self.base)
class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
class Function(use_metaclass(CachedMetaClass, pr.IsScope)):
"""
Needed because of decorators. Decorators are evaluated here.
"""
@@ -330,7 +330,7 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
self.base_func = func
self.is_decorated = is_decorated
@cache.memoize_default(None)
@memoize_default(None)
def _decorated_func(self, instance=None):
"""
Returns the function, that is to be executed in the end.
@@ -420,7 +420,7 @@ class Execution(Executable):
return [stmt] # just some arbitrary object
@property
@cache.memoize_default(None)
@memoize_default(None)
def _decorated(self):
"""Get the decorated version of the input"""
base = self.base
@@ -428,7 +428,7 @@ class Execution(Executable):
base = base.get_decorated_func()
return base
@cache.memoize_default(default=())
@memoize_default(default=())
@recursion.ExecutionRecursionDecorator
def get_return_types(self, evaluate_generator=False):
""" Get the return types of a function. """
@@ -514,7 +514,7 @@ class Execution(Executable):
stmts += self._evaluator.follow_statement(r)
return stmts
@cache.memoize_default(default=())
@memoize_default(default=())
def _get_params(self):
"""
This returns the params for an Execution/Instance and is injected as a
@@ -736,7 +736,7 @@ class Execution(Executable):
raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
return getattr(self._decorated, name)
@cache.memoize_default(None)
@memoize_default(None)
@common.rethrow_uncaught
def _scope_copy(self, scope):
""" Copies a scope (e.g. if) in an execution """
@@ -752,22 +752,22 @@ class Execution(Executable):
return copied
@property
@cache.memoize_default([])
@memoize_default([])
def returns(self):
return self._copy_properties('returns')
@property
@cache.memoize_default([])
@memoize_default([])
def asserts(self):
return self._copy_properties('asserts')
@property
@cache.memoize_default([])
@memoize_default([])
def statements(self):
return self._copy_properties('statements')
@property
@cache.memoize_default([])
@memoize_default([])
def subscopes(self):
return self._copy_properties('subscopes')
@@ -779,7 +779,7 @@ class Execution(Executable):
(type(self).__name__, self._decorated)
class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
class Generator(use_metaclass(CachedMetaClass, pr.Base, Iterable)):
""" Cares for `yield` statements. """
def __init__(self, func, var_args):
super(Generator, self).__init__()
@@ -825,7 +825,7 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
return "<%s of %s>" % (type(self).__name__, self.func)
class Array(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
class Array(use_metaclass(CachedMetaClass, pr.Base, Iterable)):
"""
Used as a mirror to pr.Array, if needed. It defines some getter
methods which are important in this module.

2
jedi/modules.py
View File

@@ -291,7 +291,7 @@ def get_sys_path():
return [p for p in sys.path if p != ""]
@cache.memoize_default([])
#@cache.memoize_default([]) TODO add some sort of cache again.
def sys_path_with_modifications(module):
def execute_code(code):
c = "import os; from os.path import *; result=%s"