From ab2c6bf45d0e2422161c4b0809d6d714c0ccbec3 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 24 Dec 2013 18:46:40 +0100 Subject: [PATCH] integrate memoize caches into evaluator --- jedi/cache.py | 48 +-------------------------------- jedi/docstrings.py | 4 +-- jedi/dynamic.py | 5 ++-- jedi/evaluate/__init__.py | 6 ++--- jedi/evaluate/cache.py | 48 +++++++++++++++++++++++++++++++++ jedi/evaluate/imports.py | 2 +- jedi/evaluate/representation.py | 48 ++++++++++++++++----------------- jedi/modules.py | 2 +- 8 files changed, 83 insertions(+), 80 deletions(-) create mode 100644 jedi/evaluate/cache.py diff --git a/jedi/cache.py b/jedi/cache.py index 98bfce77..918af40d 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -5,9 +5,6 @@ available: - module caching (`load_module` and `save_module`), which uses pickle and is really important to assure low load times of modules like ``numpy``. -- the popular ``memoize_default`` works like a typical memoize and returns the - default otherwise. -- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes. - ``time_cache`` can be used to cache something for just a limited time span, which can be useful if there's user interaction and the user cannot react faster than a certain time. @@ -34,9 +31,6 @@ from jedi import settings from jedi import common from jedi import debug -# memoize caches will be deleted after every action -memoize_caches = [] - time_caches = [] star_import_cache = {} @@ -60,12 +54,7 @@ def clear_caches(delete_all=False): :param delete_all: Deletes also the cache that is normally not deleted, like parser cache, which is important for faster parsing. """ - global memoize_caches, time_caches - - # memorize_caches must never be deleted, because the dicts will get lost in - # the wrappers. - for m in memoize_caches: - m.clear() + global time_caches if delete_all: time_caches = [] @@ -81,41 +70,6 @@ def clear_caches(delete_all=False): del tc[key] -def memoize_default(default, cache=memoize_caches): - """ This is a typical memoization decorator, BUT there is one difference: - To prevent recursion it sets defaults. - - Preventing recursion is in this case the much bigger use than speed. I - don't think, that there is a big speed difference, but there are many cases - where recursion could happen (think about a = b; b = a). - """ - def func(function): - memo = {} - cache.append(memo) - - def wrapper(*args, **kwargs): - key = (args, frozenset(kwargs.items())) - if key in memo: - return memo[key] - else: - memo[key] = default - rv = function(*args, **kwargs) - memo[key] = rv - return rv - return wrapper - return func - - -class CachedMetaClass(type): - """ This is basically almost the same than the decorator above, it just - caches class initializations. I haven't found any other way, so I do it - with meta classes. - """ - @memoize_default(None) - def __call__(self, *args, **kwargs): - return super(CachedMetaClass, self).__call__(*args, **kwargs) - - def time_cache(time_add_setting): """ This decorator works as follows: Call it with a setting and after that use the function with a callable that returns the key. diff --git a/jedi/docstrings.py b/jedi/docstrings.py index 02e4579d..477f999d 100644 --- a/jedi/docstrings.py +++ b/jedi/docstrings.py @@ -16,7 +16,7 @@ annotations. import re -from jedi import cache +from jedi.evaluate.cache import memoize_default from jedi.parser import Parser DOCSTRING_PARAM_PATTERNS = [ @@ -32,7 +32,7 @@ DOCSTRING_RETURN_PATTERNS = [ REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`') -@cache.memoize_default(None) +@memoize_default(None) def follow_param(param): func = param.parent_function # print func, param, param.parent_function diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 36a06b94..8bbda015 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -63,6 +63,7 @@ from jedi import common from jedi import debug from jedi.parser import fast as fast_parser from jedi.evaluate import imports +from jedi.evaluate.cache import memoize_default # This is something like the sys.path, but only for searching params. It means # that this is the order in which Jedi searches params. @@ -140,7 +141,7 @@ class ParamListener(object): self.param_possibilities.append(params) -@cache.memoize_default([]) +@memoize_default([]) def search_params(param): """ This is a dynamic search for params. If you try to complete a type: @@ -299,7 +300,7 @@ def _scan_statement(stmt, search_name, assignment_details=False): return result -@cache.memoize_default([]) +@memoize_default([]) def _check_array_additions(compare_array, module, is_list): """ Checks if a `pr.Array` has "add" statements: diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 39bfd332..53f90904 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -75,13 +75,13 @@ import itertools from jedi._compatibility import next, hasattr, is_py3k, unicode, reraise, u from jedi import common -from jedi import cache from jedi.parser import representation as pr from jedi import debug from jedi.evaluate import representation as er from jedi.evaluate import builtin from jedi.evaluate import imports from jedi.evaluate import recursion +from jedi.evaluate.cache import memoize_default from jedi import docstrings from jedi import dynamic @@ -117,7 +117,7 @@ def get_defined_names_for_position(scope, position=None, start_scope=None): class Evaluator(object): def __init__(self): - self.cache = None + self.memoize_cache = {} def get_names_of_scope(self, scope, position=None, star_search=True, include_builtin=True): @@ -474,7 +474,7 @@ class Evaluator(object): return filter_name(scope_generator) return descriptor_check(remove_statements(filter_name(scope_generator))) - @cache.memoize_default(default=()) + @memoize_default(default=(), cache_is_in_self=True) @recursion.RecursionDecorator def follow_statement(self, stmt, seek_name=None): """ diff --git a/jedi/evaluate/cache.py b/jedi/evaluate/cache.py new file mode 100644 index 00000000..3ef7ebc7 --- /dev/null +++ b/jedi/evaluate/cache.py @@ -0,0 +1,48 @@ +""" +- the popular ``memoize_default`` works like a typical memoize and returns the + default otherwise. +- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes. +""" + + +def memoize_default(default, cache_is_in_self=False): + """ This is a typical memoization decorator, BUT there is one difference: + To prevent recursion it sets defaults. + + Preventing recursion is in this case the much bigger use than speed. I + don't think, that there is a big speed difference, but there are many cases + where recursion could happen (think about a = b; b = a). + """ + def func(function): + def wrapper(obj, *args, **kwargs): + if cache_is_in_self: + cache = obj.memoize_cache + else: + cache = obj._evaluator.memoize_cache + + try: + memo = cache[function] + except KeyError: + memo = {} + cache[function] = function + + key = (args, frozenset(kwargs.items())) + if key in memo: + return memo[key] + else: + memo[key] = default + rv = function(obj, *args, **kwargs) + memo[key] = rv + return rv + return wrapper + return func + + +class CachedMetaClass(type): + """ This is basically almost the same than the decorator above, it just + caches class initializations. I haven't found any other way, so I do it + with meta classes. + """ + @memoize_default(None) + def __call__(self, *args, **kwargs): + return super(CachedMetaClass, self).__call__(*args, **kwargs) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 0221ec04..2d423848 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -200,7 +200,7 @@ class ImportPath(pr.Base): return [] scopes = [scope] - scopes += remove_star_imports(scope) + scopes += remove_star_imports(self._evaluator, scope) # follow the rest of the import (not FS -> classes, functions) if len(rest) > 1 or rest and self.is_like_search: diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index f0d1790c..4d539a75 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -16,16 +16,16 @@ import itertools from jedi._compatibility import use_metaclass, next, hasattr, unicode from jedi.parser import representation as pr -from jedi import cache from jedi import helpers from jedi import debug from jedi import common from jedi.evaluate import imports from jedi.evaluate import builtin from jedi.evaluate import recursion +from jedi.evaluate.cache import memoize_default, CachedMetaClass +from jedi.evaluate.interfaces import Iterable from jedi import docstrings from jedi import dynamic -from jedi.evaluate.interfaces import Iterable class Executable(pr.IsScope): @@ -53,7 +53,7 @@ class Executable(pr.IsScope): return self.base -class Instance(use_metaclass(cache.CachedMetaClass, Executable)): +class Instance(use_metaclass(CachedMetaClass, Executable)): """ This class is used to evaluate instances. """ @@ -72,7 +72,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): # (No var_args) used. self.is_generated = False - @cache.memoize_default(None) + @memoize_default(None) def _get_method_execution(self, func): func = InstanceElement(self, func, True) return Execution(func, self.var_args) @@ -87,7 +87,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): except IndexError: return None - @cache.memoize_default([]) + @memoize_default([]) def _get_self_attributes(self): def add_self_dot_name(name): """ @@ -144,7 +144,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): args = [obj, obj.base] if isinstance(obj, Instance) else [None, obj] return self.execute_subscope_by_name('__get__', args) - @cache.memoize_default([]) + @memoize_default([]) def get_defined_names(self): """ Get the instance vars of a class. This includes the vars of all @@ -190,7 +190,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): (type(self).__name__, self.base, len(self.var_args or [])) -class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)): +class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): """ InstanceElement is a wrapper for any object, that is used as an instance variable (e.g. self.variable or class methods). @@ -205,7 +205,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)): self.is_class_var = is_class_var @property - @cache.memoize_default(None) + @memoize_default(None) def parent(self): par = self.var.parent if isinstance(par, Class) and par == self.instance.base \ @@ -246,7 +246,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)): return "<%s of %s>" % (type(self).__name__, self.var) -class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): +class Class(use_metaclass(CachedMetaClass, pr.IsScope)): """ This class is not only important to extend `pr.Class`, it is also a important for descriptors (if the descriptor methods are evaluated or not). @@ -255,7 +255,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): self._evaluator = evaluator self.base = base - @cache.memoize_default(default=()) + @memoize_default(default=()) def get_super_classes(self): supers = [] # TODO care for mro stuff (multiple super classes). @@ -271,7 +271,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): supers += self._evaluator.find_name(builtin.Builtin.scope, 'object') return supers - @cache.memoize_default(default=()) + @memoize_default(default=()) def instance_names(self): def in_iterable(name, iterable): """ checks if the name is in the variable 'iterable'. """ @@ -293,7 +293,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): result += super_result return result - @cache.memoize_default(default=()) + @memoize_default(default=()) def get_defined_names(self): result = self.instance_names() type_cls = self._evaluator.find_name(builtin.Builtin.scope, 'type')[0] @@ -320,7 +320,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): return "" % (type(self).__name__, self.base) -class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)): +class Function(use_metaclass(CachedMetaClass, pr.IsScope)): """ Needed because of decorators. Decorators are evaluated here. """ @@ -330,7 +330,7 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)): self.base_func = func self.is_decorated = is_decorated - @cache.memoize_default(None) + @memoize_default(None) def _decorated_func(self, instance=None): """ Returns the function, that is to be executed in the end. @@ -420,7 +420,7 @@ class Execution(Executable): return [stmt] # just some arbitrary object @property - @cache.memoize_default(None) + @memoize_default(None) def _decorated(self): """Get the decorated version of the input""" base = self.base @@ -428,7 +428,7 @@ class Execution(Executable): base = base.get_decorated_func() return base - @cache.memoize_default(default=()) + @memoize_default(default=()) @recursion.ExecutionRecursionDecorator def get_return_types(self, evaluate_generator=False): """ Get the return types of a function. """ @@ -514,7 +514,7 @@ class Execution(Executable): stmts += self._evaluator.follow_statement(r) return stmts - @cache.memoize_default(default=()) + @memoize_default(default=()) def _get_params(self): """ This returns the params for an Execution/Instance and is injected as a @@ -736,7 +736,7 @@ class Execution(Executable): raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) return getattr(self._decorated, name) - @cache.memoize_default(None) + @memoize_default(None) @common.rethrow_uncaught def _scope_copy(self, scope): """ Copies a scope (e.g. if) in an execution """ @@ -752,22 +752,22 @@ class Execution(Executable): return copied @property - @cache.memoize_default([]) + @memoize_default([]) def returns(self): return self._copy_properties('returns') @property - @cache.memoize_default([]) + @memoize_default([]) def asserts(self): return self._copy_properties('asserts') @property - @cache.memoize_default([]) + @memoize_default([]) def statements(self): return self._copy_properties('statements') @property - @cache.memoize_default([]) + @memoize_default([]) def subscopes(self): return self._copy_properties('subscopes') @@ -779,7 +779,7 @@ class Execution(Executable): (type(self).__name__, self._decorated) -class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)): +class Generator(use_metaclass(CachedMetaClass, pr.Base, Iterable)): """ Cares for `yield` statements. """ def __init__(self, func, var_args): super(Generator, self).__init__() @@ -825,7 +825,7 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)): return "<%s of %s>" % (type(self).__name__, self.func) -class Array(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)): +class Array(use_metaclass(CachedMetaClass, pr.Base, Iterable)): """ Used as a mirror to pr.Array, if needed. It defines some getter methods which are important in this module. diff --git a/jedi/modules.py b/jedi/modules.py index c603f772..51e0d4fe 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -291,7 +291,7 @@ def get_sys_path(): return [p for p in sys.path if p != ""] -@cache.memoize_default([]) +#@cache.memoize_default([]) TODO add some sort of cache again. def sys_path_with_modifications(module): def execute_code(code): c = "import os; from os.path import *; result=%s"