integrate memoize caches into evaluator

jedi/cache.py

@@ -5,9 +5,6 @@ available:
 
 - module caching (`load_module` and `save_module`), which uses pickle and is
   really important to assure low load times of modules like ``numpy``.
-- the popular ``memoize_default`` works like a typical memoize and returns the
-  default otherwise.
-- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
 - ``time_cache`` can be used to cache something for just a limited time span,
   which can be useful if there's user interaction and the user cannot react
   faster than a certain time.
@@ -34,9 +31,6 @@ from jedi import settings
 from jedi import common
 from jedi import debug
 
-# memoize caches will be deleted after every action
-memoize_caches = []
-
 time_caches = []
 
 star_import_cache = {}
@@ -60,12 +54,7 @@ def clear_caches(delete_all=False):
     :param delete_all: Deletes also the cache that is normally not deleted,
         like parser cache, which is important for faster parsing.
     """
-    global memoize_caches, time_caches
+    global time_caches
-
-    # memorize_caches must never be deleted, because the dicts will get lost in
-    # the wrappers.
-    for m in memoize_caches:
-        m.clear()
 
     if delete_all:
         time_caches = []
@@ -81,41 +70,6 @@ def clear_caches(delete_all=False):
                     del tc[key]
 
 
-def memoize_default(default, cache=memoize_caches):
-    """ This is a typical memoization decorator, BUT there is one difference:
-    To prevent recursion it sets defaults.
-
-    Preventing recursion is in this case the much bigger use than speed. I
-    don't think, that there is a big speed difference, but there are many cases
-    where recursion could happen (think about a = b; b = a).
-    """
-    def func(function):
-        memo = {}
-        cache.append(memo)
-
-        def wrapper(*args, **kwargs):
-            key = (args, frozenset(kwargs.items()))
-            if key in memo:
-                return memo[key]
-            else:
-                memo[key] = default
-                rv = function(*args, **kwargs)
-                memo[key] = rv
-                return rv
-        return wrapper
-    return func
-
-
-class CachedMetaClass(type):
-    """ This is basically almost the same than the decorator above, it just
-    caches class initializations. I haven't found any other way, so I do it
-    with meta classes.
-    """
-    @memoize_default(None)
-    def __call__(self, *args, **kwargs):
-        return super(CachedMetaClass, self).__call__(*args, **kwargs)
-
-
 def time_cache(time_add_setting):
     """ This decorator works as follows: Call it with a setting and after that
     use the function with a callable that returns the key.

jedi/docstrings.py

@@ -16,7 +16,7 @@ annotations.
 
 import re
 
-from jedi import cache
+from jedi.evaluate.cache import memoize_default
 from jedi.parser import Parser
 
 DOCSTRING_PARAM_PATTERNS = [
@@ -32,7 +32,7 @@ DOCSTRING_RETURN_PATTERNS = [
 REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
 
 
-@cache.memoize_default(None)
+@memoize_default(None)
 def follow_param(param):
     func = param.parent_function
     # print func, param, param.parent_function

jedi/dynamic.py

@@ -63,6 +63,7 @@ from jedi import common
 from jedi import debug
 from jedi.parser import fast as fast_parser
 from jedi.evaluate import imports
+from jedi.evaluate.cache import memoize_default
 
 # This is something like the sys.path, but only for searching params. It means
 # that this is the order in which Jedi searches params.
@@ -140,7 +141,7 @@ class ParamListener(object):
         self.param_possibilities.append(params)
 
 
-@cache.memoize_default([])
+@memoize_default([])
 def search_params(param):
     """
     This is a dynamic search for params. If you try to complete a type:
@@ -299,7 +300,7 @@ def _scan_statement(stmt, search_name, assignment_details=False):
     return result
 
 
-@cache.memoize_default([])
+@memoize_default([])
 def _check_array_additions(compare_array, module, is_list):
     """
     Checks if a `pr.Array` has "add" statements:

jedi/evaluate/__init__.py

@@ -75,13 +75,13 @@ import itertools
 
 from jedi._compatibility import next, hasattr, is_py3k, unicode, reraise, u
 from jedi import common
-from jedi import cache
 from jedi.parser import representation as pr
 from jedi import debug
 from jedi.evaluate import representation as er
 from jedi.evaluate import builtin
 from jedi.evaluate import imports
 from jedi.evaluate import recursion
+from jedi.evaluate.cache import memoize_default
 from jedi import docstrings
 from jedi import dynamic
 
@@ -117,7 +117,7 @@ def get_defined_names_for_position(scope, position=None, start_scope=None):
 
 class Evaluator(object):
     def __init__(self):
-        self.cache = None
+        self.memoize_cache = {}
 
     def get_names_of_scope(self, scope, position=None, star_search=True,
                            include_builtin=True):
@@ -474,7 +474,7 @@ class Evaluator(object):
             return filter_name(scope_generator)
         return descriptor_check(remove_statements(filter_name(scope_generator)))
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=(), cache_is_in_self=True)
     @recursion.RecursionDecorator
     def follow_statement(self, stmt, seek_name=None):
         """

jedi/evaluate/cache.py

@@ -0,0 +1,48 @@
+"""
+- the popular ``memoize_default`` works like a typical memoize and returns the
+  default otherwise.
+- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
+"""
+
+
+def memoize_default(default, cache_is_in_self=False):
+    """ This is a typical memoization decorator, BUT there is one difference:
+    To prevent recursion it sets defaults.
+
+    Preventing recursion is in this case the much bigger use than speed. I
+    don't think, that there is a big speed difference, but there are many cases
+    where recursion could happen (think about a = b; b = a).
+    """
+    def func(function):
+        def wrapper(obj, *args, **kwargs):
+            if cache_is_in_self:
+                cache = obj.memoize_cache
+            else:
+                cache = obj._evaluator.memoize_cache
+
+            try:
+                memo = cache[function]
+            except KeyError:
+                memo = {}
+                cache[function] = function
+
+            key = (args, frozenset(kwargs.items()))
+            if key in memo:
+                return memo[key]
+            else:
+                memo[key] = default
+                rv = function(obj, *args, **kwargs)
+                memo[key] = rv
+                return rv
+        return wrapper
+    return func
+
+
+class CachedMetaClass(type):
+    """ This is basically almost the same than the decorator above, it just
+    caches class initializations. I haven't found any other way, so I do it
+    with meta classes.
+    """
+    @memoize_default(None)
+    def __call__(self, *args, **kwargs):
+        return super(CachedMetaClass, self).__call__(*args, **kwargs)

jedi/evaluate/imports.py

@@ -200,7 +200,7 @@ class ImportPath(pr.Base):
                 return []
 
             scopes = [scope]
-            scopes += remove_star_imports(scope)
+            scopes += remove_star_imports(self._evaluator, scope)
 
             # follow the rest of the import (not FS -> classes, functions)
             if len(rest) > 1 or rest and self.is_like_search:

jedi/evaluate/representation.py

@@ -16,16 +16,16 @@ import itertools
 
 from jedi._compatibility import use_metaclass, next, hasattr, unicode
 from jedi.parser import representation as pr
-from jedi import cache
 from jedi import helpers
 from jedi import debug
 from jedi import common
 from jedi.evaluate import imports
 from jedi.evaluate import builtin
 from jedi.evaluate import recursion
+from jedi.evaluate.cache import memoize_default, CachedMetaClass
+from jedi.evaluate.interfaces import Iterable
 from jedi import docstrings
 from jedi import dynamic
-from jedi.evaluate.interfaces import Iterable
 
 
 class Executable(pr.IsScope):
@@ -53,7 +53,7 @@ class Executable(pr.IsScope):
         return self.base
 
 
-class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
+class Instance(use_metaclass(CachedMetaClass, Executable)):
     """
     This class is used to evaluate instances.
     """
@@ -72,7 +72,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
         # (No var_args) used.
         self.is_generated = False
 
-    @cache.memoize_default(None)
+    @memoize_default(None)
     def _get_method_execution(self, func):
         func = InstanceElement(self, func, True)
         return Execution(func, self.var_args)
@@ -87,7 +87,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
         except IndexError:
             return None
 
-    @cache.memoize_default([])
+    @memoize_default([])
     def _get_self_attributes(self):
         def add_self_dot_name(name):
             """
@@ -144,7 +144,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
         args = [obj, obj.base] if isinstance(obj, Instance) else [None, obj]
         return self.execute_subscope_by_name('__get__', args)
 
-    @cache.memoize_default([])
+    @memoize_default([])
     def get_defined_names(self):
         """
         Get the instance vars of a class. This includes the vars of all
@@ -190,7 +190,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
             (type(self).__name__, self.base, len(self.var_args or []))
 
 
-class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
+class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)):
     """
     InstanceElement is a wrapper for any object, that is used as an instance
     variable (e.g. self.variable or class methods).
@@ -205,7 +205,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
         self.is_class_var = is_class_var
 
     @property
-    @cache.memoize_default(None)
+    @memoize_default(None)
     def parent(self):
         par = self.var.parent
         if isinstance(par, Class) and par == self.instance.base \
@@ -246,7 +246,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
         return "<%s of %s>" % (type(self).__name__, self.var)
 
 
-class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
+class Class(use_metaclass(CachedMetaClass, pr.IsScope)):
     """
     This class is not only important to extend `pr.Class`, it is also a
     important for descriptors (if the descriptor methods are evaluated or not).
@@ -255,7 +255,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
         self._evaluator = evaluator
         self.base = base
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=())
     def get_super_classes(self):
         supers = []
         # TODO care for mro stuff (multiple super classes).
@@ -271,7 +271,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
             supers += self._evaluator.find_name(builtin.Builtin.scope, 'object')
         return supers
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=())
     def instance_names(self):
         def in_iterable(name, iterable):
             """ checks if the name is in the variable 'iterable'. """
@@ -293,7 +293,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
         result += super_result
         return result
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=())
     def get_defined_names(self):
         result = self.instance_names()
         type_cls = self._evaluator.find_name(builtin.Builtin.scope, 'type')[0]
@@ -320,7 +320,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
         return "<e%s of %s>" % (type(self).__name__, self.base)
 
 
-class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
+class Function(use_metaclass(CachedMetaClass, pr.IsScope)):
     """
     Needed because of decorators. Decorators are evaluated here.
     """
@@ -330,7 +330,7 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
         self.base_func = func
         self.is_decorated = is_decorated
 
-    @cache.memoize_default(None)
+    @memoize_default(None)
     def _decorated_func(self, instance=None):
         """
         Returns the function, that is to be executed in the end.
@@ -420,7 +420,7 @@ class Execution(Executable):
                 return [stmt]  # just some arbitrary object
 
     @property
-    @cache.memoize_default(None)
+    @memoize_default(None)
     def _decorated(self):
         """Get the decorated version of the input"""
         base = self.base
@@ -428,7 +428,7 @@ class Execution(Executable):
             base = base.get_decorated_func()
         return base
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=())
     @recursion.ExecutionRecursionDecorator
     def get_return_types(self, evaluate_generator=False):
         """ Get the return types of a function. """
@@ -514,7 +514,7 @@ class Execution(Executable):
                     stmts += self._evaluator.follow_statement(r)
             return stmts
 
-    @cache.memoize_default(default=())
+    @memoize_default(default=())
     def _get_params(self):
         """
         This returns the params for an Execution/Instance and is injected as a
@@ -736,7 +736,7 @@ class Execution(Executable):
             raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
         return getattr(self._decorated, name)
 
-    @cache.memoize_default(None)
+    @memoize_default(None)
     @common.rethrow_uncaught
     def _scope_copy(self, scope):
         """ Copies a scope (e.g. if) in an execution """
@@ -752,22 +752,22 @@ class Execution(Executable):
             return copied
 
     @property
-    @cache.memoize_default([])
+    @memoize_default([])
     def returns(self):
         return self._copy_properties('returns')
 
     @property
-    @cache.memoize_default([])
+    @memoize_default([])
     def asserts(self):
         return self._copy_properties('asserts')
 
     @property
-    @cache.memoize_default([])
+    @memoize_default([])
     def statements(self):
         return self._copy_properties('statements')
 
     @property
-    @cache.memoize_default([])
+    @memoize_default([])
     def subscopes(self):
         return self._copy_properties('subscopes')
 
@@ -779,7 +779,7 @@ class Execution(Executable):
             (type(self).__name__, self._decorated)
 
 
-class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
+class Generator(use_metaclass(CachedMetaClass, pr.Base, Iterable)):
     """ Cares for `yield` statements. """
     def __init__(self, func, var_args):
         super(Generator, self).__init__()
@@ -825,7 +825,7 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
         return "<%s of %s>" % (type(self).__name__, self.func)
 
 
-class Array(use_metaclass(cache.CachedMetaClass, pr.Base, Iterable)):
+class Array(use_metaclass(CachedMetaClass, pr.Base, Iterable)):
     """
     Used as a mirror to pr.Array, if needed. It defines some getter
     methods which are important in this module.

jedi/modules.py

@@ -291,7 +291,7 @@ def get_sys_path():
     return [p for p in sys.path if p != ""]
 
 
-@cache.memoize_default([])
+#@cache.memoize_default([]) TODO add some sort of cache again.
 def sys_path_with_modifications(module):
     def execute_code(code):
         c = "import os; from os.path import *; result=%s"