Reenable call signature caching and move a lot of parser specific caching to the parser itself.

jedi/api/__init__.py

@@ -17,6 +17,7 @@ from jedi._compatibility import unicode
 from jedi.parser import load_grammar
 from jedi.parser import tree
 from jedi.parser.fast import FastParser
+from jedi.parser.utils import save_parser
 from jedi import debug
 from jedi import settings
 from jedi import common
@@ -131,7 +132,7 @@ class Script(object):
     def _get_module(self):
         cache.invalidate_star_import_cache(self._path)
         parser = FastParser(self._grammar, self._source, self._path)
-        cache.save_parser(self._path, parser, pickling=False)
+        save_parser(self._path, parser, pickling=False)
 
         module = self._evaluator.wrap(parser.module)
         imports.add_module(self._evaluator, unicode(module.name), module)
@@ -282,13 +283,12 @@ class Script(object):
         if call_signature_details is None:
             return []
 
-        # TODO insert caching again here.
+        with common.scale_speed_settings(settings.scale_call_signatures):
-        #with common.scale_speed_settings(settings.scale_call_signatures):
+            definitions = helpers.cache_call_signatures(
-        #    definitions = cache.cache_call_signatures(self._evaluator, stmt,
-        #                                              self._source, self._pos)
-        definitions = helpers.evaluate_goto_definition(
                 self._evaluator,
-            call_signature_details.bracket_leaf.get_previous_leaf()
+                call_signature_details.bracket_leaf,
+                self._code_lines,
+                self._pos
             )
         debug.speed('func_call followed')
 

jedi/api/helpers.py

@@ -7,6 +7,7 @@ from collections import namedtuple
 from jedi.evaluate.helpers import call_of_leaf
 from jedi import parser
 from jedi.parser import tokenize, token
+from jedi.cache import time_cache
 
 
 CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
@@ -240,3 +241,24 @@ def get_call_signature_details(module, position):
         node = node.parent
 
     return None
+
+
+@time_cache("call_signatures_validity")
+def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos):
+    """This function calculates the cache key."""
+    index = user_pos[0] - 1
+
+    before_cursor = code_lines[index][:user_pos[1]]
+    other_lines = code_lines[bracket_leaf.start_pos[0]:index]
+    whole = '\n'.join(other_lines + [before_cursor])
+    before_bracket = re.match(r'.*\(', whole, re.DOTALL)
+
+    module_path = bracket_leaf.get_parent_until().path
+    if module_path is None:
+        yield None  # Don't cache!
+    else:
+        yield (module_path, before_bracket, bracket_leaf.start_pos)
+    yield evaluate_goto_definition(
+        evaluator,
+        bracket_leaf.get_previous_leaf()
+    )

jedi/cache.py

@@ -14,36 +14,13 @@ there are global variables, which are holding the cache information. Some of
 these variables are being cleaned after every API usage.
 """
 import time
-import os
-import sys
-import json
-import hashlib
-import gc
-import inspect
-import shutil
-import re
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
 
 from jedi import settings
-from jedi import common
+from jedi.parser.utils import parser_cache
-from jedi import debug
+from jedi.parser.utils import underscore_memoization
 
 _time_caches = {}
 
-# for fast_parser, should not be deleted
-parser_cache = {}
-
-
-class ParserCacheItem(object):
-    def __init__(self, parser, change_time=None):
-        self.parser = parser
-        if change_time is None:
-            change_time = time.time()
-        self.change_time = change_time
-
 
 def clear_time_caches(delete_all=False):
     """ Jedi caches many things, that should be completed after each completion
@@ -70,11 +47,12 @@ def clear_time_caches(delete_all=False):
 
 def time_cache(time_add_setting):
     """
-    s
     This decorator works as follows: Call it with a setting and after that
     use the function with a callable that returns the key.
     But: This function is only called if the key is not available. After a
     certain amount of time (`time_add_setting`) the cache is invalid.
+
+    If the given key is None, the function will not be cached.
     """
     def _temp(key_func):
         dct = {}
@@ -99,56 +77,6 @@ def time_cache(time_add_setting):
     return _temp
 
 
-@time_cache("call_signatures_validity")
-def cache_call_signatures(evaluator, call, source, user_pos):
-    """This function calculates the cache key."""
-    index = user_pos[0] - 1
-    lines = common.splitlines(source)
-
-    before_cursor = lines[index][:user_pos[1]]
-    other_lines = lines[call.start_pos[0]:index]
-    whole = '\n'.join(other_lines + [before_cursor])
-    before_bracket = re.match(r'.*\(', whole, re.DOTALL)
-
-    module_path = call.get_parent_until().path
-    yield None if module_path is None else (module_path, before_bracket, call.start_pos)
-    yield evaluator.eval_element(call)
-
-
-def underscore_memoization(func):
-    """
-    Decorator for methods::
-
-        class A(object):
-            def x(self):
-                if self._x:
-                    self._x = 10
-                return self._x
-
-    Becomes::
-
-        class A(object):
-            @underscore_memoization
-            def x(self):
-                return 10
-
-    A now has an attribute ``_x`` written by this decorator.
-    """
-    name = '_' + func.__name__
-
-    def wrapper(self):
-        try:
-            return getattr(self, name)
-        except AttributeError:
-            result = func(self)
-            if inspect.isgenerator(result):
-                result = list(result)
-            setattr(self, name, result)
-            return result
-
-    return wrapper
-
-
 def memoize_method(method):
     """A normal memoize function."""
     def wrapper(self, *args, **kwargs):
@@ -180,6 +108,14 @@ def _invalidate_star_import_cache_module(module, only_main=False):
     else:
         del _time_caches['star_import_cache_validity'][module]
 
+        # This stuff was part of load_parser. However since we're most likely
+        # not going to use star import caching anymore, just ignore it.
+        #else:
+            # In case there is already a module cached and this module
+            # has to be reparsed, we also need to invalidate the import
+            # caches.
+        #    _invalidate_star_import_cache_module(parser_cache_item.parser.module)
+
 
 def invalidate_star_import_cache(path):
     """On success returns True."""
@@ -189,148 +125,3 @@ def invalidate_star_import_cache(path):
         pass
     else:
         _invalidate_star_import_cache_module(parser_cache_item.parser.module)
-
-
-def load_parser(path):
-    """
-    Returns the module or None, if it fails.
-    """
-    p_time = os.path.getmtime(path) if path else None
-    try:
-        parser_cache_item = parser_cache[path]
-        if not path or p_time <= parser_cache_item.change_time:
-            return parser_cache_item.parser
-        else:
-            # In case there is already a module cached and this module
-            # has to be reparsed, we also need to invalidate the import
-            # caches.
-            _invalidate_star_import_cache_module(parser_cache_item.parser.module)
-    except KeyError:
-        if settings.use_filesystem_cache:
-            return ParserPickling.load_parser(path, p_time)
-
-
-def save_parser(path, parser, pickling=True):
-    try:
-        p_time = None if path is None else os.path.getmtime(path)
-    except OSError:
-        p_time = None
-        pickling = False
-
-    item = ParserCacheItem(parser, p_time)
-    parser_cache[path] = item
-    if settings.use_filesystem_cache and pickling:
-        ParserPickling.save_parser(path, item)
-
-
-class ParserPickling(object):
-
-    version = 24
-    """
-    Version number (integer) for file system cache.
-
-    Increment this number when there are any incompatible changes in
-    parser representation classes.  For example, the following changes
-    are regarded as incompatible.
-
-    - Class name is changed.
-    - Class is moved to another module.
-    - Defined slot of the class is changed.
-    """
-
-    def __init__(self):
-        self.__index = None
-        self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
-        """
-        Short name for distinguish Python implementations and versions.
-
-        It's like `sys.implementation.cache_tag` but for Python < 3.3
-        we generate something similar.  See:
-        http://docs.python.org/3/library/sys.html#sys.implementation
-
-        .. todo:: Detect interpreter (e.g., PyPy).
-        """
-
-    def load_parser(self, path, original_changed_time):
-        try:
-            pickle_changed_time = self._index[path]
-        except KeyError:
-            return None
-        if original_changed_time is not None \
-                and pickle_changed_time < original_changed_time:
-            # the pickle file is outdated
-            return None
-
-        with open(self._get_hashed_path(path), 'rb') as f:
-            try:
-                gc.disable()
-                parser_cache_item = pickle.load(f)
-            finally:
-                gc.enable()
-
-        debug.dbg('pickle loaded: %s', path)
-        parser_cache[path] = parser_cache_item
-        return parser_cache_item.parser
-
-    def save_parser(self, path, parser_cache_item):
-        self.__index = None
-        try:
-            files = self._index
-        except KeyError:
-            files = {}
-            self._index = files
-
-        with open(self._get_hashed_path(path), 'wb') as f:
-            pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
-            files[path] = parser_cache_item.change_time
-
-        self._flush_index()
-
-    @property
-    def _index(self):
-        if self.__index is None:
-            try:
-                with open(self._get_path('index.json')) as f:
-                    data = json.load(f)
-            except (IOError, ValueError):
-                self.__index = {}
-            else:
-                # 0 means version is not defined (= always delete cache):
-                if data.get('version', 0) != self.version:
-                    self.clear_cache()
-                    self.__index = {}
-                else:
-                    self.__index = data['index']
-        return self.__index
-
-    def _remove_old_modules(self):
-        # TODO use
-        change = False
-        if change:
-            self._flush_index(self)
-            self._index  # reload index
-
-    def _flush_index(self):
-        data = {'version': self.version, 'index': self._index}
-        with open(self._get_path('index.json'), 'w') as f:
-            json.dump(data, f)
-        self.__index = None
-
-    def clear_cache(self):
-        shutil.rmtree(self._cache_directory())
-
-    def _get_hashed_path(self, path):
-        return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
-
-    def _get_path(self, file):
-        dir = self._cache_directory()
-        if not os.path.exists(dir):
-            os.makedirs(dir)
-        return os.path.join(dir, file)
-
-    def _cache_directory(self):
-        return os.path.join(settings.cache_directory, self.py_tag)
-
-
-# is a singleton
-ParserPickling = ParserPickling()

jedi/evaluate/imports.py

@@ -20,9 +20,9 @@ from itertools import chain
 from jedi._compatibility import find_module, unicode
 from jedi import common
 from jedi import debug
-from jedi import cache
 from jedi.parser import fast
 from jedi.parser import tree
+from jedi.parser.utils import save_parser, load_parser, parser_cache
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi import settings
@@ -435,7 +435,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
     def load(source):
         dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
         if path is not None and path.endswith('.py') \
-                and not dotted_path in settings.auto_import_modules:
+                and dotted_path not in settings.auto_import_modules:
             if source is None:
                 with open(path, 'rb') as f:
                     source = f.read()
@@ -443,13 +443,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
             return compiled.load_module(evaluator, path)
         p = path
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
-        cache.save_parser(path, p)
+        save_parser(path, p)
         return p.module
 
     if sys_path is None:
         sys_path = evaluator.sys_path
 
-    cached = cache.load_parser(path)
+    cached = load_parser(path)
     module = load(source) if cached is None else cached.module
     module = evaluator.wrap(module)
     return module
@@ -470,7 +470,7 @@ def get_modules_containing_name(evaluator, mods, name):
     """
     def check_python_file(path):
         try:
-            return cache.parser_cache[path].parser.module
+            return parser_cache[path].parser.module
         except KeyError:
             try:
                 return check_fs(path)

jedi/evaluate/sys_path.py

@@ -9,7 +9,7 @@ from jedi.parser import ParserWithRecovery
 from jedi.evaluate.cache import memoize_default
 from jedi import debug
 from jedi import common
-from jedi import cache
+from jedi.parser.utils import load_parser, save_parser
 
 
 def get_venv_path(venv):
@@ -211,10 +211,10 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
             return
 
         p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
-        cache.save_parser(buildout_script, p)
+        save_parser(buildout_script, p)
         return p.module
 
-    cached = cache.load_parser(buildout_script)
+    cached = load_parser(buildout_script)
     module = cached and cached.module or load(buildout_script)
     if not module:
         return

jedi/parser/fast.py

@@ -10,7 +10,7 @@ from jedi._compatibility import use_metaclass
 from jedi import settings
 from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
-from jedi import cache
+from jedi.parser.utils import underscore_memoization, parser_cache
 from jedi import debug
 from jedi.parser.tokenize import (source_tokens, NEWLINE,
                                   ENDMARKER, INDENT, DEDENT)
@@ -36,7 +36,7 @@ class FastModule(tree.Module):
             pass  # It was never used.
 
     @property
-    @cache.underscore_memoization
+    @underscore_memoization
     def used_names(self):
         return MergedNamesDict([m.used_names for m in self.modules])
 
@@ -102,7 +102,7 @@ class CachedFastParser(type):
         if not settings.fast_parser:
             return ParserWithRecovery(grammar, source, module_path)
 
-        pi = cache.parser_cache.get(module_path, None)
+        pi = parser_cache.get(module_path, None)
         if pi is None or isinstance(pi.parser, ParserWithRecovery):
             p = super(CachedFastParser, self).__call__(grammar, source, module_path)
         else:
@@ -236,7 +236,7 @@ class ParserNode(object):
             for y in n.all_sub_nodes():
                 yield y
 
-    @cache.underscore_memoization  # Should only happen once!
+    @underscore_memoization  # Should only happen once!
     def remove_last_newline(self):
         self.parser.remove_last_newline()
 

jedi/parser/tree.py

@@ -41,8 +41,8 @@ import abc
 
 from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
                                  literal_eval, use_metaclass, unicode)
-from jedi import cache
 from jedi.parser import token
+from jedi.parser.utils import underscore_memoization
 
 
 def is_node(node, *symbol_names):
@@ -788,7 +788,7 @@ class Module(Scope):
         self.path = None  # Set later.
 
     @property
-    @cache.underscore_memoization
+    @underscore_memoization
     def name(self):
         """ This is used for the goto functions. """
         if self.path is None:

test/test_api/test_api.py

@@ -7,6 +7,7 @@ from textwrap import dedent
 from jedi import api
 from jedi._compatibility import is_py3
 from pytest import raises
+from jedi.parser import utils
 
 
 def test_preload_modules():
@@ -16,16 +17,15 @@ def test_preload_modules():
         for i in modules:
             assert [i in k for k in parser_cache.keys() if k is not None]
 
-    from jedi import cache
+    temp_cache, utils.parser_cache = utils.parser_cache, {}
-    temp_cache, cache.parser_cache = cache.parser_cache, {}
+    parser_cache = utils.parser_cache
-    parser_cache = cache.parser_cache
 
     api.preload_module('sys')
     check_loaded()  # compiled (c_builtin) modules shouldn't be in the cache.
     api.preload_module('json', 'token')
     check_loaded('json', 'token')
 
-    cache.parser_cache = temp_cache
+    utils.parser_cache = temp_cache
 
 
 def test_empty_script():

test/test_cache.py

@@ -8,7 +8,7 @@ import pytest
 
 import jedi
 from jedi import settings, cache
-from jedi.cache import ParserCacheItem, ParserPickling
+from jedi.parser.utils import ParserCacheItem, ParserPickling
 
 
 ParserPicklingCls = type(ParserPickling)

test/test_parser/test_fast_parser.py

@@ -5,6 +5,7 @@ from jedi._compatibility import u
 from jedi import cache
 from jedi.parser import load_grammar
 from jedi.parser.fast import FastParser
+from jedi.parser.utils import save_parser
 
 
 def test_add_to_end():
@@ -84,7 +85,7 @@ def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0
         number_of_splits = number_parsers_used
 
     p = FastParser(load_grammar(), u(src))
-    cache.save_parser(None, p, pickling=False)
+    save_parser(None, p, pickling=False)
 
     assert src == p.module.get_code()
     assert p.number_of_splits == number_of_splits
@@ -355,7 +356,7 @@ def test_open_parentheses():
     assert p.module.get_code() == code
     assert p.number_of_splits == 2
     assert p.number_parsers_used == 2
-    cache.save_parser(None, p, pickling=False)
+    save_parser(None, p, pickling=False)
 
     # Now with a correct parser it should work perfectly well.
     check_fp('isinstance()\n' + func, 1, 2)