basic pickle implementation #102

jedi/_compatibility.py

@@ -172,3 +172,9 @@ try:
     from functools import reduce
 except ImportError:
     reduce = reduce
+
+try:
+    import json
+except ImportError:
+    # python 2.5
+    import simplejson as json

jedi/api.py

@@ -376,7 +376,7 @@ class Script(object):
                 return None, 0
 
             try:
-                timestamp, parser = cache.module_cache[self.source_path]
+                parser = cache.module_cache[self.source_path].parser
             except KeyError:
                 return None, 0
             part_parser = self._module.get_part_parser()

jedi/builtin.py

@@ -48,9 +48,8 @@ class CachedModule(object):
     def parser(self):
         """ get the parser lazy """
         if self._parser is None:
-            self._parser = cache.load_module(self.path, self.name)
+            self._parser = cache.load_module(self.path, self.name) \
-            if self._parser is None:
+                                or self._load_module()
-                self._load_module()
         return self._parser
 
     def _get_source(self):
@@ -59,8 +58,9 @@ class CachedModule(object):
     def _load_module(self):
         source = self._get_source()
         p = self.path or self.name
-        self._parser = fast_parser.FastParser(source, p)
+        p = fast_parser.FastParser(source, p)
-        cache.save_module(self.path, self.name, self._parser)
+        cache.save_module(self.path, self.name, p)
+        return p
 
 
 class Parser(CachedModule):

jedi/cache.py

@@ -1,7 +1,11 @@
 import time
 import os
+import sys
+import pickle
 
+from _compatibility import json
 import settings
+import debug
 
 # memoize caches will be deleted after every action
 memoize_caches = []
@@ -16,6 +20,14 @@ parser_cache = {}
 module_cache = {}
 
 
+class ModuleCacheItem(object):
+    def __init__(self, parser, change_time=None):
+        self.parser = parser
+        if change_time is None:
+            change_time = time.time()
+        self.change_time = change_time
+
+
 def clear_caches(delete_all=False):
     """ Jedi caches many things, that should be completed after each completion
     finishes.
@@ -156,32 +168,100 @@ def load_module(path, name):
     if path is None and name is None:
         return None
 
+    tim = os.path.getmtime(path) if path else None
     try:
-        timestamp, parser = module_cache[path or name]
+        module_cache_item = module_cache[path or name]
-        if not path or os.path.getmtime(path) <= timestamp:
+        if not path or tim <= module_cache_item.change_time:
-            return parser
+            return module_cache_item.parser
         else:
             # In case there is already a module cached and this module
             # has to be reparsed, we also need to invalidate the import
             # caches.
-            invalidate_star_import_cache(parser.module)
+            invalidate_star_import_cache(module_cache_item.parser.module)
-            return None
     except KeyError:
-        return load_pickle_module(path or name)
+        if settings.use_filesystem_cache:
+            return ModulePickling.load_module(path or name, tim)
 
 
-def save_module(path, name, parser):
+def save_module(path, name, parser, pickling=True):
     if path is None and name is None:
         return
 
     p_time = None if not path else os.path.getmtime(path)
-    module_cache[path or name] = p_time, parser
+    item = ModuleCacheItem(parser, p_time)
-    save_pickle_module(path or name)
+    module_cache[path or name] = item
+    if settings.use_filesystem_cache and pickling:
+        ModulePickling.save_module(path or name, item)
 
 
-def load_pickle_module(path):
+class _ModulePickling(object):
-    return None
+    def __init__(self):
+        self.__index = None
+        self.py_version = '%s.%s' % sys.version_info[:2]
+
+    def load_module(self, path, original_changed_time):
+        try:
+            pickle_changed_time = self._index[self.py_version][path]
+        except KeyError:
+            return None
+        if original_changed_time is not None \
+                and pickle_changed_time < original_changed_time:
+            # the pickle file is outdated
+            return None
+
+        with open(self._get_hashed_path(path)) as f:
+            module_cache_item = pickle.load(f)
+
+        parser = module_cache_item.parser
+        debug.dbg('pickle loaded', path)
+        parser_cache[path] = parser
+        module_cache[path] = module_cache_item
+        return parser
+
+    def save_module(self, path, module_cache_item):
+        try:
+            files = self._index[self.py_version]
+        except KeyError:
+            files = {}
+            self._index[self.py_version] = files
+
+        with open(self._get_hashed_path(path), 'w') as f:
+            pickle.dump(module_cache_item, f, pickle.HIGHEST_PROTOCOL)
+            files[path] = module_cache_item.change_time
+
+        self._flush_index()
+
+    @property
+    def _index(self):
+        if self.__index is None:
+            try:
+                with open(self._get_path('index.json')) as f:
+                    self.__index = json.load(f)
+            except IOError:
+                self.__index = {}
+        return self.__index
+
+    def _remove_old_modules(self):
+        # TODO use
+        change = False
+        if change:
+            self._flush_index(self)
+            self._index  # reload index
+
+    def _flush_index(self):
+        with open(self._get_path('index.json'), 'w') as f:
+            json.dump(self._index, f)
+        self.__index = None
+
+    def _get_hashed_path(self, path):
+        return self._get_path('%s_%s.pkl' % (self.py_version, hash(path)))
+
+    def _get_path(self, file):
+        dir = settings.cache_directory
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+        return dir + os.path.sep + file
 
 
-def save_pickle_module(path):
+# is a singleton
-    pass
+ModulePickling = _ModulePickling()

jedi/dynamic.py

@@ -32,7 +32,7 @@ def get_directory_modules_for_name(mods, name):
     """
     def check_python_file(path):
         try:
-            return cache.module_cache[path][1].module
+            return cache.module_cache[path].parser.module
         except KeyError:
             try:
                 return check_fs(path)

jedi/modules.py

@@ -6,7 +6,6 @@ import re
 import tokenize
 import sys
 import os
-import time
 
 import cache
 import parsing
@@ -65,10 +64,8 @@ class ModuleWithCursor(Module):
         """ get the parser lazy """
         if not self._parser:
             try:
-                ts, parser = cache.module_cache[self.path]
+                parser = cache.module_cache[self.path].parser
                 cache.invalidate_star_import_cache(parser.module)
-
-                del cache.module_cache[self.path]
             except KeyError:
                 pass
             # Call the parser already here, because it will be used anyways.
@@ -76,9 +73,9 @@ class ModuleWithCursor(Module):
             # default), therefore fill the cache here.
             self._parser = fast_parser.FastParser(self.source, self.path,
                                                         self.position)
-            if self.path is not None:
+            # don't pickle that module, because it's changing fast
-                cache.module_cache[self.path] = time.time(), \
+            cache.save_module(self.path, self.name, self._parser,
-                                                        self._parser
+                                pickling=True)
         return self._parser
 
     def get_path_until_cursor(self):

jedi/settings.py

@@ -25,7 +25,7 @@ Completion output
 Filesystem cache
 ~~~~~~~~~~~~~~~~
 
-.. autodata:: jedi_cache_directory
+.. autodata:: cache_directory
 .. autodata:: use_fs_cache
 
 
@@ -120,7 +120,7 @@ Use filesystem cache to save once parsed files with pickle.
 """
 
 import os
-jedi_cache_directory = os.getenv('HOME') + os.path.sep + '~/.jedi'
+cache_directory = os.getenv('HOME') + os.path.sep + '.jedi'
 """
 The path where all the caches can be found.
 """