From 1017db903c2a88e789c2b4426f2e3231107fa7a4 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 11 Jan 2013 22:00:03 +0100 Subject: [PATCH] basic pickle implementation #102 --- jedi/_compatibility.py | 6 +++ jedi/api.py | 2 +- jedi/builtin.py | 10 ++-- jedi/cache.py | 106 ++++++++++++++++++++++++++++++++++++----- jedi/dynamic.py | 2 +- jedi/modules.py | 11 ++--- jedi/settings.py | 4 +- 7 files changed, 112 insertions(+), 29 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 7e3fa566..1710032e 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -172,3 +172,9 @@ try: from functools import reduce except ImportError: reduce = reduce + +try: + import json +except ImportError: + # python 2.5 + import simplejson as json diff --git a/jedi/api.py b/jedi/api.py index a97872dd..ccc4d7bb 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -376,7 +376,7 @@ class Script(object): return None, 0 try: - timestamp, parser = cache.module_cache[self.source_path] + parser = cache.module_cache[self.source_path].parser except KeyError: return None, 0 part_parser = self._module.get_part_parser() diff --git a/jedi/builtin.py b/jedi/builtin.py index c48dfab0..c1b48425 100644 --- a/jedi/builtin.py +++ b/jedi/builtin.py @@ -48,9 +48,8 @@ class CachedModule(object): def parser(self): """ get the parser lazy """ if self._parser is None: - self._parser = cache.load_module(self.path, self.name) - if self._parser is None: - self._load_module() + self._parser = cache.load_module(self.path, self.name) \ + or self._load_module() return self._parser def _get_source(self): @@ -59,8 +58,9 @@ class CachedModule(object): def _load_module(self): source = self._get_source() p = self.path or self.name - self._parser = fast_parser.FastParser(source, p) - cache.save_module(self.path, self.name, self._parser) + p = fast_parser.FastParser(source, p) + cache.save_module(self.path, self.name, p) + return p class Parser(CachedModule): diff --git a/jedi/cache.py b/jedi/cache.py index 4f7f0098..73411dd8 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -1,7 +1,11 @@ import time import os +import sys +import pickle +from _compatibility import json import settings +import debug # memoize caches will be deleted after every action memoize_caches = [] @@ -16,6 +20,14 @@ parser_cache = {} module_cache = {} +class ModuleCacheItem(object): + def __init__(self, parser, change_time=None): + self.parser = parser + if change_time is None: + change_time = time.time() + self.change_time = change_time + + def clear_caches(delete_all=False): """ Jedi caches many things, that should be completed after each completion finishes. @@ -156,32 +168,100 @@ def load_module(path, name): if path is None and name is None: return None + tim = os.path.getmtime(path) if path else None try: - timestamp, parser = module_cache[path or name] - if not path or os.path.getmtime(path) <= timestamp: - return parser + module_cache_item = module_cache[path or name] + if not path or tim <= module_cache_item.change_time: + return module_cache_item.parser else: # In case there is already a module cached and this module # has to be reparsed, we also need to invalidate the import # caches. - invalidate_star_import_cache(parser.module) - return None + invalidate_star_import_cache(module_cache_item.parser.module) except KeyError: - return load_pickle_module(path or name) + if settings.use_filesystem_cache: + return ModulePickling.load_module(path or name, tim) -def save_module(path, name, parser): +def save_module(path, name, parser, pickling=True): if path is None and name is None: return p_time = None if not path else os.path.getmtime(path) - module_cache[path or name] = p_time, parser - save_pickle_module(path or name) + item = ModuleCacheItem(parser, p_time) + module_cache[path or name] = item + if settings.use_filesystem_cache and pickling: + ModulePickling.save_module(path or name, item) -def load_pickle_module(path): - return None +class _ModulePickling(object): + def __init__(self): + self.__index = None + self.py_version = '%s.%s' % sys.version_info[:2] + + def load_module(self, path, original_changed_time): + try: + pickle_changed_time = self._index[self.py_version][path] + except KeyError: + return None + if original_changed_time is not None \ + and pickle_changed_time < original_changed_time: + # the pickle file is outdated + return None + + with open(self._get_hashed_path(path)) as f: + module_cache_item = pickle.load(f) + + parser = module_cache_item.parser + debug.dbg('pickle loaded', path) + parser_cache[path] = parser + module_cache[path] = module_cache_item + return parser + + def save_module(self, path, module_cache_item): + try: + files = self._index[self.py_version] + except KeyError: + files = {} + self._index[self.py_version] = files + + with open(self._get_hashed_path(path), 'w') as f: + pickle.dump(module_cache_item, f, pickle.HIGHEST_PROTOCOL) + files[path] = module_cache_item.change_time + + self._flush_index() + + @property + def _index(self): + if self.__index is None: + try: + with open(self._get_path('index.json')) as f: + self.__index = json.load(f) + except IOError: + self.__index = {} + return self.__index + + def _remove_old_modules(self): + # TODO use + change = False + if change: + self._flush_index(self) + self._index # reload index + + def _flush_index(self): + with open(self._get_path('index.json'), 'w') as f: + json.dump(self._index, f) + self.__index = None + + def _get_hashed_path(self, path): + return self._get_path('%s_%s.pkl' % (self.py_version, hash(path))) + + def _get_path(self, file): + dir = settings.cache_directory + if not os.path.exists(dir): + os.makedirs(dir) + return dir + os.path.sep + file -def save_pickle_module(path): - pass +# is a singleton +ModulePickling = _ModulePickling() diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 7a47126b..86427f16 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -32,7 +32,7 @@ def get_directory_modules_for_name(mods, name): """ def check_python_file(path): try: - return cache.module_cache[path][1].module + return cache.module_cache[path].parser.module except KeyError: try: return check_fs(path) diff --git a/jedi/modules.py b/jedi/modules.py index 2c66f2b0..d5fd6153 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -6,7 +6,6 @@ import re import tokenize import sys import os -import time import cache import parsing @@ -65,10 +64,8 @@ class ModuleWithCursor(Module): """ get the parser lazy """ if not self._parser: try: - ts, parser = cache.module_cache[self.path] + parser = cache.module_cache[self.path].parser cache.invalidate_star_import_cache(parser.module) - - del cache.module_cache[self.path] except KeyError: pass # Call the parser already here, because it will be used anyways. @@ -76,9 +73,9 @@ class ModuleWithCursor(Module): # default), therefore fill the cache here. self._parser = fast_parser.FastParser(self.source, self.path, self.position) - if self.path is not None: - cache.module_cache[self.path] = time.time(), \ - self._parser + # don't pickle that module, because it's changing fast + cache.save_module(self.path, self.name, self._parser, + pickling=True) return self._parser def get_path_until_cursor(self): diff --git a/jedi/settings.py b/jedi/settings.py index 6c842306..3515f277 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -25,7 +25,7 @@ Completion output Filesystem cache ~~~~~~~~~~~~~~~~ -.. autodata:: jedi_cache_directory +.. autodata:: cache_directory .. autodata:: use_fs_cache @@ -120,7 +120,7 @@ Use filesystem cache to save once parsed files with pickle. """ import os -jedi_cache_directory = os.getenv('HOME') + os.path.sep + '~/.jedi' +cache_directory = os.getenv('HOME') + os.path.sep + '.jedi' """ The path where all the caches can be found. """