diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 2827d46e..a629d608 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -17,6 +17,7 @@ from jedi._compatibility import unicode from jedi.parser import load_grammar from jedi.parser import tree from jedi.parser.fast import FastParser +from jedi.parser.utils import save_parser from jedi import debug from jedi import settings from jedi import common @@ -131,7 +132,7 @@ class Script(object): def _get_module(self): cache.invalidate_star_import_cache(self._path) parser = FastParser(self._grammar, self._source, self._path) - cache.save_parser(self._path, parser, pickling=False) + save_parser(self._path, parser, pickling=False) module = self._evaluator.wrap(parser.module) imports.add_module(self._evaluator, unicode(module.name), module) @@ -282,14 +283,13 @@ class Script(object): if call_signature_details is None: return [] - # TODO insert caching again here. - #with common.scale_speed_settings(settings.scale_call_signatures): - # definitions = cache.cache_call_signatures(self._evaluator, stmt, - # self._source, self._pos) - definitions = helpers.evaluate_goto_definition( - self._evaluator, - call_signature_details.bracket_leaf.get_previous_leaf() - ) + with common.scale_speed_settings(settings.scale_call_signatures): + definitions = helpers.cache_call_signatures( + self._evaluator, + call_signature_details.bracket_leaf, + self._code_lines, + self._pos + ) debug.speed('func_call followed') return [classes.CallSignature(self._evaluator, d.name, diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index db03a2b1..efe5ab48 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -7,6 +7,7 @@ from collections import namedtuple from jedi.evaluate.helpers import call_of_leaf from jedi import parser from jedi.parser import tokenize, token +from jedi.cache import time_cache CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name']) @@ -240,3 +241,24 @@ def get_call_signature_details(module, position): node = node.parent return None + + +@time_cache("call_signatures_validity") +def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos): + """This function calculates the cache key.""" + index = user_pos[0] - 1 + + before_cursor = code_lines[index][:user_pos[1]] + other_lines = code_lines[bracket_leaf.start_pos[0]:index] + whole = '\n'.join(other_lines + [before_cursor]) + before_bracket = re.match(r'.*\(', whole, re.DOTALL) + + module_path = bracket_leaf.get_parent_until().path + if module_path is None: + yield None # Don't cache! + else: + yield (module_path, before_bracket, bracket_leaf.start_pos) + yield evaluate_goto_definition( + evaluator, + bracket_leaf.get_previous_leaf() + ) diff --git a/jedi/cache.py b/jedi/cache.py index 56769d0d..8dc82544 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -14,36 +14,13 @@ there are global variables, which are holding the cache information. Some of these variables are being cleaned after every API usage. """ import time -import os -import sys -import json -import hashlib -import gc -import inspect -import shutil -import re -try: - import cPickle as pickle -except ImportError: - import pickle from jedi import settings -from jedi import common -from jedi import debug +from jedi.parser.utils import parser_cache +from jedi.parser.utils import underscore_memoization _time_caches = {} -# for fast_parser, should not be deleted -parser_cache = {} - - -class ParserCacheItem(object): - def __init__(self, parser, change_time=None): - self.parser = parser - if change_time is None: - change_time = time.time() - self.change_time = change_time - def clear_time_caches(delete_all=False): """ Jedi caches many things, that should be completed after each completion @@ -70,11 +47,12 @@ def clear_time_caches(delete_all=False): def time_cache(time_add_setting): """ - s This decorator works as follows: Call it with a setting and after that use the function with a callable that returns the key. But: This function is only called if the key is not available. After a certain amount of time (`time_add_setting`) the cache is invalid. + + If the given key is None, the function will not be cached. """ def _temp(key_func): dct = {} @@ -99,56 +77,6 @@ def time_cache(time_add_setting): return _temp -@time_cache("call_signatures_validity") -def cache_call_signatures(evaluator, call, source, user_pos): - """This function calculates the cache key.""" - index = user_pos[0] - 1 - lines = common.splitlines(source) - - before_cursor = lines[index][:user_pos[1]] - other_lines = lines[call.start_pos[0]:index] - whole = '\n'.join(other_lines + [before_cursor]) - before_bracket = re.match(r'.*\(', whole, re.DOTALL) - - module_path = call.get_parent_until().path - yield None if module_path is None else (module_path, before_bracket, call.start_pos) - yield evaluator.eval_element(call) - - -def underscore_memoization(func): - """ - Decorator for methods:: - - class A(object): - def x(self): - if self._x: - self._x = 10 - return self._x - - Becomes:: - - class A(object): - @underscore_memoization - def x(self): - return 10 - - A now has an attribute ``_x`` written by this decorator. - """ - name = '_' + func.__name__ - - def wrapper(self): - try: - return getattr(self, name) - except AttributeError: - result = func(self) - if inspect.isgenerator(result): - result = list(result) - setattr(self, name, result) - return result - - return wrapper - - def memoize_method(method): """A normal memoize function.""" def wrapper(self, *args, **kwargs): @@ -180,6 +108,14 @@ def _invalidate_star_import_cache_module(module, only_main=False): else: del _time_caches['star_import_cache_validity'][module] + # This stuff was part of load_parser. However since we're most likely + # not going to use star import caching anymore, just ignore it. + #else: + # In case there is already a module cached and this module + # has to be reparsed, we also need to invalidate the import + # caches. + # _invalidate_star_import_cache_module(parser_cache_item.parser.module) + def invalidate_star_import_cache(path): """On success returns True.""" @@ -189,148 +125,3 @@ def invalidate_star_import_cache(path): pass else: _invalidate_star_import_cache_module(parser_cache_item.parser.module) - - -def load_parser(path): - """ - Returns the module or None, if it fails. - """ - p_time = os.path.getmtime(path) if path else None - try: - parser_cache_item = parser_cache[path] - if not path or p_time <= parser_cache_item.change_time: - return parser_cache_item.parser - else: - # In case there is already a module cached and this module - # has to be reparsed, we also need to invalidate the import - # caches. - _invalidate_star_import_cache_module(parser_cache_item.parser.module) - except KeyError: - if settings.use_filesystem_cache: - return ParserPickling.load_parser(path, p_time) - - -def save_parser(path, parser, pickling=True): - try: - p_time = None if path is None else os.path.getmtime(path) - except OSError: - p_time = None - pickling = False - - item = ParserCacheItem(parser, p_time) - parser_cache[path] = item - if settings.use_filesystem_cache and pickling: - ParserPickling.save_parser(path, item) - - -class ParserPickling(object): - - version = 24 - """ - Version number (integer) for file system cache. - - Increment this number when there are any incompatible changes in - parser representation classes. For example, the following changes - are regarded as incompatible. - - - Class name is changed. - - Class is moved to another module. - - Defined slot of the class is changed. - """ - - def __init__(self): - self.__index = None - self.py_tag = 'cpython-%s%s' % sys.version_info[:2] - """ - Short name for distinguish Python implementations and versions. - - It's like `sys.implementation.cache_tag` but for Python < 3.3 - we generate something similar. See: - http://docs.python.org/3/library/sys.html#sys.implementation - - .. todo:: Detect interpreter (e.g., PyPy). - """ - - def load_parser(self, path, original_changed_time): - try: - pickle_changed_time = self._index[path] - except KeyError: - return None - if original_changed_time is not None \ - and pickle_changed_time < original_changed_time: - # the pickle file is outdated - return None - - with open(self._get_hashed_path(path), 'rb') as f: - try: - gc.disable() - parser_cache_item = pickle.load(f) - finally: - gc.enable() - - debug.dbg('pickle loaded: %s', path) - parser_cache[path] = parser_cache_item - return parser_cache_item.parser - - def save_parser(self, path, parser_cache_item): - self.__index = None - try: - files = self._index - except KeyError: - files = {} - self._index = files - - with open(self._get_hashed_path(path), 'wb') as f: - pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL) - files[path] = parser_cache_item.change_time - - self._flush_index() - - @property - def _index(self): - if self.__index is None: - try: - with open(self._get_path('index.json')) as f: - data = json.load(f) - except (IOError, ValueError): - self.__index = {} - else: - # 0 means version is not defined (= always delete cache): - if data.get('version', 0) != self.version: - self.clear_cache() - self.__index = {} - else: - self.__index = data['index'] - return self.__index - - def _remove_old_modules(self): - # TODO use - change = False - if change: - self._flush_index(self) - self._index # reload index - - def _flush_index(self): - data = {'version': self.version, 'index': self._index} - with open(self._get_path('index.json'), 'w') as f: - json.dump(data, f) - self.__index = None - - def clear_cache(self): - shutil.rmtree(self._cache_directory()) - - def _get_hashed_path(self, path): - return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) - - def _get_path(self, file): - dir = self._cache_directory() - if not os.path.exists(dir): - os.makedirs(dir) - return os.path.join(dir, file) - - def _cache_directory(self): - return os.path.join(settings.cache_directory, self.py_tag) - - -# is a singleton -ParserPickling = ParserPickling() diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 26ecf272..a3552e2e 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -20,9 +20,9 @@ from itertools import chain from jedi._compatibility import find_module, unicode from jedi import common from jedi import debug -from jedi import cache from jedi.parser import fast from jedi.parser import tree +from jedi.parser.utils import save_parser, load_parser, parser_cache from jedi.evaluate import sys_path from jedi.evaluate import helpers from jedi import settings @@ -435,7 +435,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None): def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith('.py') \ - and not dotted_path in settings.auto_import_modules: + and dotted_path not in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() @@ -443,13 +443,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None): return compiled.load_module(evaluator, path) p = path p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) - cache.save_parser(path, p) + save_parser(path, p) return p.module if sys_path is None: sys_path = evaluator.sys_path - cached = cache.load_parser(path) + cached = load_parser(path) module = load(source) if cached is None else cached.module module = evaluator.wrap(module) return module @@ -470,7 +470,7 @@ def get_modules_containing_name(evaluator, mods, name): """ def check_python_file(path): try: - return cache.parser_cache[path].parser.module + return parser_cache[path].parser.module except KeyError: try: return check_fs(path) diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index fabb2c1d..0139fb78 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -9,7 +9,7 @@ from jedi.parser import ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi import debug from jedi import common -from jedi import cache +from jedi.parser.utils import load_parser, save_parser def get_venv_path(venv): @@ -211,10 +211,10 @@ def _get_paths_from_buildout_script(evaluator, buildout_script): return p = ParserWithRecovery(evaluator.grammar, source, buildout_script) - cache.save_parser(buildout_script, p) + save_parser(buildout_script, p) return p.module - cached = cache.load_parser(buildout_script) + cached = load_parser(buildout_script) module = cached and cached.module or load(buildout_script) if not module: return diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index 291243d9..137c91ce 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -10,7 +10,7 @@ from jedi._compatibility import use_metaclass from jedi import settings from jedi.parser import ParserWithRecovery from jedi.parser import tree -from jedi import cache +from jedi.parser.utils import underscore_memoization, parser_cache from jedi import debug from jedi.parser.tokenize import (source_tokens, NEWLINE, ENDMARKER, INDENT, DEDENT) @@ -36,7 +36,7 @@ class FastModule(tree.Module): pass # It was never used. @property - @cache.underscore_memoization + @underscore_memoization def used_names(self): return MergedNamesDict([m.used_names for m in self.modules]) @@ -102,7 +102,7 @@ class CachedFastParser(type): if not settings.fast_parser: return ParserWithRecovery(grammar, source, module_path) - pi = cache.parser_cache.get(module_path, None) + pi = parser_cache.get(module_path, None) if pi is None or isinstance(pi.parser, ParserWithRecovery): p = super(CachedFastParser, self).__call__(grammar, source, module_path) else: @@ -236,7 +236,7 @@ class ParserNode(object): for y in n.all_sub_nodes(): yield y - @cache.underscore_memoization # Should only happen once! + @underscore_memoization # Should only happen once! def remove_last_newline(self): self.parser.remove_last_newline() diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 36a68721..3a874f73 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -41,8 +41,8 @@ import abc from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, literal_eval, use_metaclass, unicode) -from jedi import cache from jedi.parser import token +from jedi.parser.utils import underscore_memoization def is_node(node, *symbol_names): @@ -788,7 +788,7 @@ class Module(Scope): self.path = None # Set later. @property - @cache.underscore_memoization + @underscore_memoization def name(self): """ This is used for the goto functions. """ if self.path is None: diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index a928fbf1..513f4d92 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -7,6 +7,7 @@ from textwrap import dedent from jedi import api from jedi._compatibility import is_py3 from pytest import raises +from jedi.parser import utils def test_preload_modules(): @@ -16,16 +17,15 @@ def test_preload_modules(): for i in modules: assert [i in k for k in parser_cache.keys() if k is not None] - from jedi import cache - temp_cache, cache.parser_cache = cache.parser_cache, {} - parser_cache = cache.parser_cache + temp_cache, utils.parser_cache = utils.parser_cache, {} + parser_cache = utils.parser_cache api.preload_module('sys') check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') check_loaded('json', 'token') - cache.parser_cache = temp_cache + utils.parser_cache = temp_cache def test_empty_script(): diff --git a/test/test_cache.py b/test/test_cache.py index b1dfb709..7cff4d4c 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -8,7 +8,7 @@ import pytest import jedi from jedi import settings, cache -from jedi.cache import ParserCacheItem, ParserPickling +from jedi.parser.utils import ParserCacheItem, ParserPickling ParserPicklingCls = type(ParserPickling) diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index ac7c5a07..b7758e29 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -5,6 +5,7 @@ from jedi._compatibility import u from jedi import cache from jedi.parser import load_grammar from jedi.parser.fast import FastParser +from jedi.parser.utils import save_parser def test_add_to_end(): @@ -84,7 +85,7 @@ def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0 number_of_splits = number_parsers_used p = FastParser(load_grammar(), u(src)) - cache.save_parser(None, p, pickling=False) + save_parser(None, p, pickling=False) assert src == p.module.get_code() assert p.number_of_splits == number_of_splits @@ -355,7 +356,7 @@ def test_open_parentheses(): assert p.module.get_code() == code assert p.number_of_splits == 2 assert p.number_parsers_used == 2 - cache.save_parser(None, p, pickling=False) + save_parser(None, p, pickling=False) # Now with a correct parser it should work perfectly well. check_fp('isinstance()\n' + func, 1, 2)