VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Reenable call signature caching and move a lot of parser specific caching to the parser itself.

Browse Source
This commit is contained in:
Dave Halter
2016-06-28 08:46:29 +02:00
parent 746a513ef9
commit 52c42c3392
10 changed files with 65 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
jedi/api/__init__.py
View File

@@ -17,6 +17,7 @@ from jedi._compatibility import unicode
from jedi.parser import load_grammar
from jedi.parser import tree
from jedi.parser.fast import FastParser
from jedi.parser.utils import save_parser
from jedi import debug
from jedi import settings
from jedi import common
@@ -131,7 +132,7 @@ class Script(object):
def _get_module(self):
cache.invalidate_star_import_cache(self._path)
parser = FastParser(self._grammar, self._source, self._path)
cache.save_parser(self._path, parser, pickling=False)
save_parser(self._path, parser, pickling=False)
module = self._evaluator.wrap(parser.module)
imports.add_module(self._evaluator, unicode(module.name), module)
@@ -282,14 +283,13 @@ class Script(object):
if call_signature_details is None:
return []
# TODO insert caching again here.
#with common.scale_speed_settings(settings.scale_call_signatures):
# definitions = cache.cache_call_signatures(self._evaluator, stmt,
# self._source, self._pos)
definitions = helpers.evaluate_goto_definition(
self._evaluator,
call_signature_details.bracket_leaf.get_previous_leaf()
)
with common.scale_speed_settings(settings.scale_call_signatures):
definitions = helpers.cache_call_signatures(
self._evaluator,
call_signature_details.bracket_leaf,
self._code_lines,
self._pos
)
debug.speed('func_call followed')
return [classes.CallSignature(self._evaluator, d.name,

22
jedi/api/helpers.py
View File

@@ -7,6 +7,7 @@ from collections import namedtuple
from jedi.evaluate.helpers import call_of_leaf
from jedi import parser
from jedi.parser import tokenize, token
from jedi.cache import time_cache
CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
@@ -240,3 +241,24 @@ def get_call_signature_details(module, position):
node = node.parent
return None
@time_cache("call_signatures_validity")
def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos):
"""This function calculates the cache key."""
index = user_pos[0] - 1
before_cursor = code_lines[index][:user_pos[1]]
other_lines = code_lines[bracket_leaf.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = bracket_leaf.get_parent_until().path
if module_path is None:
yield None # Don't cache!
else:
yield (module_path, before_bracket, bracket_leaf.start_pos)
yield evaluate_goto_definition(
evaluator,
bracket_leaf.get_previous_leaf()
)

233
jedi/cache.py
View File

@@ -14,36 +14,13 @@ there are global variables, which are holding the cache information. Some of
these variables are being cleaned after every API usage.
"""
import time
import os
import sys
import json
import hashlib
import gc
import inspect
import shutil
import re
try:
import cPickle as pickle
except ImportError:
import pickle
from jedi import settings
from jedi import common
from jedi import debug
from jedi.parser.utils import parser_cache
from jedi.parser.utils import underscore_memoization
_time_caches = {}
# for fast_parser, should not be deleted
parser_cache = {}
class ParserCacheItem(object):
def __init__(self, parser, change_time=None):
self.parser = parser
if change_time is None:
change_time = time.time()
self.change_time = change_time
def clear_time_caches(delete_all=False):
""" Jedi caches many things, that should be completed after each completion
@@ -70,11 +47,12 @@ def clear_time_caches(delete_all=False):
def time_cache(time_add_setting):
"""
s
This decorator works as follows: Call it with a setting and after that
use the function with a callable that returns the key.
But: This function is only called if the key is not available. After a
certain amount of time (`time_add_setting`) the cache is invalid.
If the given key is None, the function will not be cached.
"""
def _temp(key_func):
dct = {}
@@ -99,56 +77,6 @@ def time_cache(time_add_setting):
return _temp
@time_cache("call_signatures_validity")
def cache_call_signatures(evaluator, call, source, user_pos):
"""This function calculates the cache key."""
index = user_pos[0] - 1
lines = common.splitlines(source)
before_cursor = lines[index][:user_pos[1]]
other_lines = lines[call.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = call.get_parent_until().path
yield None if module_path is None else (module_path, before_bracket, call.start_pos)
yield evaluator.eval_element(call)
def underscore_memoization(func):
"""
Decorator for methods::
class A(object):
def x(self):
if self._x:
self._x = 10
return self._x
Becomes::
class A(object):
@underscore_memoization
def x(self):
return 10
A now has an attribute ``_x`` written by this decorator.
"""
name = '_' + func.__name__
def wrapper(self):
try:
return getattr(self, name)
except AttributeError:
result = func(self)
if inspect.isgenerator(result):
result = list(result)
setattr(self, name, result)
return result
return wrapper
def memoize_method(method):
"""A normal memoize function."""
def wrapper(self, *args, **kwargs):
@@ -180,6 +108,14 @@ def _invalidate_star_import_cache_module(module, only_main=False):
else:
del _time_caches['star_import_cache_validity'][module]
# This stuff was part of load_parser. However since we're most likely
# not going to use star import caching anymore, just ignore it.
#else:
# In case there is already a module cached and this module
# has to be reparsed, we also need to invalidate the import
# caches.
# _invalidate_star_import_cache_module(parser_cache_item.parser.module)
def invalidate_star_import_cache(path):
"""On success returns True."""
@@ -189,148 +125,3 @@ def invalidate_star_import_cache(path):
pass
else:
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
def load_parser(path):
"""
Returns the module or None, if it fails.
"""
p_time = os.path.getmtime(path) if path else None
try:
parser_cache_item = parser_cache[path]
if not path or p_time <= parser_cache_item.change_time:
return parser_cache_item.parser
else:
# In case there is already a module cached and this module
# has to be reparsed, we also need to invalidate the import
# caches.
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
except KeyError:
if settings.use_filesystem_cache:
return ParserPickling.load_parser(path, p_time)
def save_parser(path, parser, pickling=True):
try:
p_time = None if path is None else os.path.getmtime(path)
except OSError:
p_time = None
pickling = False
item = ParserCacheItem(parser, p_time)
parser_cache[path] = item
if settings.use_filesystem_cache and pickling:
ParserPickling.save_parser(path, item)
class ParserPickling(object):
version = 24
"""
Version number (integer) for file system cache.
Increment this number when there are any incompatible changes in
parser representation classes. For example, the following changes
are regarded as incompatible.
- Class name is changed.
- Class is moved to another module.
- Defined slot of the class is changed.
"""
def __init__(self):
self.__index = None
self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
"""
Short name for distinguish Python implementations and versions.
It's like `sys.implementation.cache_tag` but for Python < 3.3
we generate something similar. See:
http://docs.python.org/3/library/sys.html#sys.implementation
.. todo:: Detect interpreter (e.g., PyPy).
"""
def load_parser(self, path, original_changed_time):
try:
pickle_changed_time = self._index[path]
except KeyError:
return None
if original_changed_time is not None \
and pickle_changed_time < original_changed_time:
# the pickle file is outdated
return None
with open(self._get_hashed_path(path), 'rb') as f:
try:
gc.disable()
parser_cache_item = pickle.load(f)
finally:
gc.enable()
debug.dbg('pickle loaded: %s', path)
parser_cache[path] = parser_cache_item
return parser_cache_item.parser
def save_parser(self, path, parser_cache_item):
self.__index = None
try:
files = self._index
except KeyError:
files = {}
self._index = files
with open(self._get_hashed_path(path), 'wb') as f:
pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
files[path] = parser_cache_item.change_time
self._flush_index()
@property
def _index(self):
if self.__index is None:
try:
with open(self._get_path('index.json')) as f:
data = json.load(f)
except (IOError, ValueError):
self.__index = {}
else:
# 0 means version is not defined (= always delete cache):
if data.get('version', 0) != self.version:
self.clear_cache()
self.__index = {}
else:
self.__index = data['index']
return self.__index
def _remove_old_modules(self):
# TODO use
change = False
if change:
self._flush_index(self)
self._index # reload index
def _flush_index(self):
data = {'version': self.version, 'index': self._index}
with open(self._get_path('index.json'), 'w') as f:
json.dump(data, f)
self.__index = None
def clear_cache(self):
shutil.rmtree(self._cache_directory())
def _get_hashed_path(self, path):
return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
def _get_path(self, file):
dir = self._cache_directory()
if not os.path.exists(dir):
os.makedirs(dir)
return os.path.join(dir, file)
def _cache_directory(self):
return os.path.join(settings.cache_directory, self.py_tag)
# is a singleton
ParserPickling = ParserPickling()

10
jedi/evaluate/imports.py
View File

@@ -20,9 +20,9 @@ from itertools import chain
from jedi._compatibility import find_module, unicode
from jedi import common
from jedi import debug
from jedi import cache
from jedi.parser import fast
from jedi.parser import tree
from jedi.parser.utils import save_parser, load_parser, parser_cache
from jedi.evaluate import sys_path
from jedi.evaluate import helpers
from jedi import settings
@@ -435,7 +435,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
def load(source):
dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
if path is not None and path.endswith('.py') \
and not dotted_path in settings.auto_import_modules:
and dotted_path not in settings.auto_import_modules:
if source is None:
with open(path, 'rb') as f:
source = f.read()
@@ -443,13 +443,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
return compiled.load_module(evaluator, path)
p = path
p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
cache.save_parser(path, p)
save_parser(path, p)
return p.module
if sys_path is None:
sys_path = evaluator.sys_path
cached = cache.load_parser(path)
cached = load_parser(path)
module = load(source) if cached is None else cached.module
module = evaluator.wrap(module)
return module
@@ -470,7 +470,7 @@ def get_modules_containing_name(evaluator, mods, name):
"""
def check_python_file(path):
try:
return cache.parser_cache[path].parser.module
return parser_cache[path].parser.module
except KeyError:
try:
return check_fs(path)

6
jedi/evaluate/sys_path.py
View File

@@ -9,7 +9,7 @@ from jedi.parser import ParserWithRecovery
from jedi.evaluate.cache import memoize_default
from jedi import debug
from jedi import common
from jedi import cache
from jedi.parser.utils import load_parser, save_parser
def get_venv_path(venv):
@@ -211,10 +211,10 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
return
p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
cache.save_parser(buildout_script, p)
save_parser(buildout_script, p)
return p.module
cached = cache.load_parser(buildout_script)
cached = load_parser(buildout_script)
module = cached and cached.module or load(buildout_script)
if not module:
return

8
jedi/parser/fast.py
View File

@@ -10,7 +10,7 @@ from jedi._compatibility import use_metaclass
from jedi import settings
from jedi.parser import ParserWithRecovery
from jedi.parser import tree
from jedi import cache
from jedi.parser.utils import underscore_memoization, parser_cache
from jedi import debug
from jedi.parser.tokenize import (source_tokens, NEWLINE,
ENDMARKER, INDENT, DEDENT)
@@ -36,7 +36,7 @@ class FastModule(tree.Module):
pass # It was never used.
@property
@cache.underscore_memoization
@underscore_memoization
def used_names(self):
return MergedNamesDict([m.used_names for m in self.modules])
@@ -102,7 +102,7 @@ class CachedFastParser(type):
if not settings.fast_parser:
return ParserWithRecovery(grammar, source, module_path)
pi = cache.parser_cache.get(module_path, None)
pi = parser_cache.get(module_path, None)
if pi is None or isinstance(pi.parser, ParserWithRecovery):
p = super(CachedFastParser, self).__call__(grammar, source, module_path)
else:
@@ -236,7 +236,7 @@ class ParserNode(object):
for y in n.all_sub_nodes():
yield y
@cache.underscore_memoization # Should only happen once!
@underscore_memoization # Should only happen once!
def remove_last_newline(self):
self.parser.remove_last_newline()

4
jedi/parser/tree.py
View File

@@ -41,8 +41,8 @@ import abc
from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
literal_eval, use_metaclass, unicode)
from jedi import cache
from jedi.parser import token
from jedi.parser.utils import underscore_memoization
def is_node(node, *symbol_names):
@@ -788,7 +788,7 @@ class Module(Scope):
self.path = None # Set later.
@property
@cache.underscore_memoization
@underscore_memoization
def name(self):
""" This is used for the goto functions. """
if self.path is None:

8
test/test_api/test_api.py
View File

@@ -7,6 +7,7 @@ from textwrap import dedent
from jedi import api
from jedi._compatibility import is_py3
from pytest import raises
from jedi.parser import utils
def test_preload_modules():
@@ -16,16 +17,15 @@ def test_preload_modules():
for i in modules:
assert [i in k for k in parser_cache.keys() if k is not None]
from jedi import cache
temp_cache, cache.parser_cache = cache.parser_cache, {}
parser_cache = cache.parser_cache
temp_cache, utils.parser_cache = utils.parser_cache, {}
parser_cache = utils.parser_cache
api.preload_module('sys')
check_loaded() # compiled (c_builtin) modules shouldn't be in the cache.
api.preload_module('json', 'token')
check_loaded('json', 'token')
cache.parser_cache = temp_cache
utils.parser_cache = temp_cache
def test_empty_script():

2
test/test_cache.py
View File

@@ -8,7 +8,7 @@ import pytest
import jedi
from jedi import settings, cache
from jedi.cache import ParserCacheItem, ParserPickling
from jedi.parser.utils import ParserCacheItem, ParserPickling
ParserPicklingCls = type(ParserPickling)

5
test/test_parser/test_fast_parser.py
View File

@@ -5,6 +5,7 @@ from jedi._compatibility import u
from jedi import cache
from jedi.parser import load_grammar
from jedi.parser.fast import FastParser
from jedi.parser.utils import save_parser
def test_add_to_end():
@@ -84,7 +85,7 @@ def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0
number_of_splits = number_parsers_used
p = FastParser(load_grammar(), u(src))
cache.save_parser(None, p, pickling=False)
save_parser(None, p, pickling=False)
assert src == p.module.get_code()
assert p.number_of_splits == number_of_splits
@@ -355,7 +356,7 @@ def test_open_parentheses():
assert p.module.get_code() == code
assert p.number_of_splits == 2
assert p.number_parsers_used == 2
cache.save_parser(None, p, pickling=False)
save_parser(None, p, pickling=False)
# Now with a correct parser it should work perfectly well.
check_fp('isinstance()\n' + func, 1, 2)