VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Trying to change the import logic completely. We now have a sys.modules like cache.

Browse Source
This commit is contained in:
Dave Halter
2015-04-20 14:47:33 +02:00
parent 8fca3f78a1
commit df9452f210
7 changed files with 132 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
jedi/api/__init__.py
View File

@@ -114,7 +114,7 @@ class Script(object):
def _parsed_callback(self, parser):
module = er.wrap(self._evaluator, parser.module)
self._evaluator.module_name_cache[module] = unicode(module.name)
self._evaluator.modules[unicode(module.name)] = module
@property
def source_path(self):

19
jedi/cache.py
View File

@@ -191,17 +191,13 @@ def invalidate_star_import_cache(path):
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
def load_parser(path, name):
def load_parser(path):
"""
Returns the module or None, if it fails.
"""
if path is None and name is None:
return None
p_time = os.path.getmtime(path) if path else None
n = name if path is None else path
try:
parser_cache_item = parser_cache[n]
parser_cache_item = parser_cache[path]
if not path or p_time <= parser_cache_item.change_time:
return parser_cache_item.parser
else:
@@ -211,21 +207,20 @@ def load_parser(path, name):
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
except KeyError:
if settings.use_filesystem_cache:
return ParserPickling.load_parser(n, p_time)
return ParserPickling.load_parser(path, p_time)
def save_parser(path, name, parser, pickling=True):
def save_parser(path, parser, pickling=True):
try:
p_time = None if not path else os.path.getmtime(path)
p_time = os.path.getmtime(path)
except OSError:
p_time = None
pickling = False
n = name if path is None else path
item = ParserCacheItem(parser, p_time)
parser_cache[n] = item
parser_cache[path] = item
if settings.use_filesystem_cache and pickling:
ParserPickling.save_parser(n, item)
ParserPickling.save_parser(path, item)
class ParserPickling(object):

5
jedi/evaluate/__init__.py
View File

@@ -82,9 +82,8 @@ class Evaluator(object):
def __init__(self, grammar):
self.grammar = grammar
self.memoize_cache = {} # for memoize decorators
# To memorize module names (that are assigned to modules by the import
# logic) -> a ``__name__`` is given.
self.module_name_cache = {}
# To memorize modules -> equals `sys.modules`.
self.modules = {}
self.import_cache = {} # like `sys.modules`.
self.compiled_cache = {} # see `compiled.create()`
self.recursion_detector = recursion.RecursionDetector()

2
jedi/evaluate/compiled/__init__.py
View File

@@ -334,7 +334,7 @@ def dotted_from_fs_path(fs_path, sys_path=None):
return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.')
def load_module(path, name):
def load_module(path=None, name=None):
if path is not None:
dotted_path = dotted_from_fs_path(path)
else:

162
jedi/evaluate/imports.py
View File

@@ -163,13 +163,12 @@ def get_importer(evaluator, import_path, module, level=0):
# TODO add import error.
debug.warning('Attempted relative import beyond top-level package.')
# TODO this is just in the wrong place.
raise NotImplementedError
return _Importer(evaluator, import_path, module, level)
else:
# Here we basically rewrite the level to 0.
import_path = tuple(base) + import_path
check_import_path = tuple(unicode(i) for i in import_path)
try:
return evaluator.import_cache[check_import_path]
@@ -215,18 +214,34 @@ class _Importer(object):
@memoize_default()
def sys_path_with_modifications(self):
# If you edit e.g. gunicorn, there will be imports like this:
# `from gunicorn import something`. But gunicorn is not in the
# sys.path. Therefore look if gunicorn is a parent directory, #56.
in_path = []
if self.import_path and self.file_path is not None:
parts = self.file_path.split(os.path.sep)
for i, p in enumerate(parts):
if p == unicode(self.import_path[0]):
new = os.path.sep.join(parts[:i])
in_path.append(new)
sys_path_mod = list(sys_path_with_modifications(self._evaluator, self.module))
if self.file_path is not None:
# If you edit e.g. gunicorn, there will be imports like this:
# `from gunicorn import something`. But gunicorn is not in the
# sys.path. Therefore look if gunicorn is a parent directory, #56.
if self.import_path: # TODO is this check really needed?
parts = self.file_path.split(os.path.sep)
for i, p in enumerate(parts):
if p == unicode(self.import_path[0]):
new = os.path.sep.join(parts[:i])
in_path.append(new)
return in_path + sys_path_with_modifications(self._evaluator, self.module)
if not self.module.has_explicit_absolute_import:
# If the module explicitly asks for absolute imports,
# there's probably a bogus local one.
sys_path_mod.insert(0, self.file_path)
# First the sys path is searched normally and if that doesn't
# succeed, try to search the parent directories, because sometimes
# Jedi doesn't recognize sys.path modifications (like py.test
# stuff).
old_path, temp_path = self.file_path, os.path.dirname(self.file_path)
while old_path != temp_path:
sys_path_mod.append(temp_path)
old_path, temp_path = temp_path, os.path.dirname(temp_path)
return in_path + sys_path_mod
def follow(self, evaluator):
try:
@@ -257,6 +272,11 @@ class _Importer(object):
@memoize_default(NO_DEFAULT)
def follow_file_system(self):
module = self._do_import(self.import_path, self.sys_path_with_modifications())
return module, []
# TODO delete - move!
# Handle "magic" Flask extension imports:
# ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
if len(self.import_path) > 2 and self.str_import_path[:2] == ('flask', 'ext'):
@@ -325,50 +345,50 @@ class _Importer(object):
return follow_path((str(i) for i in import_path), sys.path)
return []
def _follow_str(self, sys_path, ns_path, string):
debug.dbg('follow_module %s in %s', string, ns_path)
path = None
if ns_path:
path = ns_path
elif self.level > 0: # is a relative import
path = self.get_relative_path()
if path is not None:
importing = find_module(string, [path])
else:
debug.dbg('search_module %s in %s', string, self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path, sys.path
try:
importing = find_module(string)
finally:
sys.path = temp
return importing
def _follow_sys_path(self, sys_path):
"""
Find a module with a path (of the module, like usb.backend.libusb10).
"""
def follow_str(ns_path, string):
debug.dbg('follow_module %s in %s', string, ns_path)
path = None
if ns_path:
path = ns_path
elif self.level > 0: # is a relative import
path = self.get_relative_path()
if path is not None:
importing = find_module(string, [path])
else:
debug.dbg('search_module %s in %s', string, self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path, sys.path
try:
importing = find_module(string)
finally:
sys.path = temp
return importing
current_namespace = (None, None, None)
# now execute those paths
rest = []
for i, s in enumerate(self.import_path):
try:
current_namespace = follow_str(current_namespace[1], unicode(s))
current_namespace = self._follow_str(sys_path, current_namespace[1], unicode(s))
except ImportError:
_continue = False
if self.level >= 1 and len(self.import_path) == 1:
# follow `from . import some_variable`
rel_path = self.get_relative_path()
with common.ignored(ImportError):
current_namespace = follow_str(rel_path, '__init__')
current_namespace = self._follow_str(sys_path, rel_path, '__init__')
elif current_namespace[2]: # is a package
path = self.str_import_path[:i]
for n in self.namespace_packages(current_namespace[1], path):
try:
current_namespace = follow_str(n, unicode(s))
current_namespace = self._follow_str(sys_path, n, unicode(s))
if current_namespace[1]:
_continue = True
break
@@ -414,6 +434,62 @@ class _Importer(object):
return _load_module(self._evaluator, name=path,
sys_path=sys_path, module_name=module_name), rest
def _do_import(self, import_path, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
import_parts = [str(i) for i in import_path]
module_name = '.'.join(import_parts)
try:
return self._evaluator.modules[module_name]
except KeyError:
try:
if len(import_path) > 1:
# This is a recursive way of importing that works great with
# the module cache.
base = self._do_import(import_path[:-1], sys_path)
path = base.py__file__()
debug.dbg('search_module %s in pkg %s', module_name, path)
module_file, module_path, is_pkg = \
find_module(import_parts[-1], [path])
raise NotImplementedError
else:
debug.dbg('search_module %s in %s', module_name, self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path, sys.path
try:
module_file, module_path, is_pkg = \
find_module(import_parts[-1])
finally:
sys.path = temp
except ImportError:
raise NotImplementedError
else:
source = None
if is_pkg:
# In this case, we don't have a file yet. Search for the
# __init__ file.
for suffix, _, _ in imp.get_suffixes():
path = os.path.join(module_path, '__init__' + suffix)
if os.path.exists(path):
if suffix == '.py':
module_path = path
break
elif module_file:
source = module_file.read()
module_file.close()
if module_file is None and not module_path.endswith('.py'):
module = compiled.load_module(module_path)
else:
module = _load_module(self._evaluator, module_path, source,
sys_path, module_name)
self._evaluator.modules[module_name] = module
return module
def _generate_name(self, name):
return helpers.FakeName(name, parent=self.module)
@@ -505,7 +581,7 @@ class _Importer(object):
return names
def _load_module(evaluator, path=None, source=None, name=None, sys_path=None, module_name=None):
def _load_module(evaluator, path=None, source=None, sys_path=None, module_name=None):
def load(source):
dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
if path is not None and path.endswith('.py') \
@@ -514,18 +590,16 @@ def _load_module(evaluator, path=None, source=None, name=None, sys_path=None, mo
with open(path, 'rb') as f:
source = f.read()
else:
return compiled.load_module(path, name)
p = path or name
return compiled.load_module(path)
p = path
p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
cache.save_parser(path, name, p)
cache.save_parser(path, p)
return p.module
cached = cache.load_parser(path, name)
cached = cache.load_parser(path)
module = load(source) if cached is None else cached.module
# TODO return mod instead of just something.
module = evaluator.wrap(module)
evaluator.module_name_cache[module] = module_name
return module

3
jedi/evaluate/sys_path.py
View File

@@ -30,6 +30,7 @@ def get_sys_path():
check_virtual_env(sys.path)
return [p for p in sys.path if p != ""]
def _get_venv_sitepackages(venv):
if os.name == 'nt':
p = os.path.join(venv, 'lib', 'site-packages')
@@ -175,7 +176,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
return
p = Parser(evaluator.grammar, source, buildout_script)
cache.save_parser(buildout_script, None, p)
cache.save_parser(buildout_script, p)
return p.module
cached = cache.load_parser(buildout_script, None)

2
jedi/parser/user_context.py
View File

@@ -281,7 +281,7 @@ class UserContextParser(object):
if self._use_fast_parser:
parser = FastParser(self._grammar, self._source, self._path)
# Don't pickle that module, because the main module is changing quickly
cache.save_parser(self._path, None, parser, pickling=False)
cache.save_parser(self._path, parser, pickling=False)
else:
parser = Parser(self._grammar, self._source, self._path)
self._parser_done_callback(parser)