From df9452f21021c3ab7be0deaac0e842043220c418 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 20 Apr 2015 14:47:33 +0200 Subject: [PATCH] Trying to change the import logic completely. We now have a sys.modules like cache. --- jedi/api/__init__.py | 2 +- jedi/cache.py | 19 ++-- jedi/evaluate/__init__.py | 5 +- jedi/evaluate/compiled/__init__.py | 2 +- jedi/evaluate/imports.py | 162 +++++++++++++++++++++-------- jedi/evaluate/sys_path.py | 3 +- jedi/parser/user_context.py | 2 +- 7 files changed, 132 insertions(+), 63 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index da2b7ed2..80a9c802 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -114,7 +114,7 @@ class Script(object): def _parsed_callback(self, parser): module = er.wrap(self._evaluator, parser.module) - self._evaluator.module_name_cache[module] = unicode(module.name) + self._evaluator.modules[unicode(module.name)] = module @property def source_path(self): diff --git a/jedi/cache.py b/jedi/cache.py index 68a3cc24..a05988e4 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -191,17 +191,13 @@ def invalidate_star_import_cache(path): _invalidate_star_import_cache_module(parser_cache_item.parser.module) -def load_parser(path, name): +def load_parser(path): """ Returns the module or None, if it fails. """ - if path is None and name is None: - return None - p_time = os.path.getmtime(path) if path else None - n = name if path is None else path try: - parser_cache_item = parser_cache[n] + parser_cache_item = parser_cache[path] if not path or p_time <= parser_cache_item.change_time: return parser_cache_item.parser else: @@ -211,21 +207,20 @@ def load_parser(path, name): _invalidate_star_import_cache_module(parser_cache_item.parser.module) except KeyError: if settings.use_filesystem_cache: - return ParserPickling.load_parser(n, p_time) + return ParserPickling.load_parser(path, p_time) -def save_parser(path, name, parser, pickling=True): +def save_parser(path, parser, pickling=True): try: - p_time = None if not path else os.path.getmtime(path) + p_time = os.path.getmtime(path) except OSError: p_time = None pickling = False - n = name if path is None else path item = ParserCacheItem(parser, p_time) - parser_cache[n] = item + parser_cache[path] = item if settings.use_filesystem_cache and pickling: - ParserPickling.save_parser(n, item) + ParserPickling.save_parser(path, item) class ParserPickling(object): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index d2e21322..82a243ec 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -82,9 +82,8 @@ class Evaluator(object): def __init__(self, grammar): self.grammar = grammar self.memoize_cache = {} # for memoize decorators - # To memorize module names (that are assigned to modules by the import - # logic) -> a ``__name__`` is given. - self.module_name_cache = {} + # To memorize modules -> equals `sys.modules`. + self.modules = {} self.import_cache = {} # like `sys.modules`. self.compiled_cache = {} # see `compiled.create()` self.recursion_detector = recursion.RecursionDetector() diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 0f5babfa..9bbce465 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -334,7 +334,7 @@ def dotted_from_fs_path(fs_path, sys_path=None): return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.') -def load_module(path, name): +def load_module(path=None, name=None): if path is not None: dotted_path = dotted_from_fs_path(path) else: diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 365b72d1..b455529a 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -163,13 +163,12 @@ def get_importer(evaluator, import_path, module, level=0): # TODO add import error. debug.warning('Attempted relative import beyond top-level package.') # TODO this is just in the wrong place. + raise NotImplementedError return _Importer(evaluator, import_path, module, level) else: # Here we basically rewrite the level to 0. import_path = tuple(base) + import_path - - check_import_path = tuple(unicode(i) for i in import_path) try: return evaluator.import_cache[check_import_path] @@ -215,18 +214,34 @@ class _Importer(object): @memoize_default() def sys_path_with_modifications(self): - # If you edit e.g. gunicorn, there will be imports like this: - # `from gunicorn import something`. But gunicorn is not in the - # sys.path. Therefore look if gunicorn is a parent directory, #56. in_path = [] - if self.import_path and self.file_path is not None: - parts = self.file_path.split(os.path.sep) - for i, p in enumerate(parts): - if p == unicode(self.import_path[0]): - new = os.path.sep.join(parts[:i]) - in_path.append(new) + sys_path_mod = list(sys_path_with_modifications(self._evaluator, self.module)) + if self.file_path is not None: + # If you edit e.g. gunicorn, there will be imports like this: + # `from gunicorn import something`. But gunicorn is not in the + # sys.path. Therefore look if gunicorn is a parent directory, #56. + if self.import_path: # TODO is this check really needed? + parts = self.file_path.split(os.path.sep) + for i, p in enumerate(parts): + if p == unicode(self.import_path[0]): + new = os.path.sep.join(parts[:i]) + in_path.append(new) - return in_path + sys_path_with_modifications(self._evaluator, self.module) + if not self.module.has_explicit_absolute_import: + # If the module explicitly asks for absolute imports, + # there's probably a bogus local one. + sys_path_mod.insert(0, self.file_path) + + # First the sys path is searched normally and if that doesn't + # succeed, try to search the parent directories, because sometimes + # Jedi doesn't recognize sys.path modifications (like py.test + # stuff). + old_path, temp_path = self.file_path, os.path.dirname(self.file_path) + while old_path != temp_path: + sys_path_mod.append(temp_path) + old_path, temp_path = temp_path, os.path.dirname(temp_path) + + return in_path + sys_path_mod def follow(self, evaluator): try: @@ -257,6 +272,11 @@ class _Importer(object): @memoize_default(NO_DEFAULT) def follow_file_system(self): + module = self._do_import(self.import_path, self.sys_path_with_modifications()) + return module, [] + + +# TODO delete - move! # Handle "magic" Flask extension imports: # ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``. if len(self.import_path) > 2 and self.str_import_path[:2] == ('flask', 'ext'): @@ -325,50 +345,50 @@ class _Importer(object): return follow_path((str(i) for i in import_path), sys.path) return [] + def _follow_str(self, sys_path, ns_path, string): + debug.dbg('follow_module %s in %s', string, ns_path) + path = None + if ns_path: + path = ns_path + elif self.level > 0: # is a relative import + path = self.get_relative_path() + + if path is not None: + importing = find_module(string, [path]) + else: + debug.dbg('search_module %s in %s', string, self.file_path) + # Override the sys.path. It works only good that way. + # Injecting the path directly into `find_module` did not work. + sys.path, temp = sys_path, sys.path + try: + importing = find_module(string) + finally: + sys.path = temp + + return importing + def _follow_sys_path(self, sys_path): """ Find a module with a path (of the module, like usb.backend.libusb10). """ - def follow_str(ns_path, string): - debug.dbg('follow_module %s in %s', string, ns_path) - path = None - if ns_path: - path = ns_path - elif self.level > 0: # is a relative import - path = self.get_relative_path() - - if path is not None: - importing = find_module(string, [path]) - else: - debug.dbg('search_module %s in %s', string, self.file_path) - # Override the sys.path. It works only good that way. - # Injecting the path directly into `find_module` did not work. - sys.path, temp = sys_path, sys.path - try: - importing = find_module(string) - finally: - sys.path = temp - - return importing - current_namespace = (None, None, None) # now execute those paths rest = [] for i, s in enumerate(self.import_path): try: - current_namespace = follow_str(current_namespace[1], unicode(s)) + current_namespace = self._follow_str(sys_path, current_namespace[1], unicode(s)) except ImportError: _continue = False if self.level >= 1 and len(self.import_path) == 1: # follow `from . import some_variable` rel_path = self.get_relative_path() with common.ignored(ImportError): - current_namespace = follow_str(rel_path, '__init__') + current_namespace = self._follow_str(sys_path, rel_path, '__init__') elif current_namespace[2]: # is a package path = self.str_import_path[:i] for n in self.namespace_packages(current_namespace[1], path): try: - current_namespace = follow_str(n, unicode(s)) + current_namespace = self._follow_str(sys_path, n, unicode(s)) if current_namespace[1]: _continue = True break @@ -414,6 +434,62 @@ class _Importer(object): return _load_module(self._evaluator, name=path, sys_path=sys_path, module_name=module_name), rest + def _do_import(self, import_path, sys_path): + """ + This method is very similar to importlib's `_gcd_import`. + """ + import_parts = [str(i) for i in import_path] + module_name = '.'.join(import_parts) + try: + return self._evaluator.modules[module_name] + except KeyError: + try: + if len(import_path) > 1: + # This is a recursive way of importing that works great with + # the module cache. + base = self._do_import(import_path[:-1], sys_path) + path = base.py__file__() + + debug.dbg('search_module %s in pkg %s', module_name, path) + module_file, module_path, is_pkg = \ + find_module(import_parts[-1], [path]) + raise NotImplementedError + else: + debug.dbg('search_module %s in %s', module_name, self.file_path) + # Override the sys.path. It works only good that way. + # Injecting the path directly into `find_module` did not work. + sys.path, temp = sys_path, sys.path + try: + module_file, module_path, is_pkg = \ + find_module(import_parts[-1]) + finally: + sys.path = temp + except ImportError: + raise NotImplementedError + else: + source = None + if is_pkg: + # In this case, we don't have a file yet. Search for the + # __init__ file. + for suffix, _, _ in imp.get_suffixes(): + path = os.path.join(module_path, '__init__' + suffix) + if os.path.exists(path): + if suffix == '.py': + module_path = path + break + elif module_file: + source = module_file.read() + module_file.close() + + if module_file is None and not module_path.endswith('.py'): + module = compiled.load_module(module_path) + else: + module = _load_module(self._evaluator, module_path, source, + sys_path, module_name) + + self._evaluator.modules[module_name] = module + return module + def _generate_name(self, name): return helpers.FakeName(name, parent=self.module) @@ -505,7 +581,7 @@ class _Importer(object): return names -def _load_module(evaluator, path=None, source=None, name=None, sys_path=None, module_name=None): +def _load_module(evaluator, path=None, source=None, sys_path=None, module_name=None): def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith('.py') \ @@ -514,18 +590,16 @@ def _load_module(evaluator, path=None, source=None, name=None, sys_path=None, mo with open(path, 'rb') as f: source = f.read() else: - return compiled.load_module(path, name) - p = path or name + return compiled.load_module(path) + p = path p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) - cache.save_parser(path, name, p) - + cache.save_parser(path, p) return p.module - cached = cache.load_parser(path, name) + cached = cache.load_parser(path) module = load(source) if cached is None else cached.module # TODO return mod instead of just something. module = evaluator.wrap(module) - evaluator.module_name_cache[module] = module_name return module diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index e2090c67..230127f0 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -30,6 +30,7 @@ def get_sys_path(): check_virtual_env(sys.path) return [p for p in sys.path if p != ""] + def _get_venv_sitepackages(venv): if os.name == 'nt': p = os.path.join(venv, 'lib', 'site-packages') @@ -175,7 +176,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script): return p = Parser(evaluator.grammar, source, buildout_script) - cache.save_parser(buildout_script, None, p) + cache.save_parser(buildout_script, p) return p.module cached = cache.load_parser(buildout_script, None) diff --git a/jedi/parser/user_context.py b/jedi/parser/user_context.py index 1e12d64d..43250313 100644 --- a/jedi/parser/user_context.py +++ b/jedi/parser/user_context.py @@ -281,7 +281,7 @@ class UserContextParser(object): if self._use_fast_parser: parser = FastParser(self._grammar, self._source, self._path) # Don't pickle that module, because the main module is changing quickly - cache.save_parser(self._path, None, parser, pickling=False) + cache.save_parser(self._path, parser, pickling=False) else: parser = Parser(self._grammar, self._source, self._path) self._parser_done_callback(parser)