From 78cbad0d08f4793479ed1b9493272c23b6574a68 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Wed, 29 Nov 2017 17:05:31 +0100 Subject: [PATCH 01/14] Fix implicit namespace autocompletion. Resolves: #959 --- AUTHORS.txt | 1 + docs/docs/features.rst | 3 +- jedi/_compatibility.py | 45 +++++++++++++++++++ jedi/evaluate/context/module.py | 4 +- jedi/evaluate/imports.py | 7 ++- .../test_implicit_namespace_package.py | 31 +++++++++++++ 6 files changed, 83 insertions(+), 8 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index 29f46765..e0d6110a 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -46,5 +46,6 @@ Simon Ruggier (@sruggier) Élie Gouzien (@ElieGouzien) Robin Roth (@robinro) Malte Plath (@langsamer) +Maksim Novikov (@m-novikov) Note: (@user) means a github user name. diff --git a/docs/docs/features.rst b/docs/docs/features.rst index aef32708..83935f39 100644 --- a/docs/docs/features.rst +++ b/docs/docs/features.rst @@ -53,7 +53,7 @@ Supported Python Features case, that doesn't work with |jedi|) - simple/usual ``sys.path`` modifications - ``isinstance`` checks for if/while/assert -- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces) +- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces) - Django / Flask / Buildout support @@ -64,7 +64,6 @@ Not yet implemented: - manipulations of instances outside the instance variables without using methods -- implicit namespace packages (Python 3.3+, `PEP 420 `_) Will probably never be implemented: diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 52a20fe2..30fe705c 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -161,6 +161,51 @@ if the module is contained in a package. """ +def _iter_modules(paths, prefix=''): + # Copy of pkgutil.iter_modules adapted to work with namespaces + for path in paths: + importer = pkgutil.get_importer(path) + + if importer.path is None or not os.path.isdir(importer.path): + return + + yielded = {} + import inspect + try: + filenames = os.listdir(importer.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + # Avoid traversing special directories + if fn.startswith(('__', '.')): + continue + + modname = inspect.getmodulename(fn) + if modname in yielded: + continue + + path = os.path.join(importer.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + ispkg = True + + if modname and '.' not in modname: + yielded[modname] = 1 + yield importer, prefix + modname, ispkg + +iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules + + class ImplicitNSInfo(object): """Stores information returned from an implicit namespace spec""" def __init__(self, name, paths): diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 5ba92cdb..f5fa9f0f 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -5,7 +5,7 @@ import os from parso import python_bytes_to_unicode -from jedi._compatibility import use_metaclass +from jedi._compatibility import use_metaclass, iter_modules from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \ AbstractNameDefinition, ParserTreeFilter, DictFilter @@ -188,7 +188,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): path = self._path names = {} if path is not None and path.endswith(os.path.sep + '__init__.py'): - mods = pkgutil.iter_modules([os.path.dirname(path)]) + mods = iter_modules([os.path.dirname(path)]) for module_loader, name, is_pkg in mods: # It's obviously a relative import to the current module. names[name] = SubModuleName(self, name) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 89dd833f..8bc74c06 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -21,7 +21,7 @@ from parso.tree import search_ancestor from parso.cache import parser_cache from parso import python_bytes_to_unicode -from jedi._compatibility import find_module, unicode, ImplicitNSInfo +from jedi._compatibility import find_module, unicode, ImplicitNSInfo, iter_modules from jedi import debug from jedi import settings from jedi.evaluate import sys_path @@ -401,7 +401,6 @@ class Importer(object): Get the names of all modules in the search_path. This means file names and not names defined in the files. """ - names = [] # add builtin module names if search_path is None and in_module is None: @@ -409,7 +408,7 @@ class Importer(object): if search_path is None: search_path = self.sys_path_with_modifications() - for module_loader, name, is_pkg in pkgutil.iter_modules(search_path): + for module_loader, name, is_pkg in iter_modules(search_path): names.append(self._generate_name(name, in_module=in_module)) return names @@ -448,7 +447,7 @@ class Importer(object): # implicit namespace packages elif isinstance(context, ImplicitNamespaceContext): paths = context.paths - names += self._get_module_names(paths) + names += self._get_module_names(paths, in_module=context) if only_modules: # In the case of an import like `from x.` we don't need to diff --git a/test/test_evaluate/test_implicit_namespace_package.py b/test/test_evaluate/test_implicit_namespace_package.py index 9bc90987..e47c255e 100644 --- a/test/test_evaluate/test_implicit_namespace_package.py +++ b/test/test_evaluate/test_implicit_namespace_package.py @@ -56,3 +56,34 @@ def test_implicit_nested_namespace_package(): result = script.goto_definitions() assert len(result) == 1 + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_implicit_namespace_package_import_autocomplete(): + CODE = 'from implicit_name' + + sys_path = [dirname(__file__)] + + script = jedi.Script(sys_path=sys_path, source=CODE) + compl = script.completions() + assert [c.name for c in compl] == ['implicit_namespace_package'] + + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_namespace_package_in_multiple_directories_autocompletion(): + CODE = 'from pkg.' + sys_path = [join(dirname(__file__), d) + for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']] + + script = jedi.Script(sys_path=sys_path, source=CODE) + compl = script.completions() + assert set(c.name for c in compl) == {'ns1_file', 'ns2_file'} + + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_namespace_package_in_multiple_directories_goto_definition(): + CODE = 'from pkg import ns1_file' + sys_path = [join(dirname(__file__), d) + for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']] + script = jedi.Script(sys_path=sys_path, source=CODE) + result = script.goto_definitions() + assert len(result) == 1 From a2031d89b1a9014e48a6fdc64b3633f17d389e36 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 18:24:38 +0100 Subject: [PATCH 02/14] Fix tests --- jedi/_compatibility.py | 3 ++- test/test_evaluate/test_implicit_namespace_package.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 30fe705c..ab00f7d2 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -166,10 +166,11 @@ def _iter_modules(paths, prefix=''): for path in paths: importer = pkgutil.get_importer(path) - if importer.path is None or not os.path.isdir(importer.path): + if importer is None or importer.path is None or not os.path.isdir(importer.path): return yielded = {} + import inspect try: filenames = os.listdir(importer.path) diff --git a/test/test_evaluate/test_implicit_namespace_package.py b/test/test_evaluate/test_implicit_namespace_package.py index e47c255e..cfd31ecf 100644 --- a/test/test_evaluate/test_implicit_namespace_package.py +++ b/test/test_evaluate/test_implicit_namespace_package.py @@ -76,7 +76,7 @@ def test_namespace_package_in_multiple_directories_autocompletion(): script = jedi.Script(sys_path=sys_path, source=CODE) compl = script.completions() - assert set(c.name for c in compl) == {'ns1_file', 'ns2_file'} + assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file']) @pytest.mark.skipif('sys.version_info[:2] < (3,4)') From 7f21fdfbc791cc97b3ee1399ed2a3dde1dc81d15 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 19:10:15 +0100 Subject: [PATCH 03/14] Fallback --- jedi/_compatibility.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index ab00f7d2..2a662f32 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -163,10 +163,15 @@ if the module is contained in a package. def _iter_modules(paths, prefix=''): # Copy of pkgutil.iter_modules adapted to work with namespaces + for path in paths: importer = pkgutil.get_importer(path) - if importer is None or importer.path is None or not os.path.isdir(importer.path): + if not isinstance(importer, importlib.machinery.FileFinder): + yield from pkgutil.iter_modules([path], prefix) + continue + + if importer.path is None or not os.path.isdir(importer.path): return yielded = {} From ff65cf8ebe9191cc3bb14aa9658e8af202f1de5d Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 19:14:12 +0100 Subject: [PATCH 04/14] Use compatible syntax --- jedi/_compatibility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 2a662f32..387a076d 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -168,7 +168,8 @@ def _iter_modules(paths, prefix=''): importer = pkgutil.get_importer(path) if not isinstance(importer, importlib.machinery.FileFinder): - yield from pkgutil.iter_modules([path], prefix) + for mod_info in pkgutil.iter_modules([path], prefix): + yield mod_info continue if importer.path is None or not os.path.isdir(importer.path): From 26774c79fb6c0dd39afc6eedcfc4d46c3439be37 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 10 Feb 2018 21:21:25 +0100 Subject: [PATCH 05/14] Add a module cache that has a bit more capabilites --- jedi/evaluate/__init__.py | 3 +-- jedi/evaluate/context/module.py | 4 ++-- jedi/evaluate/imports.py | 23 ++++++++++++++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 29438d31..e61b144b 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -95,8 +95,7 @@ class Evaluator(object): self.latest_grammar = parso.load_grammar(version='3.6') self.memoize_cache = {} # for memoize decorators - # To memorize modules -> equals `sys.modules`. - self.modules = {} # like `sys.modules`. + self.module_cache = imports.ModuleCache() # does the job of `sys.modules`. self.compiled_cache = {} # see `evaluate.compiled.create()` self.inferred_element_counts = {} self.mixed_cache = {} # see `evaluate.compiled.mixed._create()` diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 78d80df4..cedaed63 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -65,7 +65,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): # I'm not sure if the star import cache is really that effective anymore # with all the other really fast import caches. Recheck. Also we would need - # to push the star imports into Evaluator.modules, if we reenable this. + # to push the star imports into Evaluator.module_cache, if we reenable this. @evaluator_method_cache([]) def star_imports(self): modules = [] @@ -115,7 +115,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): return None def py__name__(self): - for name, module in self.evaluator.modules.items(): + for name, module in self.evaluator.module_cache.iterate_modules_with_names(): if module == self and name != '': return name diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 586458a6..5f976b49 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -31,6 +31,23 @@ from jedi.evaluate.filters import AbstractNameDefinition from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS +class ModuleCache(): + def __init__(self): + self._path_cache = {} + self._name_cache = {} + + def add(self, module, name): + path = module.py__file__() + self._path_cache[path] = module + self._name_cache[name] = module + + def iterate_modules_with_names(self): + return self._name_cache.items() + + def get(self, name): + return self._name_cache[name] + + # This memoization is needed, because otherwise we will infinitely loop on # certain imports. @evaluator_method_cache(default=NO_CONTEXTS) @@ -289,7 +306,7 @@ class Importer(object): module_name = '.'.join(import_parts) try: - return ContextSet(self._evaluator.modules[module_name]) + return ContextSet(self._evaluator.module_cache.get(module_name)) except KeyError: pass @@ -367,7 +384,7 @@ class Importer(object): # importable. return NO_CONTEXTS - self._evaluator.modules[module_name] = module + self._evaluator.module_cache.add(module, module_name) return ContextSet(module) def _generate_name(self, name, in_module=None): @@ -481,7 +498,7 @@ def add_module(evaluator, module_name, module): # the sepatator dots for nested packages. Therefore we return # `__main__` in ModuleWrapper.py__name__(), which is similar to # Python behavior. - evaluator.modules[module_name] = module + evaluator.module_cache.add(module, module_name) def get_modules_containing_name(evaluator, modules, name): From 514eaf89c39a7683fe5a64e4add6888626f83f7d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 12 Feb 2018 20:33:48 +0100 Subject: [PATCH 06/14] Prepare a test to eventually solve a relative import problem --- jedi/evaluate/imports.py | 7 ++++--- test/completion/__init__.py | 6 ++++++ test/completion/imports.py | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 5f976b49..eabb26cc 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -31,7 +31,7 @@ from jedi.evaluate.filters import AbstractNameDefinition from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS -class ModuleCache(): +class ModuleCache(object): def __init__(self): self._path_cache = {} self._name_cache = {} @@ -487,9 +487,10 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N cache_path=settings.cache_directory) from jedi.evaluate.context import ModuleContext - return ModuleContext(evaluator, module_node, path=path) + module = ModuleContext(evaluator, module_node, path=path) else: - return compiled.load_module(evaluator, path=path, sys_path=sys_path) + module = compiled.load_module(evaluator, path=path, sys_path=sys_path) + return module def add_module(evaluator, module_name, module): diff --git a/test/completion/__init__.py b/test/completion/__init__.py index 374dd947..3c1a4bb3 100644 --- a/test/completion/__init__.py +++ b/test/completion/__init__.py @@ -1,3 +1,9 @@ """ needed for some modules to test against packages. """ some_variable = 1 + + +from . import imports +# TODO this is not correct +#? +imports.relative() diff --git a/test/completion/imports.py b/test/completion/imports.py index 1afeb327..69e96cda 100644 --- a/test/completion/imports.py +++ b/test/completion/imports.py @@ -293,3 +293,4 @@ def relative(): from import_tree.pkg.mod1 import foobar #? int() foobar + return 1 From 9fec494e84095d439ecf6629486759e3b70a01d2 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 12 Feb 2018 20:39:42 +0100 Subject: [PATCH 07/14] Unify load_module access --- jedi/evaluate/imports.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index eabb26cc..2f339309 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -374,10 +374,8 @@ class Importer(object): fullname=module_path.name, paths=module_path.paths, ) - elif code is not None or module_path.endswith(('.py', '.zip', '.egg')): - module = _load_module(self._evaluator, module_path, code, sys_path, parent_module) else: - module = compiled.load_module(self._evaluator, path=module_path, sys_path=sys_path) + module = _load_module(self._evaluator, module_path, code, sys_path, parent_module) if module is None: # The file might raise an ImportError e.g. and therefore not be From a33cbc8ae3a729a53ef8b9cc0a5a818f9f6ae87e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 12 Feb 2018 20:49:45 +0100 Subject: [PATCH 08/14] Try to put all module loading in one place including namespace packages --- jedi/evaluate/imports.py | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 2f339309..6ef78df2 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -367,15 +367,9 @@ class Importer(object): _add_error(self.module_context, import_path[-1]) return NO_CONTEXTS - if isinstance(module_path, ImplicitNSInfo): - from jedi.evaluate.context.namespace import ImplicitNamespaceContext - module = ImplicitNamespaceContext( - self._evaluator, - fullname=module_path.name, - paths=module_path.paths, - ) - else: - module = _load_module(self._evaluator, module_path, code, sys_path, parent_module) + module = _load_module( + self._evaluator, module_path, code, sys_path, parent_module + ) if module is None: # The file might raise an ImportError e.g. and therefore not be @@ -473,21 +467,29 @@ class Importer(object): def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None): - if sys_path is None: - sys_path = evaluator.get_sys_path() - - dotted_path = path and dotted_from_fs_path(path, sys_path) - if path is not None and path.endswith(('.py', '.zip', '.egg')) \ - and dotted_path not in settings.auto_import_modules: - - module_node = evaluator.parse( - code=code, path=path, cache=True, diff_cache=True, - cache_path=settings.cache_directory) - - from jedi.evaluate.context import ModuleContext - module = ModuleContext(evaluator, module_node, path=path) + if isinstance(path, ImplicitNSInfo): + from jedi.evaluate.context.namespace import ImplicitNamespaceContext + module = ImplicitNamespaceContext( + evaluator, + fullname=path.name, + paths=path.paths, + ) else: - module = compiled.load_module(evaluator, path=path, sys_path=sys_path) + if sys_path is None: + sys_path = evaluator.get_sys_path() + + dotted_path = path and dotted_from_fs_path(path, sys_path) + if path is not None and path.endswith(('.py', '.zip', '.egg')) \ + and dotted_path not in settings.auto_import_modules: + + module_node = evaluator.parse( + code=code, path=path, cache=True, diff_cache=True, + cache_path=settings.cache_directory) + + from jedi.evaluate.context import ModuleContext + module = ModuleContext(evaluator, module_node, path=path) + else: + module = compiled.load_module(evaluator, path=path, sys_path=sys_path) return module From a52b6edd01c17f6b9ea7a5306f0d2c3bda7993d5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 12 Feb 2018 21:17:21 +0100 Subject: [PATCH 09/14] Better module loading --- jedi/evaluate/imports.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 6ef78df2..a349647e 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -368,7 +368,9 @@ class Importer(object): return NO_CONTEXTS module = _load_module( - self._evaluator, module_path, code, sys_path, parent_module + self._evaluator, module_path, code, sys_path, parent_module, + module_name=module_name, + safe_module_name=True, ) if module is None: @@ -376,7 +378,6 @@ class Importer(object): # importable. return NO_CONTEXTS - self._evaluator.module_cache.add(module, module_name) return ContextSet(module) def _generate_name(self, name, in_module=None): @@ -466,7 +467,8 @@ class Importer(object): return names -def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None): +def _load_module(evaluator, path=None, code=None, sys_path=None, + parent_module=None, module_name=None, safe_module_name=False): if isinstance(path, ImplicitNSInfo): from jedi.evaluate.context.namespace import ImplicitNamespaceContext module = ImplicitNamespaceContext( @@ -490,15 +492,18 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N module = ModuleContext(evaluator, module_node, path=path) else: module = compiled.load_module(evaluator, path=path, sys_path=sys_path) + add_module(evaluator, module_name, module, safe=safe_module_name) return module -def add_module(evaluator, module_name, module): - if '.' not in module_name: - # We cannot add paths with dots, because that would collide with - # the sepatator dots for nested packages. Therefore we return - # `__main__` in ModuleWrapper.py__name__(), which is similar to - # Python behavior. +def add_module(evaluator, module_name, module, safe=False): + if module_name is not None: + if not safe and '.' not in module_name: + # We cannot add paths with dots, because that would collide with + # the sepatator dots for nested packages. Therefore we return + # `__main__` in ModuleWrapper.py__name__(), which is similar to + # Python behavior. + return evaluator.module_cache.add(module, module_name) @@ -536,13 +541,11 @@ def get_modules_containing_name(evaluator, modules, name): code = python_bytes_to_unicode(f.read(), errors='replace') if name in code: e_sys_path = evaluator.get_sys_path() - module = _load_module(evaluator, path, code, sys_path=e_sys_path) - - module_name = sys_path.dotted_path_in_sys_path( - e_sys_path, path + module_name = sys_path.dotted_path_in_sys_path(e_sys_path, path) + module = _load_module( + evaluator, path, code, + sys_path=e_sys_path, module_name=module_name ) - if module_name is not None: - add_module(evaluator, module_name, module) return module # skip non python modules From 36699b77b24db197acddc461112092354255b04a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 13 Feb 2018 19:19:00 +0100 Subject: [PATCH 10/14] DOn't check the parser cache, that's parso's responsibility --- jedi/evaluate/context/module.py | 2 -- jedi/evaluate/imports.py | 16 +--------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index cedaed63..ad12b9ef 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -210,5 +210,3 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): return "<%s: %s@%s-%s>" % ( self.__class__.__name__, self._string_name, self.tree_node.start_pos[0], self.tree_node.end_pos[0]) - - diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index a349647e..d0a50ac7 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -15,7 +15,6 @@ import os from parso.python import tree from parso.tree import search_ancestor -from parso.cache import parser_cache from parso import python_bytes_to_unicode from jedi._compatibility import unicode, ImplicitNSInfo, force_unicode @@ -523,19 +522,6 @@ def get_modules_containing_name(evaluator, modules, name): if file_name.endswith('.py'): yield path - def check_python_file(path): - try: - # TODO I don't think we should use the cache here?! - node_cache_item = parser_cache[evaluator.grammar._hashed][path] - except KeyError: - try: - return check_fs(path) - except IOError: - return None - else: - module_node = node_cache_item.node - return ModuleContext(evaluator, module_node, path=path) - def check_fs(path): with open(path, 'rb') as f: code = python_bytes_to_unicode(f.read(), errors='replace') @@ -570,6 +556,6 @@ def get_modules_containing_name(evaluator, modules, name): # Sort here to make issues less random. for p in sorted(paths): # make testing easier, sort it - same results on every interpreter - m = check_python_file(p) + m = check_fs(p) if m is not None and not isinstance(m, compiled.CompiledObject): yield m From 2a56323c163b64644a2e2bc65b6bc560a6dcf70c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 13 Feb 2018 20:47:43 +0100 Subject: [PATCH 11/14] Try to avoid CachedMetaClass for modules --- jedi/api/__init__.py | 3 ++- jedi/evaluate/context/module.py | 5 ++--- jedi/evaluate/imports.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 92d0b43e..6354de3c 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -286,9 +286,10 @@ class Script(object): def _analysis(self): self._evaluator.is_analysis = True self._evaluator.analysis_modules = [self._module_node] + module = self._get_module() try: for node in get_executable_nodes(self._module_node): - context = self._get_module().create_context(node) + context = module.create_context(node) if node.type in ('funcdef', 'classdef'): # Resolve the decorators. tree_name_to_contexts(self._evaluator, context, node.children[1]) diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index ad12b9ef..56d9d93c 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -5,8 +5,7 @@ import os from parso import python_bytes_to_unicode -from jedi._compatibility import use_metaclass -from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache +from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \ AbstractNameDefinition, ParserTreeFilter, DictFilter from jedi.evaluate import compiled @@ -41,7 +40,7 @@ class ModuleName(ContextNameMixin, AbstractNameDefinition): return self._name -class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): +class ModuleContext(TreeContext): api_type = u'module' parent_context = None diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index d0a50ac7..dfa88e01 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -46,6 +46,9 @@ class ModuleCache(object): def get(self, name): return self._name_cache[name] + def get_from_path(self, path): + return self._path_cache[path] + # This memoization is needed, because otherwise we will infinitely loop on # certain imports. @@ -468,6 +471,15 @@ class Importer(object): def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None, module_name=None, safe_module_name=False): + try: + return evaluator.module_cache.get(module_name) + except KeyError: + pass + try: + return evaluator.module_cache.get_from_path(path) + except KeyError: + pass + if isinstance(path, ImplicitNSInfo): from jedi.evaluate.context.namespace import ImplicitNamespaceContext module = ImplicitNamespaceContext( From 276f2d0b522595159392a4d0b307d8988b7d34a3 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 14 Feb 2018 20:42:53 +0100 Subject: [PATCH 12/14] parent_module is not needed for loading modules --- jedi/evaluate/imports.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index dfa88e01..78a05e1f 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -355,7 +355,6 @@ class Importer(object): _add_error(self.module_context, import_path[-1]) return NO_CONTEXTS else: - parent_module = None debug.dbg('search_module %s in %s', import_parts[-1], self.file_path) # Override the sys.path. It works only good that way. # Injecting the path directly into `find_module` did not work. @@ -370,7 +369,7 @@ class Importer(object): return NO_CONTEXTS module = _load_module( - self._evaluator, module_path, code, sys_path, parent_module, + self._evaluator, module_path, code, sys_path, module_name=module_name, safe_module_name=True, ) @@ -470,7 +469,7 @@ class Importer(object): def _load_module(evaluator, path=None, code=None, sys_path=None, - parent_module=None, module_name=None, safe_module_name=False): + module_name=None, safe_module_name=False): try: return evaluator.module_cache.get(module_name) except KeyError: @@ -522,7 +521,6 @@ def get_modules_containing_name(evaluator, modules, name): """ Search a name in the directories of modules. """ - from jedi.evaluate.context import ModuleContext def check_directories(paths): for p in paths: if p is not None: From 76df356628be8d3c32fbe4ae19f4c062ec033827 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 15 Feb 2018 14:10:01 +0100 Subject: [PATCH 13/14] Relative imports should be working again even when used in more special occasions. Fixes #973 There are more fixes needed. Some things are just very unclean and might lead to further bugs. --- jedi/evaluate/imports.py | 2 +- test/completion/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 78a05e1f..b5e197c1 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -213,7 +213,7 @@ class Importer(object): if level: base = module_context.py__package__().split('.') - if base == ['']: + if base == [''] or base == ['__main__']: base = [] if level > len(base): path = module_context.py__file__() diff --git a/test/completion/__init__.py b/test/completion/__init__.py index 3c1a4bb3..dc4d7252 100644 --- a/test/completion/__init__.py +++ b/test/completion/__init__.py @@ -4,6 +4,5 @@ some_variable = 1 from . import imports -# TODO this is not correct -#? +#? int() imports.relative() From fa9364307ff72734f5677f4d4a937d389a44eb3f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 15 Feb 2018 20:25:07 +0100 Subject: [PATCH 14/14] Add comments to implicit namespaces and fix some minor things. See #1005. --- jedi/_compatibility.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 31975cae..73a981a1 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -159,10 +159,14 @@ def _iter_modules(paths, prefix=''): importer = pkgutil.get_importer(path) if not isinstance(importer, importlib.machinery.FileFinder): + # We're only modifying the case for FileFinder. All the other cases + # still need to be checked (like zip-importing). Do this by just + # calling the pkgutil version. for mod_info in pkgutil.iter_modules([path], prefix): yield mod_info continue + # START COPY OF pkutils._iter_file_finder_modules. if importer.path is None or not os.path.isdir(importer.path): return @@ -177,12 +181,12 @@ def _iter_modules(paths, prefix=''): filenames.sort() # handle packages before same-named modules for fn in filenames: - # Avoid traversing special directories - if fn.startswith(('__', '.')): + modname = inspect.getmodulename(fn) + if modname == '__init__' or modname in yielded: continue - modname = inspect.getmodulename(fn) - if modname in yielded: + # jedi addition: Avoid traversing special directories + if fn.startswith('.') or fn == '__pycache__': continue path = os.path.join(importer.path, fn) @@ -190,16 +194,19 @@ def _iter_modules(paths, prefix=''): if not modname and os.path.isdir(path) and '.' not in fn: modname = fn + # A few jedi modifications: Don't check if there's an + # __init__.py try: - dircontents = os.listdir(path) + os.listdir(path) except OSError: # ignore unreadable directories like import does - dircontents = [] + continue ispkg = True if modname and '.' not in modname: yielded[modname] = 1 yield importer, prefix + modname, ispkg + # END COPY iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules