From 78cbad0d08f4793479ed1b9493272c23b6574a68 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Wed, 29 Nov 2017 17:05:31 +0100 Subject: [PATCH 1/4] Fix implicit namespace autocompletion. Resolves: #959 --- AUTHORS.txt | 1 + docs/docs/features.rst | 3 +- jedi/_compatibility.py | 45 +++++++++++++++++++ jedi/evaluate/context/module.py | 4 +- jedi/evaluate/imports.py | 7 ++- .../test_implicit_namespace_package.py | 31 +++++++++++++ 6 files changed, 83 insertions(+), 8 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index 29f46765..e0d6110a 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -46,5 +46,6 @@ Simon Ruggier (@sruggier) Élie Gouzien (@ElieGouzien) Robin Roth (@robinro) Malte Plath (@langsamer) +Maksim Novikov (@m-novikov) Note: (@user) means a github user name. diff --git a/docs/docs/features.rst b/docs/docs/features.rst index aef32708..83935f39 100644 --- a/docs/docs/features.rst +++ b/docs/docs/features.rst @@ -53,7 +53,7 @@ Supported Python Features case, that doesn't work with |jedi|) - simple/usual ``sys.path`` modifications - ``isinstance`` checks for if/while/assert -- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces) +- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces) - Django / Flask / Buildout support @@ -64,7 +64,6 @@ Not yet implemented: - manipulations of instances outside the instance variables without using methods -- implicit namespace packages (Python 3.3+, `PEP 420 `_) Will probably never be implemented: diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 52a20fe2..30fe705c 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -161,6 +161,51 @@ if the module is contained in a package. """ +def _iter_modules(paths, prefix=''): + # Copy of pkgutil.iter_modules adapted to work with namespaces + for path in paths: + importer = pkgutil.get_importer(path) + + if importer.path is None or not os.path.isdir(importer.path): + return + + yielded = {} + import inspect + try: + filenames = os.listdir(importer.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + # Avoid traversing special directories + if fn.startswith(('__', '.')): + continue + + modname = inspect.getmodulename(fn) + if modname in yielded: + continue + + path = os.path.join(importer.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + ispkg = True + + if modname and '.' not in modname: + yielded[modname] = 1 + yield importer, prefix + modname, ispkg + +iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules + + class ImplicitNSInfo(object): """Stores information returned from an implicit namespace spec""" def __init__(self, name, paths): diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 5ba92cdb..f5fa9f0f 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -5,7 +5,7 @@ import os from parso import python_bytes_to_unicode -from jedi._compatibility import use_metaclass +from jedi._compatibility import use_metaclass, iter_modules from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \ AbstractNameDefinition, ParserTreeFilter, DictFilter @@ -188,7 +188,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): path = self._path names = {} if path is not None and path.endswith(os.path.sep + '__init__.py'): - mods = pkgutil.iter_modules([os.path.dirname(path)]) + mods = iter_modules([os.path.dirname(path)]) for module_loader, name, is_pkg in mods: # It's obviously a relative import to the current module. names[name] = SubModuleName(self, name) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 89dd833f..8bc74c06 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -21,7 +21,7 @@ from parso.tree import search_ancestor from parso.cache import parser_cache from parso import python_bytes_to_unicode -from jedi._compatibility import find_module, unicode, ImplicitNSInfo +from jedi._compatibility import find_module, unicode, ImplicitNSInfo, iter_modules from jedi import debug from jedi import settings from jedi.evaluate import sys_path @@ -401,7 +401,6 @@ class Importer(object): Get the names of all modules in the search_path. This means file names and not names defined in the files. """ - names = [] # add builtin module names if search_path is None and in_module is None: @@ -409,7 +408,7 @@ class Importer(object): if search_path is None: search_path = self.sys_path_with_modifications() - for module_loader, name, is_pkg in pkgutil.iter_modules(search_path): + for module_loader, name, is_pkg in iter_modules(search_path): names.append(self._generate_name(name, in_module=in_module)) return names @@ -448,7 +447,7 @@ class Importer(object): # implicit namespace packages elif isinstance(context, ImplicitNamespaceContext): paths = context.paths - names += self._get_module_names(paths) + names += self._get_module_names(paths, in_module=context) if only_modules: # In the case of an import like `from x.` we don't need to diff --git a/test/test_evaluate/test_implicit_namespace_package.py b/test/test_evaluate/test_implicit_namespace_package.py index 9bc90987..e47c255e 100644 --- a/test/test_evaluate/test_implicit_namespace_package.py +++ b/test/test_evaluate/test_implicit_namespace_package.py @@ -56,3 +56,34 @@ def test_implicit_nested_namespace_package(): result = script.goto_definitions() assert len(result) == 1 + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_implicit_namespace_package_import_autocomplete(): + CODE = 'from implicit_name' + + sys_path = [dirname(__file__)] + + script = jedi.Script(sys_path=sys_path, source=CODE) + compl = script.completions() + assert [c.name for c in compl] == ['implicit_namespace_package'] + + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_namespace_package_in_multiple_directories_autocompletion(): + CODE = 'from pkg.' + sys_path = [join(dirname(__file__), d) + for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']] + + script = jedi.Script(sys_path=sys_path, source=CODE) + compl = script.completions() + assert set(c.name for c in compl) == {'ns1_file', 'ns2_file'} + + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_namespace_package_in_multiple_directories_goto_definition(): + CODE = 'from pkg import ns1_file' + sys_path = [join(dirname(__file__), d) + for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']] + script = jedi.Script(sys_path=sys_path, source=CODE) + result = script.goto_definitions() + assert len(result) == 1 From a2031d89b1a9014e48a6fdc64b3633f17d389e36 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 18:24:38 +0100 Subject: [PATCH 2/4] Fix tests --- jedi/_compatibility.py | 3 ++- test/test_evaluate/test_implicit_namespace_package.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 30fe705c..ab00f7d2 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -166,10 +166,11 @@ def _iter_modules(paths, prefix=''): for path in paths: importer = pkgutil.get_importer(path) - if importer.path is None or not os.path.isdir(importer.path): + if importer is None or importer.path is None or not os.path.isdir(importer.path): return yielded = {} + import inspect try: filenames = os.listdir(importer.path) diff --git a/test/test_evaluate/test_implicit_namespace_package.py b/test/test_evaluate/test_implicit_namespace_package.py index e47c255e..cfd31ecf 100644 --- a/test/test_evaluate/test_implicit_namespace_package.py +++ b/test/test_evaluate/test_implicit_namespace_package.py @@ -76,7 +76,7 @@ def test_namespace_package_in_multiple_directories_autocompletion(): script = jedi.Script(sys_path=sys_path, source=CODE) compl = script.completions() - assert set(c.name for c in compl) == {'ns1_file', 'ns2_file'} + assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file']) @pytest.mark.skipif('sys.version_info[:2] < (3,4)') From 7f21fdfbc791cc97b3ee1399ed2a3dde1dc81d15 Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 19:10:15 +0100 Subject: [PATCH 3/4] Fallback --- jedi/_compatibility.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index ab00f7d2..2a662f32 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -163,10 +163,15 @@ if the module is contained in a package. def _iter_modules(paths, prefix=''): # Copy of pkgutil.iter_modules adapted to work with namespaces + for path in paths: importer = pkgutil.get_importer(path) - if importer is None or importer.path is None or not os.path.isdir(importer.path): + if not isinstance(importer, importlib.machinery.FileFinder): + yield from pkgutil.iter_modules([path], prefix) + continue + + if importer.path is None or not os.path.isdir(importer.path): return yielded = {} From ff65cf8ebe9191cc3bb14aa9658e8af202f1de5d Mon Sep 17 00:00:00 2001 From: Maxim Novikov Date: Tue, 2 Jan 2018 19:14:12 +0100 Subject: [PATCH 4/4] Use compatible syntax --- jedi/_compatibility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 2a662f32..387a076d 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -168,7 +168,8 @@ def _iter_modules(paths, prefix=''): importer = pkgutil.get_importer(path) if not isinstance(importer, importlib.machinery.FileFinder): - yield from pkgutil.iter_modules([path], prefix) + for mod_info in pkgutil.iter_modules([path], prefix): + yield mod_info continue if importer.path is None or not os.path.isdir(importer.path):