Fix implicit namespace autocompletion. Resolves: #959

AUTHORS.txt

@@ -46,5 +46,6 @@ Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
 Robin Roth (@robinro)
 Malte Plath (@langsamer)
+Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
 
 Note: (@user) means a github user name.

docs/docs/features.rst

@@ -53,7 +53,7 @@ Supported Python Features
   case, that doesn't work with |jedi|)
 - simple/usual ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
-- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 
 
@@ -64,7 +64,6 @@ Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
-- implicit namespace packages (Python 3.3+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 

jedi/_compatibility.py

@@ -161,6 +161,51 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+        import inspect
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            # Avoid traversing special directories
+            if fn.startswith(('__', '.')):
+                continue
+
+            modname = inspect.getmodulename(fn)
+            if modname in yielded:
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
 class ImplicitNSInfo(object):
     """Stores information returned from an implicit namespace spec"""
     def __init__(self, name, paths):

jedi/evaluate/context/module.py

@@ -5,7 +5,7 @@ import os
 
 from parso import python_bytes_to_unicode
 
-from jedi._compatibility import use_metaclass
+from jedi._compatibility import use_metaclass, iter_modules
 from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache
 from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
     AbstractNameDefinition, ParserTreeFilter, DictFilter
@@ -188,7 +188,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
         path = self._path
         names = {}
         if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            mods = iter_modules([os.path.dirname(path)])
             for module_loader, name, is_pkg in mods:
                 # It's obviously a relative import to the current module.
                 names[name] = SubModuleName(self, name)

jedi/evaluate/imports.py

@@ -21,7 +21,7 @@ from parso.tree import search_ancestor
 from parso.cache import parser_cache
 from parso import python_bytes_to_unicode
 
-from jedi._compatibility import find_module, unicode, ImplicitNSInfo
+from jedi._compatibility import find_module, unicode, ImplicitNSInfo, iter_modules
 from jedi import debug
 from jedi import settings
 from jedi.evaluate import sys_path
@@ -401,7 +401,6 @@ class Importer(object):
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
@@ -409,7 +408,7 @@ class Importer(object):
 
         if search_path is None:
             search_path = self.sys_path_with_modifications()
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
+        for module_loader, name, is_pkg in iter_modules(search_path):
             names.append(self._generate_name(name, in_module=in_module))
         return names
 
@@ -448,7 +447,7 @@ class Importer(object):
                 # implicit namespace packages
                 elif isinstance(context, ImplicitNamespaceContext):
                     paths = context.paths
-                    names += self._get_module_names(paths)
+                    names += self._get_module_names(paths, in_module=context)
 
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to

test/test_evaluate/test_implicit_namespace_package.py

@@ -56,3 +56,34 @@ def test_implicit_nested_namespace_package():
     result = script.goto_definitions()
 
     assert len(result) == 1
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_implicit_namespace_package_import_autocomplete():
+    CODE = 'from implicit_name'
+
+    sys_path = [dirname(__file__)]
+
+    script = jedi.Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert [c.name for c in compl] == ['implicit_namespace_package']
+
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_namespace_package_in_multiple_directories_autocompletion():
+    CODE = 'from pkg.'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+
+    script = jedi.Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert set(c.name for c in compl) == {'ns1_file', 'ns2_file'}
+
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_namespace_package_in_multiple_directories_goto_definition():
+    CODE = 'from pkg import ns1_file'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+    script = jedi.Script(sys_path=sys_path, source=CODE)
+    result = script.goto_definitions()
+    assert len(result) == 1