Merge the implicit namespace improvement (pkgutils.itermodules modification)

There are still a few issues that need to be addressed.

AUTHORS.txt

@@ -47,5 +47,6 @@ Simon Ruggier (@sruggier)
 Robin Roth (@robinro)
 Malte Plath (@langsamer)
 Anton Zub (@zabulazza)
+Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
 
 Note: (@user) means a github user name.

docs/docs/features.rst

@@ -53,7 +53,7 @@ Supported Python Features
   case, that doesn't work with |jedi|)
 - simple/usual ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
-- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 
 
@@ -64,7 +64,6 @@ Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
-- implicit namespace packages (Python 3.4+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 

jedi/_compatibility.py

@@ -152,6 +152,58 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if not isinstance(importer, importlib.machinery.FileFinder):
+            for mod_info in pkgutil.iter_modules([path], prefix):
+                yield mod_info
+            continue
+
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+
+        import inspect
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            # Avoid traversing special directories
+            if fn.startswith(('__', '.')):
+                continue
+
+            modname = inspect.getmodulename(fn)
+            if modname in yielded:
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
 class ImplicitNSInfo(object):
     """Stores information returned from an implicit namespace spec"""
     def __init__(self, name, paths):

jedi/evaluate/compiled/subprocess/functions.py

@@ -1,9 +1,8 @@
 import sys
 import os
 import imp
-import pkgutil
 
-from jedi._compatibility import find_module, cast_path, force_unicode
+from jedi._compatibility import find_module, cast_path, force_unicode, iter_modules
 from jedi.evaluate.compiled import access
 from jedi import parser_utils
 
@@ -71,7 +70,7 @@ def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
 def list_module_names(evaluator, search_path):
     return [
         name
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path)
+        for module_loader, name, is_pkg in iter_modules(search_path)
     ]
 
 

jedi/evaluate/context/module.py

@@ -1,4 +1,3 @@
-import pkgutil
 import imp
 import re
 import os
@@ -6,6 +5,7 @@ import os
 from parso import python_bytes_to_unicode
 
 from jedi.evaluate.cache import evaluator_method_cache
+from jedi._compatibility import iter_modules
 from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
     AbstractNameDefinition, ParserTreeFilter, DictFilter
 from jedi.evaluate import compiled
@@ -188,7 +188,7 @@ class ModuleContext(TreeContext):
         path = self._path
         names = {}
         if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            mods = iter_modules([os.path.dirname(path)])
             for module_loader, name, is_pkg in mods:
                 # It's obviously a relative import to the current module.
                 names[name] = SubModuleName(self, name)

jedi/evaluate/imports.py

@@ -393,6 +393,7 @@ class Importer(object):
         and not names defined in the files.
         """
         sub = self._evaluator.compiled_subprocess
+
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
@@ -440,7 +441,7 @@ class Importer(object):
                 # implicit namespace packages
                 elif isinstance(context, ImplicitNamespaceContext):
                     paths = context.paths
-                    names += self._get_module_names(paths)
+                    names += self._get_module_names(paths, in_module=context)
 
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to

test/test_evaluate/test_implicit_namespace_package.py

@@ -3,10 +3,13 @@ from os.path import dirname, join
 import pytest
 
 
-def test_implicit_namespace_package(Script, environment):
+@pytest.fixture(autouse=True)
+def skip_not_supported_versions(environment):
     if environment.version_info < (3, 4):
         pytest.skip()
 
+
+def test_implicit_namespace_package(Script):
     sys_path = [join(dirname(__file__), d)
                 for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
 
@@ -47,10 +50,7 @@ def test_implicit_namespace_package(Script, environment):
         assert completion.description == solution
 
 
-def test_implicit_nested_namespace_package(Script, environment):
+def test_implicit_nested_namespace_package(Script):
-    if environment.version_info < (3, 4):
-        pytest.skip()
-
     code = 'from implicit_nested_namespaces.namespace.pkg.module import CONST'
 
     sys_path = [dirname(__file__)]
@@ -64,3 +64,32 @@ def test_implicit_nested_namespace_package(Script, environment):
     implicit_pkg, = Script(code, column=10, sys_path=sys_path).goto_definitions()
     assert implicit_pkg.type == 'module'
     assert implicit_pkg.module_path is None
+
+
+def test_implicit_namespace_package_import_autocomplete(Script):
+    CODE = 'from implicit_name'
+
+    sys_path = [dirname(__file__)]
+
+    script = Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert [c.name for c in compl] == ['implicit_namespace_package']
+
+
+def test_namespace_package_in_multiple_directories_autocompletion(Script):
+    CODE = 'from pkg.'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+
+    script = Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file'])
+
+
+def test_namespace_package_in_multiple_directories_goto_definition(Script):
+    CODE = 'from pkg import ns1_file'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+    script = Script(sys_path=sys_path, source=CODE)
+    result = script.goto_definitions()
+    assert len(result) == 1