Merge the implicit namespace improvement (pkgutils.itermodules modification)

There are still a few issues that need to be addressed.

jedi/_compatibility.py

@@ -152,6 +152,58 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if not isinstance(importer, importlib.machinery.FileFinder):
+            for mod_info in pkgutil.iter_modules([path], prefix):
+                yield mod_info
+            continue
+
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+
+        import inspect
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            # Avoid traversing special directories
+            if fn.startswith(('__', '.')):
+                continue
+
+            modname = inspect.getmodulename(fn)
+            if modname in yielded:
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
 class ImplicitNSInfo(object):
     """Stores information returned from an implicit namespace spec"""
     def __init__(self, name, paths):

jedi/evaluate/compiled/subprocess/functions.py

@@ -1,9 +1,8 @@
 import sys
 import os
 import imp
-import pkgutil
 
-from jedi._compatibility import find_module, cast_path, force_unicode
+from jedi._compatibility import find_module, cast_path, force_unicode, iter_modules
 from jedi.evaluate.compiled import access
 from jedi import parser_utils
 
@@ -71,7 +70,7 @@ def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
 def list_module_names(evaluator, search_path):
     return [
         name
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path)
+        for module_loader, name, is_pkg in iter_modules(search_path)
     ]
 
 

jedi/evaluate/context/module.py

@@ -1,4 +1,3 @@
-import pkgutil
 import imp
 import re
 import os
@@ -6,6 +5,7 @@ import os
 from parso import python_bytes_to_unicode
 
 from jedi.evaluate.cache import evaluator_method_cache
+from jedi._compatibility import iter_modules
 from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
     AbstractNameDefinition, ParserTreeFilter, DictFilter
 from jedi.evaluate import compiled
@@ -188,7 +188,7 @@ class ModuleContext(TreeContext):
         path = self._path
         names = {}
         if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            mods = iter_modules([os.path.dirname(path)])
             for module_loader, name, is_pkg in mods:
                 # It's obviously a relative import to the current module.
                 names[name] = SubModuleName(self, name)

jedi/evaluate/imports.py

@@ -393,6 +393,7 @@ class Importer(object):
         and not names defined in the files.
         """
         sub = self._evaluator.compiled_subprocess
+
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
@@ -440,7 +441,7 @@ class Importer(object):
                 # implicit namespace packages
                 elif isinstance(context, ImplicitNamespaceContext):
                     paths = context.paths
-                    names += self._get_module_names(paths)
+                    names += self._get_module_names(paths, in_module=context)
 
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to