Better scanning for module names, now includes namespace packages and stubs

jedi/debug.py

@@ -108,9 +108,6 @@ def warning(message, *args, **kwargs):
         debug_function('RED', i + 'warning: ' + message)
 
 
-error = warning  # Just for compatibility with logging.
-
-
 def speed(name):
     if debug_function and enable_speed:
         now = time.time()

jedi/evaluate/context/module.py

@@ -1,5 +1,6 @@
 import re
 import os
+import logging
 
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.names import ContextNameMixin, AbstractNameDefinition
@@ -8,6 +9,8 @@ from jedi.evaluate import compiled
 from jedi.evaluate.base_context import TreeContext
 from jedi.evaluate.names import SubModuleName
 
+logger = logging.getLogger(__name__)
+
 
 class _ModuleAttributeName(AbstractNameDefinition):
     """
@@ -35,6 +38,32 @@ class ModuleName(ContextNameMixin, AbstractNameDefinition):
         return self._name
 
 
+def iter_module_names(evaluator, paths):
+    # Python modules/packages
+    for n in evaluator.compiled_subprocess.list_module_names(paths):
+        yield n
+
+    for path in paths:
+        try:
+            dirs = os.listdir(path)
+        except OSError:
+            # The file might not exist or reading it might lead to an error.
+            logger.error("Not possible to list directory: %s", path)
+            continue
+        for name in dirs:
+            # Namespaces
+            if os.path.isdir(os.path.join(path, name)):
+                # pycache is obviously not an interestin namespace. Also the
+                # name must be a valid identifier.
+                # TODO use str.isidentifier, once Python 2 is removed
+                if name != '__pycache__' and not re.search('\W|^\d', name):
+                    yield name
+            # Stub files
+            if name.endswith('.pyi'):
+                if name != '__init__.pyi':
+                    yield name[:-4]
+
+
 class SubModuleDictMixin(object):
     @evaluator_method_cache()
     def sub_modules_dict(self):
@@ -48,7 +77,7 @@ class SubModuleDictMixin(object):
         except AttributeError:
             pass
         else:
-            mods = self._iter_module_names(method())
+            mods = iter_module_names(self.evaluator, method())
             for name in mods:
                 # It's obviously a relative import to the current module.
                 names[name] = SubModuleName(self, name)
@@ -57,9 +86,6 @@ class SubModuleDictMixin(object):
         # add all the variables, this is only about submodules.
         return names
 
-    def _iter_module_names(self, path):
-        return self.evaluator.compiled_subprocess.list_module_names(path)
-
 
 class ModuleMixin(SubModuleDictMixin):
     def get_filters(self, search_global=False, until_position=None, origin_scope=None):

jedi/evaluate/gradual/stub_context.py

@@ -1,5 +1,3 @@
-import os
-
 from jedi.evaluate.base_context import ContextWrapper
 from jedi.evaluate.context.module import ModuleContext
 from jedi.evaluate.filters import ParserTreeFilter, \
@@ -61,17 +59,6 @@ class StubModuleContext(ModuleContext):
         for f in filters:
             yield f
 
-    def _iter_module_names(self, paths):
-        for path in paths:
-            dirs = os.listdir(path)
-            for name in dirs:
-                if os.path.isdir(os.path.join(path, name)):
-                    if name != '__pycache__':
-                        yield name
-                if name.endswith('.pyi'):
-                    if name != '__init__.pyi':
-                        yield name[:-4]
-
 
 class TypingModuleWrapper(StubModuleContext):
     def get_filters(self, *args, **kwargs):

jedi/evaluate/imports.py

@@ -32,6 +32,7 @@ from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.names import ImportName, SubModuleName
 from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
 from jedi.evaluate.gradual.typeshed import import_module_decorator
+from jedi.evaluate.context.module import iter_module_names
 
 
 class ModuleCache(object):
@@ -309,18 +310,16 @@ class Importer(object):
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-        sub = self._evaluator.compiled_subprocess
-
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
             names += [ImportName(self.module_context, name)
-                      for name in sub.get_builtin_module_names()]
+                      for name in self._evaluator.compiled_subprocess.get_builtin_module_names()]
 
         if search_path is None:
             search_path = self._sys_path_with_modifications()
 
-        for name in sub.list_module_names(search_path):
+        for name in iter_module_names(self._evaluator, search_path):
             if in_module is None:
                 n = ImportName(self.module_context, name)
             else:
@@ -361,7 +360,6 @@ class Importer(object):
 
             if not only_modules:
                 from jedi.evaluate.gradual.conversion import stub_to_actual_context_set
-                contexts = ContextSet([context])
                 both_contexts = ContextSet.from_sets(
                     stub_to_actual_context_set(context, ignore_compiled=True)
                     for context in contexts

test/test_api/test_classes.py

@@ -178,9 +178,9 @@ def test_hashlib_params(Script, environment):
     if environment.version_info < (3,):
         pytest.skip()
 
-    script = Script(source='from hashlib import ', line=1, column=20)
+    script = Script(source='from hashlib import sha256')
-    c = script.completions()
+    c, = script.completions()
-    assert c[2].params
+    assert [p.name for p in c.params] == ['arg']
 
 
 def test_signature_params(Script):