Merge remote-tracking branch 'origin/master'

AUTHORS.txt

@@ -47,5 +47,6 @@ Simon Ruggier (@sruggier)
 Robin Roth (@robinro)
 Malte Plath (@langsamer)
 Anton Zub (@zabulazza)
+Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
 
 Note: (@user) means a github user name.

docs/docs/features.rst

@@ -53,7 +53,7 @@ Supported Python Features
   case, that doesn't work with |jedi|)
 - simple/usual ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
-- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 
 
@@ -64,7 +64,6 @@ Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
-- implicit namespace packages (Python 3.4+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 

jedi/_compatibility.py

@@ -152,6 +152,65 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if not isinstance(importer, importlib.machinery.FileFinder):
+            # We're only modifying the case for FileFinder. All the other cases
+            # still need to be checked (like zip-importing). Do this by just
+            # calling the pkgutil version.
+            for mod_info in pkgutil.iter_modules([path], prefix):
+                yield mod_info
+            continue
+
+        # START COPY OF pkutils._iter_file_finder_modules.
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+
+        import inspect
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            modname = inspect.getmodulename(fn)
+            if modname == '__init__' or modname in yielded:
+                continue
+
+            # jedi addition: Avoid traversing special directories
+            if fn.startswith('.') or fn == '__pycache__':
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                # A few jedi modifications: Don't check if there's an
+                # __init__.py
+                try:
+                    os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    continue
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+        # END COPY
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
 class ImplicitNSInfo(object):
     """Stores information returned from an implicit namespace spec"""
     def __init__(self, name, paths):

jedi/api/__init__.py

@@ -286,9 +286,10 @@ class Script(object):
     def _analysis(self):
         self._evaluator.is_analysis = True
         self._evaluator.analysis_modules = [self._module_node]
+        module = self._get_module()
         try:
             for node in get_executable_nodes(self._module_node):
-                context = self._get_module().create_context(node)
+                context = module.create_context(node)
                 if node.type in ('funcdef', 'classdef'):
                     # Resolve the decorators.
                     tree_name_to_contexts(self._evaluator, context, node.children[1])

jedi/evaluate/__init__.py

@@ -95,8 +95,7 @@ class Evaluator(object):
 
         self.latest_grammar = parso.load_grammar(version='3.6')
         self.memoize_cache = {}  # for memoize decorators
-        # To memorize modules -> equals `sys.modules`.
+        self.module_cache = imports.ModuleCache()  # does the job of `sys.modules`.
-        self.modules = {}  # like `sys.modules`.
         self.compiled_cache = {}  # see `evaluate.compiled.create()`
         self.inferred_element_counts = {}
         self.mixed_cache = {}  # see `evaluate.compiled.mixed._create()`

jedi/evaluate/compiled/subprocess/functions.py

@@ -1,9 +1,8 @@
 import sys
 import os
 import imp
-import pkgutil
 
-from jedi._compatibility import find_module, cast_path, force_unicode
+from jedi._compatibility import find_module, cast_path, force_unicode, iter_modules
 from jedi.evaluate.compiled import access
 from jedi import parser_utils
 
@@ -71,7 +70,7 @@ def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
 def list_module_names(evaluator, search_path):
     return [
         name
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path)
+        for module_loader, name, is_pkg in iter_modules(search_path)
     ]
 
 

jedi/evaluate/context/module.py

@@ -1,12 +1,11 @@
-import pkgutil
 import imp
 import re
 import os
 
 from parso import python_bytes_to_unicode
 
-from jedi._compatibility import use_metaclass
+from jedi.evaluate.cache import evaluator_method_cache
-from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache
+from jedi._compatibility import iter_modules
 from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
     AbstractNameDefinition, ParserTreeFilter, DictFilter
 from jedi.evaluate import compiled
@@ -41,7 +40,7 @@ class ModuleName(ContextNameMixin, AbstractNameDefinition):
         return self._name
 
 
-class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
+class ModuleContext(TreeContext):
     api_type = u'module'
     parent_context = None
 
@@ -65,7 +64,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
 
     # I'm not sure if the star import cache is really that effective anymore
     # with all the other really fast import caches. Recheck. Also we would need
-    # to push the star imports into Evaluator.modules, if we reenable this.
+    # to push the star imports into Evaluator.module_cache, if we reenable this.
     @evaluator_method_cache([])
     def star_imports(self):
         modules = []
@@ -115,7 +114,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
         return None
 
     def py__name__(self):
-        for name, module in self.evaluator.modules.items():
+        for name, module in self.evaluator.module_cache.iterate_modules_with_names():
             if module == self and name != '':
                 return name
 
@@ -189,7 +188,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
         path = self._path
         names = {}
         if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            mods = iter_modules([os.path.dirname(path)])
             for module_loader, name, is_pkg in mods:
                 # It's obviously a relative import to the current module.
                 names[name] = SubModuleName(self, name)
@@ -210,5 +209,3 @@ class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)):
         return "<%s: %s@%s-%s>" % (
             self.__class__.__name__, self._string_name,
             self.tree_node.start_pos[0], self.tree_node.end_pos[0])
-
-

jedi/evaluate/imports.py

@@ -15,7 +15,6 @@ import os
 
 from parso.python import tree
 from parso.tree import search_ancestor
-from parso.cache import parser_cache
 from parso import python_bytes_to_unicode
 
 from jedi._compatibility import unicode, ImplicitNSInfo, force_unicode
@@ -31,6 +30,26 @@ from jedi.evaluate.filters import AbstractNameDefinition
 from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
 
 
+class ModuleCache(object):
+    def __init__(self):
+        self._path_cache = {}
+        self._name_cache = {}
+
+    def add(self, module, name):
+        path = module.py__file__()
+        self._path_cache[path] = module
+        self._name_cache[name] = module
+
+    def iterate_modules_with_names(self):
+        return self._name_cache.items()
+
+    def get(self, name):
+        return self._name_cache[name]
+
+    def get_from_path(self, path):
+        return self._path_cache[path]
+
+
 # This memoization is needed, because otherwise we will infinitely loop on
 # certain imports.
 @evaluator_method_cache(default=NO_CONTEXTS)
@@ -194,7 +213,7 @@ class Importer(object):
 
         if level:
             base = module_context.py__package__().split('.')
-            if base == ['']:
+            if base == [''] or base == ['__main__']:
                 base = []
             if level > len(base):
                 path = module_context.py__file__()
@@ -289,7 +308,7 @@ class Importer(object):
 
         module_name = '.'.join(import_parts)
         try:
-            return ContextSet(self._evaluator.modules[module_name])
+            return ContextSet(self._evaluator.module_cache.get(module_name))
         except KeyError:
             pass
 
@@ -336,7 +355,6 @@ class Importer(object):
                     _add_error(self.module_context, import_path[-1])
                     return NO_CONTEXTS
         else:
-            parent_module = None
             debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
             # Override the sys.path. It works only good that way.
             # Injecting the path directly into `find_module` did not work.
@@ -350,24 +368,17 @@ class Importer(object):
                 _add_error(self.module_context, import_path[-1])
                 return NO_CONTEXTS
 
-        if isinstance(module_path, ImplicitNSInfo):
+        module = _load_module(
-            from jedi.evaluate.context.namespace import ImplicitNamespaceContext
+            self._evaluator, module_path, code, sys_path,
-            module = ImplicitNamespaceContext(
+            module_name=module_name,
-                self._evaluator,
+            safe_module_name=True,
-                fullname=module_path.name,
+        )
-                paths=module_path.paths,
-            )
-        elif code is not None or module_path.endswith(('.py', '.zip', '.egg')):
-            module = _load_module(self._evaluator, module_path, code, sys_path, parent_module)
-        else:
-            module = compiled.load_module(self._evaluator, path=module_path, sys_path=sys_path)
 
         if module is None:
             # The file might raise an ImportError e.g. and therefore not be
             # importable.
             return NO_CONTEXTS
 
-        self._evaluator.modules[module_name] = module
         return ContextSet(module)
 
     def _generate_name(self, name, in_module=None):
@@ -382,6 +393,7 @@ class Importer(object):
         and not names defined in the files.
         """
         sub = self._evaluator.compiled_subprocess
+
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
@@ -429,7 +441,7 @@ class Importer(object):
                 # implicit namespace packages
                 elif isinstance(context, ImplicitNamespaceContext):
                     paths = context.paths
-                    names += self._get_module_names(paths)
+                    names += self._get_module_names(paths, in_module=context)
 
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to
@@ -457,38 +469,59 @@ class Importer(object):
         return names
 
 
-def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None):
+def _load_module(evaluator, path=None, code=None, sys_path=None,
-    if sys_path is None:
+                 module_name=None, safe_module_name=False):
-        sys_path = evaluator.get_sys_path()
+    try:
+        return evaluator.module_cache.get(module_name)
+    except KeyError:
+        pass
+    try:
+        return evaluator.module_cache.get_from_path(path)
+    except KeyError:
+        pass
 
-    dotted_path = path and dotted_from_fs_path(path, sys_path)
+    if isinstance(path, ImplicitNSInfo):
-    if path is not None and path.endswith(('.py', '.zip', '.egg')) \
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
-            and dotted_path not in settings.auto_import_modules:
+        module = ImplicitNamespaceContext(
-
+            evaluator,
-        module_node = evaluator.parse(
+            fullname=path.name,
-            code=code, path=path, cache=True, diff_cache=True,
+            paths=path.paths,
-            cache_path=settings.cache_directory)
+        )
-
-        from jedi.evaluate.context import ModuleContext
-        return ModuleContext(evaluator, module_node, path=path)
     else:
-        return compiled.load_module(evaluator, path=path, sys_path=sys_path)
+        if sys_path is None:
+            sys_path = evaluator.get_sys_path()
+
+        dotted_path = path and dotted_from_fs_path(path, sys_path)
+        if path is not None and path.endswith(('.py', '.zip', '.egg')) \
+                and dotted_path not in settings.auto_import_modules:
+
+            module_node = evaluator.parse(
+                code=code, path=path, cache=True, diff_cache=True,
+                cache_path=settings.cache_directory)
+
+            from jedi.evaluate.context import ModuleContext
+            module = ModuleContext(evaluator, module_node, path=path)
+        else:
+            module = compiled.load_module(evaluator, path=path, sys_path=sys_path)
+    add_module(evaluator, module_name, module, safe=safe_module_name)
+    return module
 
 
-def add_module(evaluator, module_name, module):
+def add_module(evaluator, module_name, module, safe=False):
-    if '.' not in module_name:
+    if module_name is not None:
-        # We cannot add paths with dots, because that would collide with
+        if not safe and '.' not in module_name:
-        # the sepatator dots for nested packages. Therefore we return
+            # We cannot add paths with dots, because that would collide with
-        # `__main__` in ModuleWrapper.py__name__(), which is similar to
+            # the sepatator dots for nested packages. Therefore we return
-        # Python behavior.
+            # `__main__` in ModuleWrapper.py__name__(), which is similar to
-        evaluator.modules[module_name] = module
+            # Python behavior.
+            return
+        evaluator.module_cache.add(module, module_name)
 
 
 def get_modules_containing_name(evaluator, modules, name):
     """
     Search a name in the directories of modules.
     """
-    from jedi.evaluate.context import ModuleContext
     def check_directories(paths):
         for p in paths:
             if p is not None:
@@ -500,31 +533,16 @@ def get_modules_containing_name(evaluator, modules, name):
                     if file_name.endswith('.py'):
                         yield path
 
-    def check_python_file(path):
-        try:
-            # TODO I don't think we should use the cache here?!
-            node_cache_item = parser_cache[evaluator.grammar._hashed][path]
-        except KeyError:
-            try:
-                return check_fs(path)
-            except IOError:
-                return None
-        else:
-            module_node = node_cache_item.node
-            return ModuleContext(evaluator, module_node, path=path)
-
     def check_fs(path):
         with open(path, 'rb') as f:
             code = python_bytes_to_unicode(f.read(), errors='replace')
             if name in code:
                 e_sys_path = evaluator.get_sys_path()
-                module = _load_module(evaluator, path, code, sys_path=e_sys_path)
+                module_name = sys_path.dotted_path_in_sys_path(e_sys_path, path)
-
+                module = _load_module(
-                module_name = sys_path.dotted_path_in_sys_path(
+                    evaluator, path, code,
-                    e_sys_path, path
+                    sys_path=e_sys_path, module_name=module_name
                 )
-                if module_name is not None:
-                    add_module(evaluator, module_name, module)
                 return module
 
     # skip non python modules
@@ -549,6 +567,6 @@ def get_modules_containing_name(evaluator, modules, name):
     # Sort here to make issues less random.
     for p in sorted(paths):
         # make testing easier, sort it - same results on every interpreter
-        m = check_python_file(p)
+        m = check_fs(p)
         if m is not None and not isinstance(m, compiled.CompiledObject):
             yield m

test/completion/__init__.py

@@ -1,3 +1,8 @@
 """ needed for some modules to test against packages. """
 
 some_variable = 1
+
+
+from . import imports
+#? int()
+imports.relative()

test/completion/imports.py

@@ -293,3 +293,4 @@ def relative():
     from import_tree.pkg.mod1 import foobar
     #? int()
     foobar
+    return 1

test/test_evaluate/test_implicit_namespace_package.py

@@ -3,10 +3,13 @@ from os.path import dirname, join
 import pytest
 
 
-def test_implicit_namespace_package(Script, environment):
+@pytest.fixture(autouse=True)
+def skip_not_supported_versions(environment):
     if environment.version_info < (3, 4):
         pytest.skip()
 
+
+def test_implicit_namespace_package(Script):
     sys_path = [join(dirname(__file__), d)
                 for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
 
@@ -47,10 +50,7 @@ def test_implicit_namespace_package(Script, environment):
         assert completion.description == solution
 
 
-def test_implicit_nested_namespace_package(Script, environment):
+def test_implicit_nested_namespace_package(Script):
-    if environment.version_info < (3, 4):
-        pytest.skip()
-
     code = 'from implicit_nested_namespaces.namespace.pkg.module import CONST'
 
     sys_path = [dirname(__file__)]
@@ -64,3 +64,32 @@ def test_implicit_nested_namespace_package(Script, environment):
     implicit_pkg, = Script(code, column=10, sys_path=sys_path).goto_definitions()
     assert implicit_pkg.type == 'module'
     assert implicit_pkg.module_path is None
+
+
+def test_implicit_namespace_package_import_autocomplete(Script):
+    CODE = 'from implicit_name'
+
+    sys_path = [dirname(__file__)]
+
+    script = Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert [c.name for c in compl] == ['implicit_namespace_package']
+
+
+def test_namespace_package_in_multiple_directories_autocompletion(Script):
+    CODE = 'from pkg.'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+
+    script = Script(sys_path=sys_path, source=CODE)
+    compl = script.completions()
+    assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file'])
+
+
+def test_namespace_package_in_multiple_directories_goto_definition(Script):
+    CODE = 'from pkg import ns1_file'
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+    script = Script(sys_path=sys_path, source=CODE)
+    result = script.goto_definitions()
+    assert len(result) == 1