added support for implicit ns packages and added tests

AUTHORS.txt

@@ -41,6 +41,7 @@ Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
 Cristi Burcă (@scribu)
 bstaint (@bstaint)
 Mathias Rav (@Mortal) <rav@cs.au.dk>
+Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 
 
 Note: (@user) means a github user name.

jedi/_compatibility.py

@@ -34,8 +34,29 @@ class DummyFile(object):
         del self.loader
 
 
-def find_module_py33(string, path=None):
-    loader = importlib.machinery.PathFinder.find_module(string, path)
+def find_module_py34(string, path=None, fullname=None):
+    implicit_namespace_pkg = False
+    spec = None
+    loader = None
+
+    spec = importlib.machinery.PathFinder.find_spec(string, path)
+    if hasattr(spec, 'origin'):
+        origin = spec.origin
+        implicit_namespace_pkg = origin == 'namespace'
+
+    # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs
+    if implicit_namespace_pkg:
+        fullname = string if not path else fullname
+        implicit_ns_info = ImplicitNSInfo(fullname, spec.submodule_search_locations._path)
+        return None, implicit_ns_info, False
+
+    # we have found the tail end of the dotted path
+    if hasattr(spec, 'loader'):
+        loader = spec.loader
+    return find_module_py33(string, path, loader)
+
+def find_module_py33(string, path=None, loader=None, fullname=None):
+    loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
         try:
@@ -81,7 +102,7 @@ def find_module_py33(string, path=None):
     return module_file, module_path, is_package
 
 
-def find_module_pre_py33(string, path=None):
+def find_module_pre_py33(string, path=None, fullname=None):
     try:
         module_file, module_path, description = imp.find_module(string, path)
         module_type = description[2]
@@ -121,6 +142,7 @@ def find_module_pre_py33(string, path=None):
 
 
 find_module = find_module_py33 if is_py33 else find_module_pre_py33
+find_module = find_module_py34 if is_py34  else find_module
 find_module.__doc__ = """
 Provides information about a module.
 
@@ -132,6 +154,12 @@ if the module is contained in a package.
 """
 
 
+class ImplicitNSInfo(object):
+    """Stores information returned from an implicit namespace spec"""
+    def __init__(self, name, paths):
+        self.name = name
+        self.paths = paths
+
 # unicode function
 try:
     unicode = unicode

jedi/evaluate/imports.py

@@ -16,7 +16,7 @@ import os
 import pkgutil
 import sys
 
-from jedi._compatibility import find_module, unicode
+from jedi._compatibility import find_module, unicode, ImplicitNSInfo
 from jedi import debug
 from jedi import settings
 from jedi.common import source_to_unicode, unite
@@ -306,8 +306,10 @@ class Importer(object):
                     # At the moment we are only using one path. So this is
                     # not important to be correct.
                     try:
+                        if not isinstance(path, list):
+                            path = [path]
                         module_file, module_path, is_pkg = \
-                            find_module(import_parts[-1], [path])
+                            find_module(import_parts[-1], path, fullname=module_name)
                         break
                     except ImportError:
                         module_path = None
@@ -323,7 +325,7 @@ class Importer(object):
                 sys.path, temp = sys_path, sys.path
                 try:
                     module_file, module_path, is_pkg = \
-                        find_module(import_parts[-1])
+                        find_module(import_parts[-1], fullname=module_name)
                 finally:
                     sys.path = temp
             except ImportError:
@@ -343,7 +345,12 @@ class Importer(object):
             source = module_file.read()
             module_file.close()
 
-        if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
+        if isinstance(module_path, ImplicitNSInfo):
+            from jedi.evaluate.representation import ImplicitNamespaceContext
+            fullname, paths = module_path.name, module_path.paths
+            module = ImplicitNamespaceContext(self._evaluator, fullname=fullname)
+            module.paths = paths
+        elif module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
             module = compiled.load_module(self._evaluator, module_path)
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path, parent_module)
@@ -384,7 +391,7 @@ class Importer(object):
         :param only_modules: Indicates wheter it's possible to import a
             definition that is not defined in a module.
         """
-        from jedi.evaluate.representation import ModuleContext
+        from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext
         names = []
         if self.import_path:
             # flask
@@ -405,13 +412,16 @@ class Importer(object):
                 # Non-modules are not completable.
                 if context.api_type != 'module':  # not a module
                     continue
-
                 # namespace packages
-                if isinstance(context, ModuleContext) and \
-                        context.py__file__().endswith('__init__.py'):
+                if isinstance(context, ModuleContext) and context.py__file__().endswith('__init__.py'):
                     paths = context.py__path__()
                     names += self._get_module_names(paths, in_module=context)
 
+                # implicit namespace packages
+                elif isinstance(context, ImplicitNamespaceContext):
+                    paths = context.paths
+                    names += self._get_module_names(paths)
+
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to
                     # add all the variables.

jedi/evaluate/representation.py

@@ -39,6 +39,7 @@ import os
 import pkgutil
 import imp
 import re
+from itertools import chain
 
 from jedi._compatibility import use_metaclass
 from jedi.parser import tree
@@ -557,3 +558,65 @@ class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)):
 
     def py__class__(self):
         return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')
+
+
+class ImplicitNSName(AbstractNameDefinition):
+    """
+    Accessing names for implicit namespace packages should infer to nothing.
+    This object will prevent Jedi from raising exceptions
+    """
+    def __init__(self, implicit_ns_context, string_name):
+        self.implicit_ns_context = implicit_ns_context
+        self.string_name = string_name
+
+    def infer(self):
+        return []
+
+    def get_root_context(self):
+        return self.implicit_ns_context
+
+
+class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, context.TreeContext)):
+    """
+    Provides support for implicit namespace packages
+    """
+    api_type = 'module'
+    parent_context = None
+
+    def __init__(self, evaluator, fullname):
+        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
+        self.evaluator = evaluator
+        self.fullname = fullname
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield DictFilter(self._sub_modules_dict())
+
+    @property
+    @memoize_default()
+    def name(self):
+        string_name = self.py__package__().rpartition('.')[-1]
+        return ImplicitNSName(self, string_name)
+
+    def py__file__(self):
+        return None
+
+    def py__package__(self):
+        """Return the fullname
+        """
+        return self.fullname
+
+    @property
+    def py__path__(self):
+        return lambda: [self.paths]
+
+    @memoize_default()
+    def _sub_modules_dict(self):
+        names = {}
+
+        paths = self.paths
+        file_names = chain.from_iterable(os.listdir(path) for path in paths)
+        mods = [file_name.rpartition('.')[0] if '.' in file_name else file_name for file_name in file_names]
+
+        for name in mods:
+            names[name] = imports.SubModuleName(self, name)
+        return names

test/test_evaluate/implicit_namespace_package/ns1/pkg/ns1_file.py

@@ -0,0 +1 @@
+foo = 'ns1_file!'

test/test_evaluate/implicit_namespace_package/ns2/pkg/ns2_file.py

@@ -0,0 +1 @@
+foo = 'ns2_file!'

test/test_evaluate/implicit_nested_namespaces/namespace/pkg/module.py

@@ -0,0 +1 @@
+CONST = 1

test/test_evaluate/test_implicit_namespace_package.py

@@ -0,0 +1,58 @@
+from os.path import dirname, join
+
+import jedi
+import pytest
+
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_implicit_namespace_package():
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+
+    def script_with_path(*args, **kwargs):
+        return jedi.Script(sys_path=sys_path, *args, **kwargs)
+
+    # goto definition
+    assert script_with_path('from pkg import ns1_file').goto_definitions()
+    assert script_with_path('from pkg import ns2_file').goto_definitions()
+    assert not script_with_path('from pkg import ns3_file').goto_definitions()
+
+    # goto assignment
+    tests = {
+        'from pkg.ns2_file import foo': 'ns2_file!',
+        'from pkg.ns1_file import foo': 'ns1_file!',
+    }
+    for source, solution in tests.items():
+        ass = script_with_path(source).goto_assignments()
+        assert len(ass) == 1
+        assert ass[0].description == "foo = '%s'" % solution
+
+    # completion
+    completions = script_with_path('from pkg import ').completions()
+    names = [str(c.name) for c in completions]  # str because of unicode
+    compare = ['ns1_file', 'ns2_file']
+    # must at least contain these items, other items are not important
+    assert set(compare) == set(names)
+
+    tests = {
+        'from pkg import ns2_file as x': 'ns2_file!',
+        'from pkg import ns1_file as x': 'ns1_file!'
+    }
+    for source, solution in tests.items():
+        for c in script_with_path(source + '; x.').completions():
+            if c.name == 'foo':
+                completion = c
+        solution = "foo = '%s'" % solution
+        assert completion.description == solution
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_implicit_nested_namespace_package():
+    CODE = 'from implicit_nested_namespaces.namespace.pkg.module import CONST'
+
+    sys_path = [dirname(__file__)]
+
+    script = jedi.Script(sys_path=sys_path, source=CODE, line=1, column=61)
+
+    result = script.goto_definitions()
+
+    assert len(result) == 1