WIP import improvement, getting rid of bad old code

jedi/_compatibility.py

@@ -81,32 +81,30 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s
     if loader is None:
         raise ImportError("Couldn't find a loader for {}".format(string))
 
-    try:
-        is_package = loader.is_package(string)
-        if is_package:
-            if hasattr(loader, 'path'):
-                module_path = os.path.dirname(loader.path)
-            else:
-                # At least zipimporter does not have path attribute
-                module_path = os.path.dirname(loader.get_filename(string))
-            if hasattr(loader, 'archive'):
-                module_file = DummyFile(loader, string)
-            else:
-                module_file = None
+    is_package = loader.is_package(string)
+    if is_package:
+        if hasattr(loader, 'path'):
+            module_path = os.path.dirname(loader.path)
         else:
+            # At least zipimporter does not have path attribute
+            module_path = os.path.dirname(loader.get_filename(string))
+        if hasattr(loader, 'archive'):
+            module_file = DummyFile(loader, string)
+        else:
+            module_file = None
+    else:
+        try:
             module_path = loader.get_filename(string)
             module_file = DummyFile(loader, string)
-    except AttributeError:
-        # ExtensionLoader has not attribute get_filename, instead it has a
-        # path attribute that we can use to retrieve the module path
-        try:
-            module_path = loader.path
-            module_file = DummyFile(loader, string)
         except AttributeError:
-            module_path = string
-            module_file = None
-        finally:
-            is_package = False
+            # ExtensionLoader has not attribute get_filename, instead it has a
+            # path attribute that we can use to retrieve the module path
+            try:
+                module_path = loader.path
+                module_file = DummyFile(loader, string)
+            except AttributeError:
+                module_path = string
+                module_file = None
 
     if hasattr(loader, 'archive'):
         module_path = loader.archive

jedi/api/__init__.py

@@ -150,15 +150,21 @@ class Script(object):
     @cache.memoize_method
     def _get_module(self):
         names = ('__main__',)
+        is_package = False
         if self.path is not None:
-            import_names = transform_path_to_dotted(self._evaluator.get_sys_path(), self.path)
+            import_names, is_p = transform_path_to_dotted(
+                self._evaluator.get_sys_path(),
+                self.path
+            )
             if import_names is not None:
                 names = import_names
+                is_package = is_p
 
         module = ModuleContext(
             self._evaluator, self._module_node, cast_path(self.path),
             string_names=names,
             code_lines=self._code_lines,
+            is_package=is_package,
         )
         self._evaluator.module_cache.add(names, ContextSet([module]))
         return module

jedi/api/interpreter.py

@@ -21,6 +21,7 @@ class NamespaceObject(object):
 
 
 class MixedModuleContext(Context):
+    # TODO use ContextWrapper!
     type = 'mixed_module'
 
     def __init__(self, evaluator, tree_module, namespaces, path, code_lines):

jedi/evaluate/compiled/mixed.py

@@ -211,6 +211,7 @@ def _create(evaluator, access_handle, parent_context, *args):
             path=path,
             string_names=string_names,
             code_lines=code_lines,
+            is_package=hasattr(compiled_object, 'py__path__'),
         )
         if name is not None:
             evaluator.module_cache.add(string_names, ContextSet([module_context]))

jedi/evaluate/context/module.py

@@ -139,7 +139,7 @@ class ModuleContext(ModuleMixin, TreeContext):
     api_type = u'module'
     parent_context = None
 
-    def __init__(self, evaluator, module_node, path, string_names, code_lines):
+    def __init__(self, evaluator, module_node, path, string_names, code_lines, is_package=False):
         super(ModuleContext, self).__init__(
             evaluator,
             parent_context=None,
@@ -148,19 +148,7 @@ class ModuleContext(ModuleMixin, TreeContext):
         self._path = path
         self.string_names = string_names
         self.code_lines = code_lines
-
-    def _get_init_directory(self):
-        """
-        :return: The path to the directory of a package. None in case it's not
-                 a package.
-        """
-        for suffix in all_suffixes() + ['.pyi']:
-            ending = '__init__' + suffix
-            py__file__ = self.py__file__()
-            if py__file__ is not None and py__file__.endswith(ending):
-                # Remove the ending, including the separator.
-                return self.py__file__()[:-len(ending) - 1]
-        return None
+        self.is_package = is_package
 
     def py__name__(self):
         if self.string_names is None:
@@ -176,39 +164,36 @@ class ModuleContext(ModuleMixin, TreeContext):
 
         return os.path.abspath(self._path)
 
-    def is_package(self):
-        return self._get_init_directory() is not None
-
     def py__package__(self):
-        if self._get_init_directory() is None:
-            return re.sub(r'\.?[^.]+$', '', self.py__name__()).split('.')
-        else:
+        if self.is_package:
             return self.string_names
+        return self.string_names[:-1]
 
     def _py__path__(self):
-        search_path = self.evaluator.get_sys_path()
-        init_path = self.py__file__()
-        if os.path.basename(init_path) in ('__init__.py', '__init__.pyi'):
-            with open(init_path, 'rb') as f:
-                content = python_bytes_to_unicode(f.read(), errors='replace')
-                # these are strings that need to be used for namespace packages,
-                # the first one is ``pkgutil``, the second ``pkg_resources``.
-                options = ('declare_namespace(__name__)', 'extend_path(__path__')
-                if options[0] in content or options[1] in content:
-                    # It is a namespace, now try to find the rest of the
-                    # modules on sys_path or whatever the search_path is.
-                    paths = set()
-                    for s in search_path:
-                        other = os.path.join(s, self.name.string_name)
-                        if os.path.isdir(other):
-                            paths.add(other)
-                    if paths:
-                        return list(paths)
-                    # TODO I'm not sure if this is how nested namespace
-                    # packages work. The tests are not really good enough to
-                    # show that.
-        # Default to this.
-        return [self._get_init_directory()]
+        # A namespace package is typically auto generated and ~10 lines long.
+        first_few_lines = ''.join(self.code_lines[:50])
+        # these are strings that need to be used for namespace packages,
+        # the first one is ``pkgutil``, the second ``pkg_resources``.
+        options = ('declare_namespace(__name__)', 'extend_path(__path__')
+        if options[0] in first_few_lines or options[1] in first_few_lines:
+            # It is a namespace, now try to find the rest of the
+            # modules on sys_path or whatever the search_path is.
+            paths = set()
+            for s in self.evaluator.get_sys_path():
+                other = os.path.join(s, self.name.string_name)
+                if os.path.isdir(other):
+                    paths.add(other)
+            if paths:
+                return list(paths)
+            # Nested namespace packages will not be supported. Nobody ever
+            # asked for it and in Python 3 they are there without using all the
+            # crap above.
+
+        # Default to the of this file.
+        file = self.py__file__()
+        if file is None:
+            return None
+        return os.path.dirname(file)
 
     @property
     def py__path__(self):
@@ -222,7 +207,7 @@ class ModuleContext(ModuleMixin, TreeContext):
         is a list of paths (strings).
         Raises an AttributeError if the module is not a package.
         """
-        if self.is_package():
+        if self.is_package:
             return self._py__path__
         else:
             raise AttributeError('Only packages have __path__ attributes.')

jedi/evaluate/gradual/typeshed.py

@@ -140,7 +140,7 @@ def import_module_decorator(func):
         if len(import_names) == 1:
             map_ = _cache_stub_file_map(evaluator.grammar.version_info)
         elif isinstance(parent_module_context, StubModuleContext):
-            if not parent_module_context.stub_context.is_package():
+            if not parent_module_context.stub_context.is_package:
                 # Only if it's a package (= a folder) something can be
                 # imported.
                 return context_set
@@ -161,6 +161,7 @@ def import_module_decorator(func):
                         module_cls = TypingModuleWrapper
                     else:
                         module_cls = StubOnlyModuleContext
+                    file_name = os.path.basename(path)
                     stub_module_context = module_cls(
                         context_set, evaluator, stub_module_node,
                         path=path,
@@ -168,6 +169,7 @@ def import_module_decorator(func):
                         # The code was loaded with latest_grammar, so use
                         # that.
                         code_lines=get_cached_code_lines(evaluator.latest_grammar, path),
+                        is_package=file_name == '__init__.pyi',
                     )
                     modules = _merge_modules(context_set, stub_module_context)
                     return ContextSet(modules)

jedi/evaluate/imports.py

@@ -249,6 +249,7 @@ class Importer(object):
         if level:
             base = module_context.py__package__()
             if base == [''] or base == ['__main__']:
+                raise NotImplementedError(module_context.py__package__())
                 base = []
             # We need to care for two cases, the first one is if it's a valid
             # Python import. This import has a properly defined module name
@@ -487,6 +488,7 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
         evaluator, module_path, code, sys_path,
         import_names=import_names,
         safe_module_name=True,
+        is_package=is_pkg,
     )
 
     if parent_module_context is None:
@@ -497,7 +499,7 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
 
 
 def _load_module(evaluator, path=None, code=None, sys_path=None,
-                 import_names=None, safe_module_name=False):
+                 import_names=None, safe_module_name=False, is_package=False):
     if import_names is None:
         dotted_name = None
     else:
@@ -530,6 +532,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None,
                 path=path,
                 string_names=import_names,
                 code_lines=get_cached_code_lines(evaluator.grammar, path),
+                is_package=is_package,
             )
         else:
             assert dotted_name is not None
@@ -561,19 +564,23 @@ def get_modules_containing_name(evaluator, modules, name):
             code = python_bytes_to_unicode(f.read(), errors='replace')
             if name in code:
                 e_sys_path = evaluator.get_sys_path()
-                module_name = os.path.basename(path)
-                if module_name.endswith('.py'):
-                    module_name = module_name[:-3]
-
                 if base_names:
+                    module_name = os.path.basename(path)
+                    module_name = sys_path.remove_python_path_suffix(module_name)
+                    is_package = module_name == '__init__'
+                    if is_package:
+                        raise NotImplementedError(
+                            "This is probably not possible yet, please add a failing test first")
+                        module_name = os.path.basename(os.path.dirname(path))
                     import_names = base_names + (module_name,)
                 else:
-                    import_names = sys_path.transform_path_to_dotted(e_sys_path, path)
+                    import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
 
                 module = _load_module(
                     evaluator, path, code,
                     sys_path=e_sys_path,
                     import_names=import_names,
+                    is_package=is_package,
                 )
                 evaluator.module_cache.add(import_names, ContextSet([module]))
                 return module
@@ -590,10 +597,7 @@ def get_modules_containing_name(evaluator, modules, name):
             if path is not None:
                 if path not in used_mod_paths:
                     used_mod_paths.add(path)
-                    string_names = m.string_names
-                    if not m.is_package() and string_names is not None:
-                        string_names = string_names[:-1]
-                    path_with_names_to_be_checked.append((path, string_names))
+                    path_with_names_to_be_checked.append((path, m.py__package__()))
         yield m
 
     if not settings.dynamic_params_for_other_modules:

jedi/evaluate/sys_path.py

@@ -197,25 +197,33 @@ def _get_buildout_script_paths(search_path):
             continue
 
 
+def remove_python_path_suffix(path):
+    for suffix in all_suffixes():
+        if path.endswith(suffix):
+            path = path[:-len(suffix)]
+            break
+    return path
+
+
 def transform_path_to_dotted(sys_path, module_path):
     """
     Returns the dotted path inside a sys.path as a list of names. e.g.
 
     >>> transform_path_to_dotted(["/foo"], '/foo/bar/baz.py')
-    ('bar', 'baz')
+    ('bar', 'baz'), False
 
-    Returns None if the path doesn't really resolve to anything.
+    Returns (None, False) if the path doesn't really resolve to anything.
+    The second return part is if it is a package.
     """
     # First remove the suffix.
-    for suffix in all_suffixes():
-        if module_path.endswith(suffix):
-            module_path = module_path[:-len(suffix)]
-            break
+    module_path = remove_python_path_suffix(module_path)
+
     # Once the suffix was removed we are using the files as we know them. This
     # means that if someone uses an ending like .vim for a Python file, .vim
     # will be part of the returned dotted part.
 
-    if module_path.endswith(os.path.sep + '__init__'):
+    is_package = module_path.endswith(os.path.sep + '__init__')
+    if is_package:
         # -1 to remove the separator
         module_path = module_path[:-len('__init__') - 1]
 
@@ -231,6 +239,6 @@ def transform_path_to_dotted(sys_path, module_path):
                 split = rest.split(os.path.sep)
                 for string in split:
                     if not string:
-                        return None
-                return tuple(split)
-    return None
+                        return None, False
+                return tuple(split), is_package
+    return None, False

test/test_evaluate/test_imports.py

@@ -36,9 +36,11 @@ def test_find_module_not_package():
     assert is_package is False
 
 
+pkg_zip_path = os.path.join(os.path.dirname(__file__), 'zipped_imports/pkg.zip')
+
+
 def test_find_module_package_zipped(Script, evaluator, environment):
-    path = os.path.join(os.path.dirname(__file__), 'zipped_imports/pkg.zip')
-    sys_path = environment.get_sys_path() + [path]
+    sys_path = environment.get_sys_path() + [pkg_zip_path]
     script = Script('import pkg; pkg.mod', sys_path=sys_path)
     assert len(script.completions()) == 1
 
@@ -52,6 +54,16 @@ def test_find_module_package_zipped(Script, evaluator, environment):
     assert is_package is True
 
 
+def test_correct_zip_package_behavior(Script, evaluator, environment):
+    sys_path = environment.get_sys_path() + [pkg_zip_path]
+    pkg, = Script('import pkg', sys_path=sys_path).goto_definitions()
+    context, = pkg._name.infer()
+    assert context.py__file__() == pkg_zip_path
+    assert context.is_package is True
+    assert context.py__package__() == ('pkg',)
+    assert context.py__path__() == [pkg_zip_path]
+
+
 def test_find_module_not_package_zipped(Script, evaluator, environment):
     path = os.path.join(os.path.dirname(__file__), 'zipped_imports/not_pkg.zip')
     sys_path = environment.get_sys_path() + [path]

test/test_evaluate/test_sys_path.py

@@ -67,30 +67,31 @@ _s = ['/a', '/b', '/c/d/']
 
 
 @pytest.mark.parametrize(
-    'sys_path_, module_path, result', [
-        (_s, '/a/b', ('b',)),
-        (_s, '/a/b/c', ('b', 'c')),
-        (_s, '/a/b.py', ('b',)),
-        (_s, '/a/b/c.py', ('b', 'c')),
-        (_s, '/x/b.py', None),
-        (_s, '/c/d/x.py', ('x',)),
-        (_s, '/c/d/x.py', ('x',)),
-        (_s, '/c/d/x/y.py', ('x', 'y')),
+    'sys_path_, module_path, expected, is_package', [
+        (_s, '/a/b', ('b',), False),
+        (_s, '/a/b/c', ('b', 'c'), False),
+        (_s, '/a/b.py', ('b',), False),
+        (_s, '/a/b/c.py', ('b', 'c'), False),
+        (_s, '/x/b.py', None, False),
+        (_s, '/c/d/x.py', ('x',), False),
+        (_s, '/c/d/x.py', ('x',), False),
+        (_s, '/c/d/x/y.py', ('x', 'y'), False),
         # If dots are in there they also resolve. These are obviously illegal
         # in Python, but Jedi can handle them. Give the user a bit more freedom
         # that he will have to correct eventually.
-        (_s, '/a/b.c.py', ('b.c',)),
-        (_s, '/a/b.d/foo.bar.py', ('b.d', 'foo.bar')),
+        (_s, '/a/b.c.py', ('b.c',), False),
+        (_s, '/a/b.d/foo.bar.py', ('b.d', 'foo.bar'), False),
 
-        (_s, '/a/.py', None),
-        (_s, '/a/c/.py', None),
+        (_s, '/a/.py', None, False),
+        (_s, '/a/c/.py', None, False),
 
-        (['/foo'], '/foo/bar/__init__.py', ('bar',)),
-        (['/foo'], '/foo/bar/baz/__init__.py', ('bar', 'baz')),
-        (['/foo'], '/foo/bar.so', ('bar',)),
-        (['/foo'], '/foo/bar/__init__.so', ('bar',)),
-        (['/foo'], '/x/bar.py', None),
-        (['/foo'], '/foo/bar.xyz', ('bar.xyz',)),
+        (['/foo'], '/foo/bar/__init__.py', ('bar',), True),
+        (['/foo'], '/foo/bar/baz/__init__.py', ('bar', 'baz'), True),
+        (['/foo'], '/foo/bar.so', ('bar',), False),
+        (['/foo'], '/foo/bar/__init__.so', ('bar',), True),
+        (['/foo'], '/x/bar.py', None, False),
+        (['/foo'], '/foo/bar.xyz', ('bar.xyz',), False),
     ])
-def test_calculate_dotted_from_path(sys_path_, module_path, result):
-    assert sys_path.transform_path_to_dotted(sys_path_, module_path) == result
+def test_calculate_dotted_from_path(sys_path_, module_path, expected, is_package):
+    assert sys_path.transform_path_to_dotted(sys_path_, module_path) \
+        == (expected, is_package)