Fix test_imports tests, now zip imports work again

jedi/_compatibility.py

@@ -14,6 +14,9 @@ try:
     import importlib
 except ImportError:
     pass
+from zipimport import zipimporter
+
+from parso.file_io import KnownContentFileIO
 
 is_py3 = sys.version_info[0] >= 3
 is_py35 = is_py3 and sys.version_info[1] >= 5
@@ -55,7 +58,7 @@ def find_module_py34(string, path=None, full_name=None, is_global_search=True):
                 # This is a namespace package.
                 full_name = string if not path else full_name
                 implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
-                return None, implicit_ns_info, False
+                return implicit_ns_info, True
             break
 
     return find_module_py33(string, path, loader)
@@ -81,13 +84,9 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s
     if loader is None:
         raise ImportError("Couldn't find a loader for {}".format(string))
 
+    return _from_loader(loader, string)
     is_package = loader.is_package(string)
     if is_package:
-        if hasattr(loader, 'path'):
-            module_path = os.path.dirname(loader.path)
-        else:
-            # At least zipimporter does not have path attribute
-            module_path = os.path.dirname(loader.get_filename(string))
         if hasattr(loader, 'archive'):
             module_file = DummyFile(loader, string)
         else:
@@ -97,29 +96,61 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s
             module_path = loader.get_filename(string)
             module_file = DummyFile(loader, string)
         except AttributeError:
-            # ExtensionLoader has not attribute get_filename, instead it has a
-            # path attribute that we can use to retrieve the module path
             try:
-                module_path = loader.path
                 module_file = DummyFile(loader, string)
             except AttributeError:
                 module_path = string
                 module_file = None
 
-    if hasattr(loader, 'archive'):
-        module_path = loader.archive
-
     return module_file, module_path, is_package
 
 
-def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True):
+class ZipFileIO(KnownContentFileIO):
+    """For .zip and .egg archives"""
+    def __init__(self, path, code, zip_path):
+        super(ZipFileIO, self).__init__(path, code)
+        self._zip_path = zip_path
+
+    def get_last_modified(self):
+        return os.path.getmtime(self._zip_path)
+
+
+def _from_loader(loader, string):
+    is_package = loader.is_package(string)
+    #if isinstance(loader, ExtensionLoader):
+        # ExtensionLoader has not attribute get_filename, instead it has a
+        # path attribute that we can use to retrieve the module path
+    #    module_path = loader.path
+    #else:
+    try:
+        get_filename = loader.get_filename
+    except AttributeError:
+        return None, is_package
+    else:
+        module_path = get_filename(string)
+
+    code = loader.get_source(string)
+    if isinstance(loader, zipimporter):
+        return ZipFileIO(module_path, code, loader.archive), is_package
+
+    # Unfortunately we are reading unicode here already, not bytes.
+    # It seems however hard to get bytes, because the zip importer
+    # logic just unpacks the zip file and returns a file descriptor
+    # that we cannot as easily access. Therefore we just read it as
+    # a string.
+    return KnownContentFileIO(module_path, code), is_package
+
+
+def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True):
     # This import is here, because in other places it will raise a
     # DeprecationWarning.
     import imp
     try:
         module_file, module_path, description = imp.find_module(string, path)
         module_type = description[2]
-        return module_file, module_path, module_type is imp.PKG_DIRECTORY
+        with module_file:
+            code = module_file.read()
+        return KnownContentFileIO(module_path, code), module_type is imp.PKG_DIRECTORY
     except ImportError:
         pass
 
@@ -128,26 +159,12 @@ def find_module_pre_py34(string, path=None, full_name=None, is_global_search=Tru
     for item in path:
         loader = pkgutil.get_importer(item)
         if loader:
-            try:
             loader = loader.find_module(string)
-                if loader:
+            return _from_loader(loader, string)
-                    is_package = loader.is_package(string)
-                    is_archive = hasattr(loader, 'archive')
-                    module_path = loader.get_filename(string)
-                    if is_package:
-                        module_path = os.path.dirname(module_path)
-                    if is_archive:
-                        module_path = loader.archive
-                    file = None
-                    if not is_package or is_archive:
-                        file = DummyFile(loader, string)
-                    return file, module_path, is_package
-            except ImportError:
-                pass
     raise ImportError("No module named {}".format(string))
 
 
-find_module = find_module_py34 if is_py3 else find_module_pre_py34
+find_module = find_module_py34 if is_py3 else find_module_pre_py3
 find_module.__doc__ = """
 Provides information about a module.
 

jedi/evaluate/__init__.py

@@ -427,15 +427,15 @@ class Evaluator(object):
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
 
     def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
-                           use_latest_grammar=False, **kwargs):
+                           use_latest_grammar=False, file_io=None, **kwargs):
         if self.allow_different_encoding:
             if code is None:
-                with open(path, 'rb') as f:
+                assert file_io is not None
-                    code = f.read()
+                code = file_io.read()
             code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
 
         grammar = self.latest_grammar if use_latest_grammar else self.grammar
-        return grammar.parse(code=code, path=path, **kwargs), code
+        return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code
 
     def parse(self, *args, **kwargs):
         return self.parse_and_get_code(*args, **kwargs)[0]

jedi/evaluate/compiled/subprocess/functions.py

@@ -1,6 +1,8 @@
 import sys
 import os
 
+from parso.file_io import KnownContentFileIO
+
 from jedi._compatibility import find_module, cast_path, force_unicode, \
     iter_modules, all_suffixes, print_to_stderr
 from jedi.evaluate.compiled import access
@@ -29,23 +31,40 @@ def create_simple_object(evaluator, obj):
 
 
 def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
+    """
+    Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
+    """
     if sys_path is not None:
         sys.path, temp = sys_path, sys.path
     try:
-        module_file, module_path, is_pkg = find_module(full_name=full_name, **kwargs)
+        return find_module(full_name=full_name, **kwargs)
     except ImportError:
-        return None, None, None
+        return None, None
     finally:
         if sys_path is not None:
             sys.path = temp
 
-    code = None
+    # Unfortunately we are reading unicode here already, not bytes.
+    # It seems however hard to get bytes, because the zip importer
+    # logic just unpacks the zip file and returns a file descriptor
+    # that we cannot as easily access. Therefore we just read it as
+    # a string.
+    code = module_file.read()
+    module_path = cast_path(module_path)
+    if module_path.endswith(('.zip', '.egg')):
+        file_io = ZipFileIO(module_path, code, x)
+    else:
+        file_io = KnownContentFileIO(module_path, code)
+    return code, module_path, is_pkg
     if is_pkg:
         # In this case, we don't have a file yet. Search for the
         # __init__ file.
         if module_path.endswith(('.zip', '.egg')):
             code = module_file.loader.get_source(full_name)
+            print(module_path)
         else:
+            print('xxxx')
+            raise 1
             module_path = _get_init_path(module_path)
     elif module_file:
         if module_path.endswith(('.zip', '.egg')):

jedi/evaluate/context/module.py

@@ -191,9 +191,8 @@ class ModuleContext(ModuleMixin, TreeContext):
 
         # Default to the of this file.
         file = self.py__file__()
-        if file is None:
+        assert file is not None  # Shouldn't be a package in the first place.
-            return None
+        return [os.path.dirname(file)]
-        return os.path.dirname(file)
 
     @property
     def py__path__(self):

jedi/evaluate/gradual/typeshed.py

@@ -1,6 +1,7 @@
 import os
 import re
 
+from parso.file_io import FileIO
 from jedi._compatibility import FileNotFoundError
 from jedi.parser_utils import get_cached_code_lines
 from jedi.evaluate.cache import evaluator_function_cache
@@ -64,7 +65,7 @@ def _get_typeshed_directories(version_info):
 
 @evaluator_function_cache()
 def _load_stub(evaluator, path):
-    return evaluator.parse(path=path, cache=True, use_latest_grammar=True)
+    return evaluator.parse(file_io=FileIO(path), cache=True, use_latest_grammar=True)
 
 
 def _merge_modules(context_set, stub_context):

jedi/evaluate/imports.py

@@ -16,6 +16,7 @@ import os
 from parso.python import tree
 from parso.tree import search_ancestor
 from parso import python_bytes_to_unicode
+from parso.file_io import KnownContentFileIO
 
 from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo,
                                  force_unicode, unicode)
@@ -441,24 +442,20 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
     This method is very similar to importlib's `_gcd_import`.
     """
     if import_names[0] in settings.auto_import_modules:
-        module = _load_module(
+        module = _load_builtin_module(evaluator, import_names, sys_path)
-            evaluator,
-            import_names=import_names,
-            sys_path=sys_path,
-        )
         return ContextSet([module])
 
     module_name = '.'.join(import_names)
     if parent_module_context is None:
         # Override the sys.path. It works only good that way.
         # Injecting the path directly into `find_module` did not work.
-        code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info(
+        file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
             string=import_names[-1],
             full_name=module_name,
             sys_path=sys_path,
             is_global_search=True,
         )
-        if module_path is None:
+        if is_pkg is None:
             raise JediImportError(import_names)
     else:
         try:
@@ -473,21 +470,30 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
                 # not important to be correct.
                 if not isinstance(path, list):
                     path = [path]
-                code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info(
+                file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
                     string=import_names[-1],
                     path=path,
                     full_name=module_name,
                     is_global_search=False,
                 )
-                if module_path is not None:
+                if is_pkg is not None:
                     break
             else:
                 raise JediImportError(import_names)
 
-    module = _load_module(
+    if isinstance(file_io_or_ns, ImplicitNSInfo):
-        evaluator, module_path, code, sys_path,
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
+        module = ImplicitNamespaceContext(
+            evaluator,
+            fullname=file_io_or_ns.name,
+            paths=file_io_or_ns.paths,
+        )
+    elif file_io_or_ns is None:
+        module = _load_builtin_module(evaluator, import_names, sys_path)
+    else:
+        module = _load_python_module(
+            evaluator, file_io_or_ns, sys_path,
             import_names=import_names,
-        safe_module_name=True,
             is_package=is_pkg,
         )
 
@@ -498,43 +504,35 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
     return ContextSet([module])
 
 
-def _load_module(evaluator, path=None, code=None, sys_path=None,
+def _load_python_module(evaluator, file_io, sys_path=None,
-                 import_names=None, safe_module_name=False, is_package=False):
+                        import_names=None, is_package=False):
-    if import_names is None:
-        dotted_name = None
-    else:
-        dotted_name = '.'.join(import_names)
     try:
-        return evaluator.module_cache.get_from_path(path)
+        return evaluator.module_cache.get_from_path(file_io.path)
     except KeyError:
         pass
 
-    if isinstance(path, ImplicitNSInfo):
+    module_node = evaluator.parse(
-        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
+        file_io=file_io,
-        module = ImplicitNamespaceContext(
+        cache=True,
-            evaluator,
+        diff_cache=settings.fast_parser,
-            fullname=path.name,
+        cache_path=settings.cache_directory
-            paths=path.paths,
     )
-    else:
+
+    from jedi.evaluate.context import ModuleContext
+    return ModuleContext(
+        evaluator, module_node,
+        path=file_io.path,
+        string_names=import_names,
+        code_lines=get_cached_code_lines(evaluator.grammar, file_io.path),
+        is_package=is_package,
+    )
+
+
+def _load_builtin_module(evaluator, import_names=None, sys_path=None):
     if sys_path is None:
         sys_path = evaluator.get_sys_path()
 
-        if path is not None and path.endswith(('.py', '.zip', '.egg')):
+    dotted_name = '.'.join(import_names)
-            module_node = evaluator.parse(
-                code=code, path=path, cache=True,
-                diff_cache=settings.fast_parser,
-                cache_path=settings.cache_directory)
-
-            from jedi.evaluate.context import ModuleContext
-            module = ModuleContext(
-                evaluator, module_node,
-                path=path,
-                string_names=import_names,
-                code_lines=get_cached_code_lines(evaluator.grammar, path),
-                is_package=is_package,
-            )
-        else:
     assert dotted_name is not None
     module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path)
     if module is None:
@@ -576,8 +574,8 @@ def get_modules_containing_name(evaluator, modules, name):
                 else:
                     import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
 
-                module = _load_module(
+                module = _load_python_module(
-                    evaluator, path, code,
+                    evaluator, KnownContentFileIO(path, code),
                     sys_path=e_sys_path,
                     import_names=import_names,
                     is_package=is_package,

test/test_evaluate/test_imports.py

@@ -6,6 +6,7 @@ Tests".
 import os
 
 import pytest
+from parso.file_io import FileIO
 
 from jedi._compatibility import find_module_py33, find_module
 from jedi.evaluate import compiled
@@ -19,20 +20,20 @@ THIS_DIR = os.path.dirname(__file__)
 @pytest.mark.skipif('sys.version_info < (3,3)')
 def test_find_module_py33():
     """Needs to work like the old find_module."""
-    assert find_module_py33('_io') == (None, '_io', False)
+    assert find_module_py33('_io') == (None, False)
+    with pytest.raises(ImportError):
+        assert find_module_py33('_DOESNTEXIST_') == (None, None)
 
 
 def test_find_module_package():
-    file, path, is_package = find_module('json')
+    file_io, is_package = find_module('json')
-    assert file is None
+    assert file_io.path.endswith(os.path.join('json', '__init__.py'))
-    assert path.endswith('json')
     assert is_package is True
 
 
 def test_find_module_not_package():
-    file, path, is_package = find_module('io')
+    file_io, is_package = find_module('io')
-    assert file is not None
+    assert file_io.path.endswith('io.py')
-    assert path.endswith('io.py')
     assert is_package is False
 
 
@@ -44,13 +45,14 @@ def test_find_module_package_zipped(Script, evaluator, environment):
     script = Script('import pkg; pkg.mod', sys_path=sys_path)
     assert len(script.completions()) == 1
 
-    code, path, is_package = evaluator.compiled_subprocess.get_module_info(
+    file_io, is_package = evaluator.compiled_subprocess.get_module_info(
         sys_path=sys_path,
         string=u'pkg',
         full_name=u'pkg'
     )
-    assert code is not None
+    assert file_io is not None
-    assert path.endswith('pkg.zip')
+    assert file_io.path.endswith(os.path.join('pkg.zip', 'pkg', '__init__.py'))
+    assert file_io._zip_path.endswith('pkg.zip')
     assert is_package is True
 
 
@@ -58,10 +60,10 @@ def test_correct_zip_package_behavior(Script, evaluator, environment):
     sys_path = environment.get_sys_path() + [pkg_zip_path]
     pkg, = Script('import pkg', sys_path=sys_path).goto_definitions()
     context, = pkg._name.infer()
-    assert context.py__file__() == pkg_zip_path
+    assert context.py__file__() == os.path.join(pkg_zip_path, 'pkg', '__init__.py')
     assert context.is_package is True
     assert context.py__package__() == ('pkg',)
-    assert context.py__path__() == [pkg_zip_path]
+    assert context.py__path__() == [os.path.join(pkg_zip_path, 'pkg')]
 
 
 def test_find_module_not_package_zipped(Script, evaluator, environment):
@@ -70,13 +72,12 @@ def test_find_module_not_package_zipped(Script, evaluator, environment):
     script = Script('import not_pkg; not_pkg.val', sys_path=sys_path)
     assert len(script.completions()) == 1
 
-    code, path, is_package = evaluator.compiled_subprocess.get_module_info(
+    file_io, is_package = evaluator.compiled_subprocess.get_module_info(
         sys_path=sys_path,
         string=u'not_pkg',
         full_name=u'not_pkg'
     )
-    assert code is not None
+    assert file_io.path.endswith(os.path.join('not_pkg.zip', 'not_pkg.py'))
-    assert path.endswith('not_pkg.zip')
     assert is_package is False
 
 
@@ -270,13 +271,18 @@ def test_compiled_import_none(monkeypatch, Script):
 
 
 @pytest.mark.parametrize(
-    ('path', 'goal'), [
+    ('path', 'is_package', 'goal'), [
-        (os.path.join(THIS_DIR, 'test_docstring.py'), ('ok', 'lala', 'test_imports')),
+        (os.path.join(THIS_DIR, 'test_docstring.py'), False, ('ok', 'lala', 'test_imports')),
-        (os.path.join(THIS_DIR, '__init__.py'), ('ok', 'lala', 'x', 'test_imports')),
+        (os.path.join(THIS_DIR, '__init__.py'), True, ('ok', 'lala', 'x', 'test_imports')),
     ]
 )
-def test_get_modules_containing_name(evaluator, path, goal):
+def test_get_modules_containing_name(evaluator, path, goal, is_package):
-    module = imports._load_module(evaluator, path, import_names=('ok', 'lala', 'x'))
+    module = imports._load_python_module(
+        evaluator,
+        FileIO(path),
+        import_names=('ok', 'lala', 'x'),
+        is_package=is_package,
+    )
     assert module
     input_module, found_module = imports.get_modules_containing_name(
         evaluator,