VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Fix test_imports tests, now zip imports work again

Browse Source
This commit is contained in:
Dave Halter
2019-03-26 09:16:38 +01:00
parent b6612a83c3
commit a437c2cb02
7 changed files with 156 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
jedi/_compatibility.py
View File

@@ -14,6 +14,9 @@ try:
import importlib
except ImportError:
pass
from zipimport import zipimporter
from parso.file_io import KnownContentFileIO
is_py3 = sys.version_info[0] >= 3
is_py35 = is_py3 and sys.version_info[1] >= 5
@@ -55,7 +58,7 @@ def find_module_py34(string, path=None, full_name=None, is_global_search=True):
# This is a namespace package.
full_name = string if not path else full_name
implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
return None, implicit_ns_info, False
return implicit_ns_info, True
break
return find_module_py33(string, path, loader)
@@ -81,13 +84,9 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s
if loader is None:
raise ImportError("Couldn't find a loader for {}".format(string))
return _from_loader(loader, string)
is_package = loader.is_package(string)
if is_package:
if hasattr(loader, 'path'):
module_path = os.path.dirname(loader.path)
else:
# At least zipimporter does not have path attribute
module_path = os.path.dirname(loader.get_filename(string))
if hasattr(loader, 'archive'):
module_file = DummyFile(loader, string)
else:
@@ -97,29 +96,61 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s
module_path = loader.get_filename(string)
module_file = DummyFile(loader, string)
except AttributeError:
# ExtensionLoader has not attribute get_filename, instead it has a
# path attribute that we can use to retrieve the module path
try:
module_path = loader.path
module_file = DummyFile(loader, string)
except AttributeError:
module_path = string
module_file = None
if hasattr(loader, 'archive'):
module_path = loader.archive
return module_file, module_path, is_package
def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True):
class ZipFileIO(KnownContentFileIO):
"""For .zip and .egg archives"""
def __init__(self, path, code, zip_path):
super(ZipFileIO, self).__init__(path, code)
self._zip_path = zip_path
def get_last_modified(self):
return os.path.getmtime(self._zip_path)
def _from_loader(loader, string):
is_package = loader.is_package(string)
#if isinstance(loader, ExtensionLoader):
# ExtensionLoader has not attribute get_filename, instead it has a
# path attribute that we can use to retrieve the module path
# module_path = loader.path
#else:
try:
get_filename = loader.get_filename
except AttributeError:
return None, is_package
else:
module_path = get_filename(string)
code = loader.get_source(string)
if isinstance(loader, zipimporter):
return ZipFileIO(module_path, code, loader.archive), is_package
# Unfortunately we are reading unicode here already, not bytes.
# It seems however hard to get bytes, because the zip importer
# logic just unpacks the zip file and returns a file descriptor
# that we cannot as easily access. Therefore we just read it as
# a string.
return KnownContentFileIO(module_path, code), is_package
def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True):
# This import is here, because in other places it will raise a
# DeprecationWarning.
import imp
try:
module_file, module_path, description = imp.find_module(string, path)
module_type = description[2]
return module_file, module_path, module_type is imp.PKG_DIRECTORY
with module_file:
code = module_file.read()
return KnownContentFileIO(module_path, code), module_type is imp.PKG_DIRECTORY
except ImportError:
pass
@@ -128,26 +159,12 @@ def find_module_pre_py34(string, path=None, full_name=None, is_global_search=Tru
for item in path:
loader = pkgutil.get_importer(item)
if loader:
try:
loader = loader.find_module(string)
if loader:
is_package = loader.is_package(string)
is_archive = hasattr(loader, 'archive')
module_path = loader.get_filename(string)
if is_package:
module_path = os.path.dirname(module_path)
if is_archive:
module_path = loader.archive
file = None
if not is_package or is_archive:
file = DummyFile(loader, string)
return file, module_path, is_package
except ImportError:
pass
return _from_loader(loader, string)
raise ImportError("No module named {}".format(string))
find_module = find_module_py34 if is_py3 else find_module_pre_py34
find_module = find_module_py34 if is_py3 else find_module_pre_py3
find_module.__doc__ = """
Provides information about a module.

8
jedi/evaluate/__init__.py
View File

@@ -427,15 +427,15 @@ class Evaluator(object):
return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
use_latest_grammar=False, **kwargs):
use_latest_grammar=False, file_io=None, **kwargs):
if self.allow_different_encoding:
if code is None:
with open(path, 'rb') as f:
code = f.read()
assert file_io is not None
code = file_io.read()
code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
grammar = self.latest_grammar if use_latest_grammar else self.grammar
return grammar.parse(code=code, path=path, **kwargs), code
return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code
def parse(self, *args, **kwargs):
return self.parse_and_get_code(*args, **kwargs)[0]

25
jedi/evaluate/compiled/subprocess/functions.py
View File

@@ -1,6 +1,8 @@
import sys
import os
from parso.file_io import KnownContentFileIO
from jedi._compatibility import find_module, cast_path, force_unicode, \
iter_modules, all_suffixes, print_to_stderr
from jedi.evaluate.compiled import access
@@ -29,23 +31,40 @@ def create_simple_object(evaluator, obj):
def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
"""
Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
"""
if sys_path is not None:
sys.path, temp = sys_path, sys.path
try:
module_file, module_path, is_pkg = find_module(full_name=full_name, **kwargs)
return find_module(full_name=full_name, **kwargs)
except ImportError:
return None, None, None
return None, None
finally:
if sys_path is not None:
sys.path = temp
code = None
# Unfortunately we are reading unicode here already, not bytes.
# It seems however hard to get bytes, because the zip importer
# logic just unpacks the zip file and returns a file descriptor
# that we cannot as easily access. Therefore we just read it as
# a string.
code = module_file.read()
module_path = cast_path(module_path)
if module_path.endswith(('.zip', '.egg')):
file_io = ZipFileIO(module_path, code, x)
else:
file_io = KnownContentFileIO(module_path, code)
return code, module_path, is_pkg
if is_pkg:
# In this case, we don't have a file yet. Search for the
# __init__ file.
if module_path.endswith(('.zip', '.egg')):
code = module_file.loader.get_source(full_name)
print(module_path)
else:
print('xxxx')
raise 1
module_path = _get_init_path(module_path)
elif module_file:
if module_path.endswith(('.zip', '.egg')):

5
jedi/evaluate/context/module.py
View File

@@ -191,9 +191,8 @@ class ModuleContext(ModuleMixin, TreeContext):
# Default to the of this file.
file = self.py__file__()
if file is None:
return None
return os.path.dirname(file)
assert file is not None # Shouldn't be a package in the first place.
return [os.path.dirname(file)]
@property
def py__path__(self):

3
jedi/evaluate/gradual/typeshed.py
View File

@@ -1,6 +1,7 @@
import os
import re
from parso.file_io import FileIO
from jedi._compatibility import FileNotFoundError
from jedi.parser_utils import get_cached_code_lines
from jedi.evaluate.cache import evaluator_function_cache
@@ -64,7 +65,7 @@ def _get_typeshed_directories(version_info):
@evaluator_function_cache()
def _load_stub(evaluator, path):
return evaluator.parse(path=path, cache=True, use_latest_grammar=True)
return evaluator.parse(file_io=FileIO(path), cache=True, use_latest_grammar=True)
def _merge_modules(context_set, stub_context):

84
jedi/evaluate/imports.py
View File

@@ -16,6 +16,7 @@ import os
from parso.python import tree
from parso.tree import search_ancestor
from parso import python_bytes_to_unicode
from parso.file_io import KnownContentFileIO
from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo,
force_unicode, unicode)
@@ -441,24 +442,20 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
This method is very similar to importlib's `_gcd_import`.
"""
if import_names[0] in settings.auto_import_modules:
module = _load_module(
evaluator,
import_names=import_names,
sys_path=sys_path,
)
module = _load_builtin_module(evaluator, import_names, sys_path)
return ContextSet([module])
module_name = '.'.join(import_names)
if parent_module_context is None:
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info(
file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
string=import_names[-1],
full_name=module_name,
sys_path=sys_path,
is_global_search=True,
)
if module_path is None:
if is_pkg is None:
raise JediImportError(import_names)
else:
try:
@@ -473,21 +470,30 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
# not important to be correct.
if not isinstance(path, list):
path = [path]
code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info(
file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
string=import_names[-1],
path=path,
full_name=module_name,
is_global_search=False,
)
if module_path is not None:
if is_pkg is not None:
break
else:
raise JediImportError(import_names)
module = _load_module(
evaluator, module_path, code, sys_path,
if isinstance(file_io_or_ns, ImplicitNSInfo):
from jedi.evaluate.context.namespace import ImplicitNamespaceContext
module = ImplicitNamespaceContext(
evaluator,
fullname=file_io_or_ns.name,
paths=file_io_or_ns.paths,
)
elif file_io_or_ns is None:
module = _load_builtin_module(evaluator, import_names, sys_path)
else:
module = _load_python_module(
evaluator, file_io_or_ns, sys_path,
import_names=import_names,
safe_module_name=True,
is_package=is_pkg,
)
@@ -498,43 +504,35 @@ def import_module(evaluator, import_names, parent_module_context, sys_path):
return ContextSet([module])
def _load_module(evaluator, path=None, code=None, sys_path=None,
import_names=None, safe_module_name=False, is_package=False):
if import_names is None:
dotted_name = None
else:
dotted_name = '.'.join(import_names)
def _load_python_module(evaluator, file_io, sys_path=None,
import_names=None, is_package=False):
try:
return evaluator.module_cache.get_from_path(path)
return evaluator.module_cache.get_from_path(file_io.path)
except KeyError:
pass
if isinstance(path, ImplicitNSInfo):
from jedi.evaluate.context.namespace import ImplicitNamespaceContext
module = ImplicitNamespaceContext(
evaluator,
fullname=path.name,
paths=path.paths,
module_node = evaluator.parse(
file_io=file_io,
cache=True,
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory
)
else:
from jedi.evaluate.context import ModuleContext
return ModuleContext(
evaluator, module_node,
path=file_io.path,
string_names=import_names,
code_lines=get_cached_code_lines(evaluator.grammar, file_io.path),
is_package=is_package,
)
def _load_builtin_module(evaluator, import_names=None, sys_path=None):
if sys_path is None:
sys_path = evaluator.get_sys_path()
if path is not None and path.endswith(('.py', '.zip', '.egg')):
module_node = evaluator.parse(
code=code, path=path, cache=True,
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory)
from jedi.evaluate.context import ModuleContext
module = ModuleContext(
evaluator, module_node,
path=path,
string_names=import_names,
code_lines=get_cached_code_lines(evaluator.grammar, path),
is_package=is_package,
)
else:
dotted_name = '.'.join(import_names)
assert dotted_name is not None
module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path)
if module is None:
@@ -576,8 +574,8 @@ def get_modules_containing_name(evaluator, modules, name):
else:
import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
module = _load_module(
evaluator, path, code,
module = _load_python_module(
evaluator, KnownContentFileIO(path, code),
sys_path=e_sys_path,
import_names=import_names,
is_package=is_package,

46
test/test_evaluate/test_imports.py
View File

@@ -6,6 +6,7 @@ Tests".
import os
import pytest
from parso.file_io import FileIO
from jedi._compatibility import find_module_py33, find_module
from jedi.evaluate import compiled
@@ -19,20 +20,20 @@ THIS_DIR = os.path.dirname(__file__)
@pytest.mark.skipif('sys.version_info < (3,3)')
def test_find_module_py33():
"""Needs to work like the old find_module."""
assert find_module_py33('_io') == (None, '_io', False)
assert find_module_py33('_io') == (None, False)
with pytest.raises(ImportError):
assert find_module_py33('_DOESNTEXIST_') == (None, None)
def test_find_module_package():
file, path, is_package = find_module('json')
assert file is None
assert path.endswith('json')
file_io, is_package = find_module('json')
assert file_io.path.endswith(os.path.join('json', '__init__.py'))
assert is_package is True
def test_find_module_not_package():
file, path, is_package = find_module('io')
assert file is not None
assert path.endswith('io.py')
file_io, is_package = find_module('io')
assert file_io.path.endswith('io.py')
assert is_package is False
@@ -44,13 +45,14 @@ def test_find_module_package_zipped(Script, evaluator, environment):
script = Script('import pkg; pkg.mod', sys_path=sys_path)
assert len(script.completions()) == 1
code, path, is_package = evaluator.compiled_subprocess.get_module_info(
file_io, is_package = evaluator.compiled_subprocess.get_module_info(
sys_path=sys_path,
string=u'pkg',
full_name=u'pkg'
)
assert code is not None
assert path.endswith('pkg.zip')
assert file_io is not None
assert file_io.path.endswith(os.path.join('pkg.zip', 'pkg', '__init__.py'))
assert file_io._zip_path.endswith('pkg.zip')
assert is_package is True
@@ -58,10 +60,10 @@ def test_correct_zip_package_behavior(Script, evaluator, environment):
sys_path = environment.get_sys_path() + [pkg_zip_path]
pkg, = Script('import pkg', sys_path=sys_path).goto_definitions()
context, = pkg._name.infer()
assert context.py__file__() == pkg_zip_path
assert context.py__file__() == os.path.join(pkg_zip_path, 'pkg', '__init__.py')
assert context.is_package is True
assert context.py__package__() == ('pkg',)
assert context.py__path__() == [pkg_zip_path]
assert context.py__path__() == [os.path.join(pkg_zip_path, 'pkg')]
def test_find_module_not_package_zipped(Script, evaluator, environment):
@@ -70,13 +72,12 @@ def test_find_module_not_package_zipped(Script, evaluator, environment):
script = Script('import not_pkg; not_pkg.val', sys_path=sys_path)
assert len(script.completions()) == 1
code, path, is_package = evaluator.compiled_subprocess.get_module_info(
file_io, is_package = evaluator.compiled_subprocess.get_module_info(
sys_path=sys_path,
string=u'not_pkg',
full_name=u'not_pkg'
)
assert code is not None
assert path.endswith('not_pkg.zip')
assert file_io.path.endswith(os.path.join('not_pkg.zip', 'not_pkg.py'))
assert is_package is False
@@ -270,13 +271,18 @@ def test_compiled_import_none(monkeypatch, Script):
@pytest.mark.parametrize(
('path', 'goal'), [
(os.path.join(THIS_DIR, 'test_docstring.py'), ('ok', 'lala', 'test_imports')),
(os.path.join(THIS_DIR, '__init__.py'), ('ok', 'lala', 'x', 'test_imports')),
('path', 'is_package', 'goal'), [
(os.path.join(THIS_DIR, 'test_docstring.py'), False, ('ok', 'lala', 'test_imports')),
(os.path.join(THIS_DIR, '__init__.py'), True, ('ok', 'lala', 'x', 'test_imports')),
]
)
def test_get_modules_containing_name(evaluator, path, goal):
module = imports._load_module(evaluator, path, import_names=('ok', 'lala', 'x'))
def test_get_modules_containing_name(evaluator, path, goal, is_package):
module = imports._load_python_module(
evaluator,
FileIO(path),
import_names=('ok', 'lala', 'x'),
is_package=is_package,
)
assert module
input_module, found_module = imports.get_modules_containing_name(
evaluator,