diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 21ffa82a..c46b6402 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -14,6 +14,9 @@ try: import importlib except ImportError: pass +from zipimport import zipimporter + +from parso.file_io import KnownContentFileIO is_py3 = sys.version_info[0] >= 3 is_py35 = is_py3 and sys.version_info[1] >= 5 @@ -55,7 +58,7 @@ def find_module_py34(string, path=None, full_name=None, is_global_search=True): # This is a namespace package. full_name = string if not path else full_name implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path) - return None, implicit_ns_info, False + return implicit_ns_info, True break return find_module_py33(string, path, loader) @@ -81,13 +84,9 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s if loader is None: raise ImportError("Couldn't find a loader for {}".format(string)) + return _from_loader(loader, string) is_package = loader.is_package(string) if is_package: - if hasattr(loader, 'path'): - module_path = os.path.dirname(loader.path) - else: - # At least zipimporter does not have path attribute - module_path = os.path.dirname(loader.get_filename(string)) if hasattr(loader, 'archive'): module_file = DummyFile(loader, string) else: @@ -97,29 +96,61 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s module_path = loader.get_filename(string) module_file = DummyFile(loader, string) except AttributeError: - # ExtensionLoader has not attribute get_filename, instead it has a - # path attribute that we can use to retrieve the module path try: - module_path = loader.path module_file = DummyFile(loader, string) except AttributeError: module_path = string module_file = None - if hasattr(loader, 'archive'): - module_path = loader.archive - return module_file, module_path, is_package -def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True): +class ZipFileIO(KnownContentFileIO): + """For .zip and .egg archives""" + def __init__(self, path, code, zip_path): + super(ZipFileIO, self).__init__(path, code) + self._zip_path = zip_path + + def get_last_modified(self): + return os.path.getmtime(self._zip_path) + + +def _from_loader(loader, string): + is_package = loader.is_package(string) + #if isinstance(loader, ExtensionLoader): + # ExtensionLoader has not attribute get_filename, instead it has a + # path attribute that we can use to retrieve the module path + # module_path = loader.path + #else: + try: + get_filename = loader.get_filename + except AttributeError: + return None, is_package + else: + module_path = get_filename(string) + + code = loader.get_source(string) + if isinstance(loader, zipimporter): + return ZipFileIO(module_path, code, loader.archive), is_package + + # Unfortunately we are reading unicode here already, not bytes. + # It seems however hard to get bytes, because the zip importer + # logic just unpacks the zip file and returns a file descriptor + # that we cannot as easily access. Therefore we just read it as + # a string. + return KnownContentFileIO(module_path, code), is_package + + +def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True): # This import is here, because in other places it will raise a # DeprecationWarning. import imp try: module_file, module_path, description = imp.find_module(string, path) module_type = description[2] - return module_file, module_path, module_type is imp.PKG_DIRECTORY + with module_file: + code = module_file.read() + return KnownContentFileIO(module_path, code), module_type is imp.PKG_DIRECTORY except ImportError: pass @@ -128,26 +159,12 @@ def find_module_pre_py34(string, path=None, full_name=None, is_global_search=Tru for item in path: loader = pkgutil.get_importer(item) if loader: - try: - loader = loader.find_module(string) - if loader: - is_package = loader.is_package(string) - is_archive = hasattr(loader, 'archive') - module_path = loader.get_filename(string) - if is_package: - module_path = os.path.dirname(module_path) - if is_archive: - module_path = loader.archive - file = None - if not is_package or is_archive: - file = DummyFile(loader, string) - return file, module_path, is_package - except ImportError: - pass + loader = loader.find_module(string) + return _from_loader(loader, string) raise ImportError("No module named {}".format(string)) -find_module = find_module_py34 if is_py3 else find_module_pre_py34 +find_module = find_module_py34 if is_py3 else find_module_pre_py3 find_module.__doc__ = """ Provides information about a module. diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index db7d32b5..e0e89061 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -427,15 +427,15 @@ class Evaluator(object): return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object) def parse_and_get_code(self, code=None, path=None, encoding='utf-8', - use_latest_grammar=False, **kwargs): + use_latest_grammar=False, file_io=None, **kwargs): if self.allow_different_encoding: if code is None: - with open(path, 'rb') as f: - code = f.read() + assert file_io is not None + code = file_io.read() code = python_bytes_to_unicode(code, encoding=encoding, errors='replace') grammar = self.latest_grammar if use_latest_grammar else self.grammar - return grammar.parse(code=code, path=path, **kwargs), code + return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code def parse(self, *args, **kwargs): return self.parse_and_get_code(*args, **kwargs)[0] diff --git a/jedi/evaluate/compiled/subprocess/functions.py b/jedi/evaluate/compiled/subprocess/functions.py index c0fc6d13..10adcf17 100644 --- a/jedi/evaluate/compiled/subprocess/functions.py +++ b/jedi/evaluate/compiled/subprocess/functions.py @@ -1,6 +1,8 @@ import sys import os +from parso.file_io import KnownContentFileIO + from jedi._compatibility import find_module, cast_path, force_unicode, \ iter_modules, all_suffixes, print_to_stderr from jedi.evaluate.compiled import access @@ -29,23 +31,40 @@ def create_simple_object(evaluator, obj): def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs): + """ + Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]] + """ if sys_path is not None: sys.path, temp = sys_path, sys.path try: - module_file, module_path, is_pkg = find_module(full_name=full_name, **kwargs) + return find_module(full_name=full_name, **kwargs) except ImportError: - return None, None, None + return None, None finally: if sys_path is not None: sys.path = temp - code = None + # Unfortunately we are reading unicode here already, not bytes. + # It seems however hard to get bytes, because the zip importer + # logic just unpacks the zip file and returns a file descriptor + # that we cannot as easily access. Therefore we just read it as + # a string. + code = module_file.read() + module_path = cast_path(module_path) + if module_path.endswith(('.zip', '.egg')): + file_io = ZipFileIO(module_path, code, x) + else: + file_io = KnownContentFileIO(module_path, code) + return code, module_path, is_pkg if is_pkg: # In this case, we don't have a file yet. Search for the # __init__ file. if module_path.endswith(('.zip', '.egg')): code = module_file.loader.get_source(full_name) + print(module_path) else: + print('xxxx') + raise 1 module_path = _get_init_path(module_path) elif module_file: if module_path.endswith(('.zip', '.egg')): diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 704d1696..eb4195da 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -191,9 +191,8 @@ class ModuleContext(ModuleMixin, TreeContext): # Default to the of this file. file = self.py__file__() - if file is None: - return None - return os.path.dirname(file) + assert file is not None # Shouldn't be a package in the first place. + return [os.path.dirname(file)] @property def py__path__(self): diff --git a/jedi/evaluate/gradual/typeshed.py b/jedi/evaluate/gradual/typeshed.py index 76d806e9..8fd60c1d 100644 --- a/jedi/evaluate/gradual/typeshed.py +++ b/jedi/evaluate/gradual/typeshed.py @@ -1,6 +1,7 @@ import os import re +from parso.file_io import FileIO from jedi._compatibility import FileNotFoundError from jedi.parser_utils import get_cached_code_lines from jedi.evaluate.cache import evaluator_function_cache @@ -64,7 +65,7 @@ def _get_typeshed_directories(version_info): @evaluator_function_cache() def _load_stub(evaluator, path): - return evaluator.parse(path=path, cache=True, use_latest_grammar=True) + return evaluator.parse(file_io=FileIO(path), cache=True, use_latest_grammar=True) def _merge_modules(context_set, stub_context): diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 98172587..e4dbd9ba 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -16,6 +16,7 @@ import os from parso.python import tree from parso.tree import search_ancestor from parso import python_bytes_to_unicode +from parso.file_io import KnownContentFileIO from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo, force_unicode, unicode) @@ -441,24 +442,20 @@ def import_module(evaluator, import_names, parent_module_context, sys_path): This method is very similar to importlib's `_gcd_import`. """ if import_names[0] in settings.auto_import_modules: - module = _load_module( - evaluator, - import_names=import_names, - sys_path=sys_path, - ) + module = _load_builtin_module(evaluator, import_names, sys_path) return ContextSet([module]) module_name = '.'.join(import_names) if parent_module_context is None: # Override the sys.path. It works only good that way. # Injecting the path directly into `find_module` did not work. - code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info( + file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info( string=import_names[-1], full_name=module_name, sys_path=sys_path, is_global_search=True, ) - if module_path is None: + if is_pkg is None: raise JediImportError(import_names) else: try: @@ -473,23 +470,32 @@ def import_module(evaluator, import_names, parent_module_context, sys_path): # not important to be correct. if not isinstance(path, list): path = [path] - code, module_path, is_pkg = evaluator.compiled_subprocess.get_module_info( + file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info( string=import_names[-1], path=path, full_name=module_name, is_global_search=False, ) - if module_path is not None: + if is_pkg is not None: break else: raise JediImportError(import_names) - module = _load_module( - evaluator, module_path, code, sys_path, - import_names=import_names, - safe_module_name=True, - is_package=is_pkg, - ) + if isinstance(file_io_or_ns, ImplicitNSInfo): + from jedi.evaluate.context.namespace import ImplicitNamespaceContext + module = ImplicitNamespaceContext( + evaluator, + fullname=file_io_or_ns.name, + paths=file_io_or_ns.paths, + ) + elif file_io_or_ns is None: + module = _load_builtin_module(evaluator, import_names, sys_path) + else: + module = _load_python_module( + evaluator, file_io_or_ns, sys_path, + import_names=import_names, + is_package=is_pkg, + ) if parent_module_context is None: debug.dbg('global search_module %s: %s', import_names[-1], module) @@ -498,49 +504,41 @@ def import_module(evaluator, import_names, parent_module_context, sys_path): return ContextSet([module]) -def _load_module(evaluator, path=None, code=None, sys_path=None, - import_names=None, safe_module_name=False, is_package=False): - if import_names is None: - dotted_name = None - else: - dotted_name = '.'.join(import_names) +def _load_python_module(evaluator, file_io, sys_path=None, + import_names=None, is_package=False): try: - return evaluator.module_cache.get_from_path(path) + return evaluator.module_cache.get_from_path(file_io.path) except KeyError: pass - if isinstance(path, ImplicitNSInfo): - from jedi.evaluate.context.namespace import ImplicitNamespaceContext - module = ImplicitNamespaceContext( - evaluator, - fullname=path.name, - paths=path.paths, - ) - else: - if sys_path is None: - sys_path = evaluator.get_sys_path() + module_node = evaluator.parse( + file_io=file_io, + cache=True, + diff_cache=settings.fast_parser, + cache_path=settings.cache_directory + ) - if path is not None and path.endswith(('.py', '.zip', '.egg')): - module_node = evaluator.parse( - code=code, path=path, cache=True, - diff_cache=settings.fast_parser, - cache_path=settings.cache_directory) + from jedi.evaluate.context import ModuleContext + return ModuleContext( + evaluator, module_node, + path=file_io.path, + string_names=import_names, + code_lines=get_cached_code_lines(evaluator.grammar, file_io.path), + is_package=is_package, + ) - from jedi.evaluate.context import ModuleContext - module = ModuleContext( - evaluator, module_node, - path=path, - string_names=import_names, - code_lines=get_cached_code_lines(evaluator.grammar, path), - is_package=is_package, - ) - else: - assert dotted_name is not None - module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path) - if module is None: - # The file might raise an ImportError e.g. and therefore not be - # importable. - raise JediImportError(import_names) + +def _load_builtin_module(evaluator, import_names=None, sys_path=None): + if sys_path is None: + sys_path = evaluator.get_sys_path() + + dotted_name = '.'.join(import_names) + assert dotted_name is not None + module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path) + if module is None: + # The file might raise an ImportError e.g. and therefore not be + # importable. + raise JediImportError(import_names) return module @@ -576,8 +574,8 @@ def get_modules_containing_name(evaluator, modules, name): else: import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) - module = _load_module( - evaluator, path, code, + module = _load_python_module( + evaluator, KnownContentFileIO(path, code), sys_path=e_sys_path, import_names=import_names, is_package=is_package, diff --git a/test/test_evaluate/test_imports.py b/test/test_evaluate/test_imports.py index c3cf96fc..c6af770c 100644 --- a/test/test_evaluate/test_imports.py +++ b/test/test_evaluate/test_imports.py @@ -6,6 +6,7 @@ Tests". import os import pytest +from parso.file_io import FileIO from jedi._compatibility import find_module_py33, find_module from jedi.evaluate import compiled @@ -19,20 +20,20 @@ THIS_DIR = os.path.dirname(__file__) @pytest.mark.skipif('sys.version_info < (3,3)') def test_find_module_py33(): """Needs to work like the old find_module.""" - assert find_module_py33('_io') == (None, '_io', False) + assert find_module_py33('_io') == (None, False) + with pytest.raises(ImportError): + assert find_module_py33('_DOESNTEXIST_') == (None, None) def test_find_module_package(): - file, path, is_package = find_module('json') - assert file is None - assert path.endswith('json') + file_io, is_package = find_module('json') + assert file_io.path.endswith(os.path.join('json', '__init__.py')) assert is_package is True def test_find_module_not_package(): - file, path, is_package = find_module('io') - assert file is not None - assert path.endswith('io.py') + file_io, is_package = find_module('io') + assert file_io.path.endswith('io.py') assert is_package is False @@ -44,13 +45,14 @@ def test_find_module_package_zipped(Script, evaluator, environment): script = Script('import pkg; pkg.mod', sys_path=sys_path) assert len(script.completions()) == 1 - code, path, is_package = evaluator.compiled_subprocess.get_module_info( + file_io, is_package = evaluator.compiled_subprocess.get_module_info( sys_path=sys_path, string=u'pkg', full_name=u'pkg' ) - assert code is not None - assert path.endswith('pkg.zip') + assert file_io is not None + assert file_io.path.endswith(os.path.join('pkg.zip', 'pkg', '__init__.py')) + assert file_io._zip_path.endswith('pkg.zip') assert is_package is True @@ -58,10 +60,10 @@ def test_correct_zip_package_behavior(Script, evaluator, environment): sys_path = environment.get_sys_path() + [pkg_zip_path] pkg, = Script('import pkg', sys_path=sys_path).goto_definitions() context, = pkg._name.infer() - assert context.py__file__() == pkg_zip_path + assert context.py__file__() == os.path.join(pkg_zip_path, 'pkg', '__init__.py') assert context.is_package is True assert context.py__package__() == ('pkg',) - assert context.py__path__() == [pkg_zip_path] + assert context.py__path__() == [os.path.join(pkg_zip_path, 'pkg')] def test_find_module_not_package_zipped(Script, evaluator, environment): @@ -70,13 +72,12 @@ def test_find_module_not_package_zipped(Script, evaluator, environment): script = Script('import not_pkg; not_pkg.val', sys_path=sys_path) assert len(script.completions()) == 1 - code, path, is_package = evaluator.compiled_subprocess.get_module_info( + file_io, is_package = evaluator.compiled_subprocess.get_module_info( sys_path=sys_path, string=u'not_pkg', full_name=u'not_pkg' ) - assert code is not None - assert path.endswith('not_pkg.zip') + assert file_io.path.endswith(os.path.join('not_pkg.zip', 'not_pkg.py')) assert is_package is False @@ -270,13 +271,18 @@ def test_compiled_import_none(monkeypatch, Script): @pytest.mark.parametrize( - ('path', 'goal'), [ - (os.path.join(THIS_DIR, 'test_docstring.py'), ('ok', 'lala', 'test_imports')), - (os.path.join(THIS_DIR, '__init__.py'), ('ok', 'lala', 'x', 'test_imports')), + ('path', 'is_package', 'goal'), [ + (os.path.join(THIS_DIR, 'test_docstring.py'), False, ('ok', 'lala', 'test_imports')), + (os.path.join(THIS_DIR, '__init__.py'), True, ('ok', 'lala', 'x', 'test_imports')), ] ) -def test_get_modules_containing_name(evaluator, path, goal): - module = imports._load_module(evaluator, path, import_names=('ok', 'lala', 'x')) +def test_get_modules_containing_name(evaluator, path, goal, is_package): + module = imports._load_python_module( + evaluator, + FileIO(path), + import_names=('ok', 'lala', 'x'), + is_package=is_package, + ) assert module input_module, found_module = imports.get_modules_containing_name( evaluator,