diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index f8f93c10..21ffa82a 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -81,32 +81,30 @@ def find_module_py33(string, path=None, loader=None, full_name=None, is_global_s if loader is None: raise ImportError("Couldn't find a loader for {}".format(string)) - try: - is_package = loader.is_package(string) - if is_package: - if hasattr(loader, 'path'): - module_path = os.path.dirname(loader.path) - else: - # At least zipimporter does not have path attribute - module_path = os.path.dirname(loader.get_filename(string)) - if hasattr(loader, 'archive'): - module_file = DummyFile(loader, string) - else: - module_file = None + is_package = loader.is_package(string) + if is_package: + if hasattr(loader, 'path'): + module_path = os.path.dirname(loader.path) else: + # At least zipimporter does not have path attribute + module_path = os.path.dirname(loader.get_filename(string)) + if hasattr(loader, 'archive'): + module_file = DummyFile(loader, string) + else: + module_file = None + else: + try: module_path = loader.get_filename(string) module_file = DummyFile(loader, string) - except AttributeError: - # ExtensionLoader has not attribute get_filename, instead it has a - # path attribute that we can use to retrieve the module path - try: - module_path = loader.path - module_file = DummyFile(loader, string) except AttributeError: - module_path = string - module_file = None - finally: - is_package = False + # ExtensionLoader has not attribute get_filename, instead it has a + # path attribute that we can use to retrieve the module path + try: + module_path = loader.path + module_file = DummyFile(loader, string) + except AttributeError: + module_path = string + module_file = None if hasattr(loader, 'archive'): module_path = loader.archive diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 46400bd7..4ae422a1 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -150,15 +150,21 @@ class Script(object): @cache.memoize_method def _get_module(self): names = ('__main__',) + is_package = False if self.path is not None: - import_names = transform_path_to_dotted(self._evaluator.get_sys_path(), self.path) + import_names, is_p = transform_path_to_dotted( + self._evaluator.get_sys_path(), + self.path + ) if import_names is not None: names = import_names + is_package = is_p module = ModuleContext( self._evaluator, self._module_node, cast_path(self.path), string_names=names, code_lines=self._code_lines, + is_package=is_package, ) self._evaluator.module_cache.add(names, ContextSet([module])) return module diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index 6702cf80..56bb3586 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -21,6 +21,7 @@ class NamespaceObject(object): class MixedModuleContext(Context): + # TODO use ContextWrapper! type = 'mixed_module' def __init__(self, evaluator, tree_module, namespaces, path, code_lines): diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index a03be567..3a2aea7e 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -211,6 +211,7 @@ def _create(evaluator, access_handle, parent_context, *args): path=path, string_names=string_names, code_lines=code_lines, + is_package=hasattr(compiled_object, 'py__path__'), ) if name is not None: evaluator.module_cache.add(string_names, ContextSet([module_context])) diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 138ec498..704d1696 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -139,7 +139,7 @@ class ModuleContext(ModuleMixin, TreeContext): api_type = u'module' parent_context = None - def __init__(self, evaluator, module_node, path, string_names, code_lines): + def __init__(self, evaluator, module_node, path, string_names, code_lines, is_package=False): super(ModuleContext, self).__init__( evaluator, parent_context=None, @@ -148,19 +148,7 @@ class ModuleContext(ModuleMixin, TreeContext): self._path = path self.string_names = string_names self.code_lines = code_lines - - def _get_init_directory(self): - """ - :return: The path to the directory of a package. None in case it's not - a package. - """ - for suffix in all_suffixes() + ['.pyi']: - ending = '__init__' + suffix - py__file__ = self.py__file__() - if py__file__ is not None and py__file__.endswith(ending): - # Remove the ending, including the separator. - return self.py__file__()[:-len(ending) - 1] - return None + self.is_package = is_package def py__name__(self): if self.string_names is None: @@ -176,39 +164,36 @@ class ModuleContext(ModuleMixin, TreeContext): return os.path.abspath(self._path) - def is_package(self): - return self._get_init_directory() is not None - def py__package__(self): - if self._get_init_directory() is None: - return re.sub(r'\.?[^.]+$', '', self.py__name__()).split('.') - else: + if self.is_package: return self.string_names + return self.string_names[:-1] def _py__path__(self): - search_path = self.evaluator.get_sys_path() - init_path = self.py__file__() - if os.path.basename(init_path) in ('__init__.py', '__init__.pyi'): - with open(init_path, 'rb') as f: - content = python_bytes_to_unicode(f.read(), errors='replace') - # these are strings that need to be used for namespace packages, - # the first one is ``pkgutil``, the second ``pkg_resources``. - options = ('declare_namespace(__name__)', 'extend_path(__path__') - if options[0] in content or options[1] in content: - # It is a namespace, now try to find the rest of the - # modules on sys_path or whatever the search_path is. - paths = set() - for s in search_path: - other = os.path.join(s, self.name.string_name) - if os.path.isdir(other): - paths.add(other) - if paths: - return list(paths) - # TODO I'm not sure if this is how nested namespace - # packages work. The tests are not really good enough to - # show that. - # Default to this. - return [self._get_init_directory()] + # A namespace package is typically auto generated and ~10 lines long. + first_few_lines = ''.join(self.code_lines[:50]) + # these are strings that need to be used for namespace packages, + # the first one is ``pkgutil``, the second ``pkg_resources``. + options = ('declare_namespace(__name__)', 'extend_path(__path__') + if options[0] in first_few_lines or options[1] in first_few_lines: + # It is a namespace, now try to find the rest of the + # modules on sys_path or whatever the search_path is. + paths = set() + for s in self.evaluator.get_sys_path(): + other = os.path.join(s, self.name.string_name) + if os.path.isdir(other): + paths.add(other) + if paths: + return list(paths) + # Nested namespace packages will not be supported. Nobody ever + # asked for it and in Python 3 they are there without using all the + # crap above. + + # Default to the of this file. + file = self.py__file__() + if file is None: + return None + return os.path.dirname(file) @property def py__path__(self): @@ -222,7 +207,7 @@ class ModuleContext(ModuleMixin, TreeContext): is a list of paths (strings). Raises an AttributeError if the module is not a package. """ - if self.is_package(): + if self.is_package: return self._py__path__ else: raise AttributeError('Only packages have __path__ attributes.') diff --git a/jedi/evaluate/gradual/typeshed.py b/jedi/evaluate/gradual/typeshed.py index 06b1ca3b..76d806e9 100644 --- a/jedi/evaluate/gradual/typeshed.py +++ b/jedi/evaluate/gradual/typeshed.py @@ -140,7 +140,7 @@ def import_module_decorator(func): if len(import_names) == 1: map_ = _cache_stub_file_map(evaluator.grammar.version_info) elif isinstance(parent_module_context, StubModuleContext): - if not parent_module_context.stub_context.is_package(): + if not parent_module_context.stub_context.is_package: # Only if it's a package (= a folder) something can be # imported. return context_set @@ -161,6 +161,7 @@ def import_module_decorator(func): module_cls = TypingModuleWrapper else: module_cls = StubOnlyModuleContext + file_name = os.path.basename(path) stub_module_context = module_cls( context_set, evaluator, stub_module_node, path=path, @@ -168,6 +169,7 @@ def import_module_decorator(func): # The code was loaded with latest_grammar, so use # that. code_lines=get_cached_code_lines(evaluator.latest_grammar, path), + is_package=file_name == '__init__.pyi', ) modules = _merge_modules(context_set, stub_module_context) return ContextSet(modules) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index b0b2b10e..98172587 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -249,6 +249,7 @@ class Importer(object): if level: base = module_context.py__package__() if base == [''] or base == ['__main__']: + raise NotImplementedError(module_context.py__package__()) base = [] # We need to care for two cases, the first one is if it's a valid # Python import. This import has a properly defined module name @@ -487,6 +488,7 @@ def import_module(evaluator, import_names, parent_module_context, sys_path): evaluator, module_path, code, sys_path, import_names=import_names, safe_module_name=True, + is_package=is_pkg, ) if parent_module_context is None: @@ -497,7 +499,7 @@ def import_module(evaluator, import_names, parent_module_context, sys_path): def _load_module(evaluator, path=None, code=None, sys_path=None, - import_names=None, safe_module_name=False): + import_names=None, safe_module_name=False, is_package=False): if import_names is None: dotted_name = None else: @@ -530,6 +532,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, path=path, string_names=import_names, code_lines=get_cached_code_lines(evaluator.grammar, path), + is_package=is_package, ) else: assert dotted_name is not None @@ -561,19 +564,23 @@ def get_modules_containing_name(evaluator, modules, name): code = python_bytes_to_unicode(f.read(), errors='replace') if name in code: e_sys_path = evaluator.get_sys_path() - module_name = os.path.basename(path) - if module_name.endswith('.py'): - module_name = module_name[:-3] - if base_names: + module_name = os.path.basename(path) + module_name = sys_path.remove_python_path_suffix(module_name) + is_package = module_name == '__init__' + if is_package: + raise NotImplementedError( + "This is probably not possible yet, please add a failing test first") + module_name = os.path.basename(os.path.dirname(path)) import_names = base_names + (module_name,) else: - import_names = sys_path.transform_path_to_dotted(e_sys_path, path) + import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) module = _load_module( evaluator, path, code, sys_path=e_sys_path, import_names=import_names, + is_package=is_package, ) evaluator.module_cache.add(import_names, ContextSet([module])) return module @@ -590,10 +597,7 @@ def get_modules_containing_name(evaluator, modules, name): if path is not None: if path not in used_mod_paths: used_mod_paths.add(path) - string_names = m.string_names - if not m.is_package() and string_names is not None: - string_names = string_names[:-1] - path_with_names_to_be_checked.append((path, string_names)) + path_with_names_to_be_checked.append((path, m.py__package__())) yield m if not settings.dynamic_params_for_other_modules: diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index d720421b..ae5c4dac 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -197,25 +197,33 @@ def _get_buildout_script_paths(search_path): continue +def remove_python_path_suffix(path): + for suffix in all_suffixes(): + if path.endswith(suffix): + path = path[:-len(suffix)] + break + return path + + def transform_path_to_dotted(sys_path, module_path): """ Returns the dotted path inside a sys.path as a list of names. e.g. >>> transform_path_to_dotted(["/foo"], '/foo/bar/baz.py') - ('bar', 'baz') + ('bar', 'baz'), False - Returns None if the path doesn't really resolve to anything. + Returns (None, False) if the path doesn't really resolve to anything. + The second return part is if it is a package. """ # First remove the suffix. - for suffix in all_suffixes(): - if module_path.endswith(suffix): - module_path = module_path[:-len(suffix)] - break + module_path = remove_python_path_suffix(module_path) + # Once the suffix was removed we are using the files as we know them. This # means that if someone uses an ending like .vim for a Python file, .vim # will be part of the returned dotted part. - if module_path.endswith(os.path.sep + '__init__'): + is_package = module_path.endswith(os.path.sep + '__init__') + if is_package: # -1 to remove the separator module_path = module_path[:-len('__init__') - 1] @@ -231,6 +239,6 @@ def transform_path_to_dotted(sys_path, module_path): split = rest.split(os.path.sep) for string in split: if not string: - return None - return tuple(split) - return None + return None, False + return tuple(split), is_package + return None, False diff --git a/test/test_evaluate/test_imports.py b/test/test_evaluate/test_imports.py index c2fbbaae..c3cf96fc 100644 --- a/test/test_evaluate/test_imports.py +++ b/test/test_evaluate/test_imports.py @@ -36,9 +36,11 @@ def test_find_module_not_package(): assert is_package is False +pkg_zip_path = os.path.join(os.path.dirname(__file__), 'zipped_imports/pkg.zip') + + def test_find_module_package_zipped(Script, evaluator, environment): - path = os.path.join(os.path.dirname(__file__), 'zipped_imports/pkg.zip') - sys_path = environment.get_sys_path() + [path] + sys_path = environment.get_sys_path() + [pkg_zip_path] script = Script('import pkg; pkg.mod', sys_path=sys_path) assert len(script.completions()) == 1 @@ -52,6 +54,16 @@ def test_find_module_package_zipped(Script, evaluator, environment): assert is_package is True +def test_correct_zip_package_behavior(Script, evaluator, environment): + sys_path = environment.get_sys_path() + [pkg_zip_path] + pkg, = Script('import pkg', sys_path=sys_path).goto_definitions() + context, = pkg._name.infer() + assert context.py__file__() == pkg_zip_path + assert context.is_package is True + assert context.py__package__() == ('pkg',) + assert context.py__path__() == [pkg_zip_path] + + def test_find_module_not_package_zipped(Script, evaluator, environment): path = os.path.join(os.path.dirname(__file__), 'zipped_imports/not_pkg.zip') sys_path = environment.get_sys_path() + [path] diff --git a/test/test_evaluate/test_sys_path.py b/test/test_evaluate/test_sys_path.py index b457e7f7..d3e38ce5 100644 --- a/test/test_evaluate/test_sys_path.py +++ b/test/test_evaluate/test_sys_path.py @@ -67,30 +67,31 @@ _s = ['/a', '/b', '/c/d/'] @pytest.mark.parametrize( - 'sys_path_, module_path, result', [ - (_s, '/a/b', ('b',)), - (_s, '/a/b/c', ('b', 'c')), - (_s, '/a/b.py', ('b',)), - (_s, '/a/b/c.py', ('b', 'c')), - (_s, '/x/b.py', None), - (_s, '/c/d/x.py', ('x',)), - (_s, '/c/d/x.py', ('x',)), - (_s, '/c/d/x/y.py', ('x', 'y')), + 'sys_path_, module_path, expected, is_package', [ + (_s, '/a/b', ('b',), False), + (_s, '/a/b/c', ('b', 'c'), False), + (_s, '/a/b.py', ('b',), False), + (_s, '/a/b/c.py', ('b', 'c'), False), + (_s, '/x/b.py', None, False), + (_s, '/c/d/x.py', ('x',), False), + (_s, '/c/d/x.py', ('x',), False), + (_s, '/c/d/x/y.py', ('x', 'y'), False), # If dots are in there they also resolve. These are obviously illegal # in Python, but Jedi can handle them. Give the user a bit more freedom # that he will have to correct eventually. - (_s, '/a/b.c.py', ('b.c',)), - (_s, '/a/b.d/foo.bar.py', ('b.d', 'foo.bar')), + (_s, '/a/b.c.py', ('b.c',), False), + (_s, '/a/b.d/foo.bar.py', ('b.d', 'foo.bar'), False), - (_s, '/a/.py', None), - (_s, '/a/c/.py', None), + (_s, '/a/.py', None, False), + (_s, '/a/c/.py', None, False), - (['/foo'], '/foo/bar/__init__.py', ('bar',)), - (['/foo'], '/foo/bar/baz/__init__.py', ('bar', 'baz')), - (['/foo'], '/foo/bar.so', ('bar',)), - (['/foo'], '/foo/bar/__init__.so', ('bar',)), - (['/foo'], '/x/bar.py', None), - (['/foo'], '/foo/bar.xyz', ('bar.xyz',)), + (['/foo'], '/foo/bar/__init__.py', ('bar',), True), + (['/foo'], '/foo/bar/baz/__init__.py', ('bar', 'baz'), True), + (['/foo'], '/foo/bar.so', ('bar',), False), + (['/foo'], '/foo/bar/__init__.so', ('bar',), True), + (['/foo'], '/x/bar.py', None, False), + (['/foo'], '/foo/bar.xyz', ('bar.xyz',), False), ]) -def test_calculate_dotted_from_path(sys_path_, module_path, result): - assert sys_path.transform_path_to_dotted(sys_path_, module_path) == result +def test_calculate_dotted_from_path(sys_path_, module_path, expected, is_package): + assert sys_path.transform_path_to_dotted(sys_path_, module_path) \ + == (expected, is_package)