diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 43410a49..c03940fe 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -26,7 +26,6 @@ from jedi.api import usages from jedi.api import helpers from jedi.api.completion import Completion from jedi.evaluate import Evaluator -from jedi.evaluate import representation as er from jedi.evaluate import imports from jedi.evaluate.param import try_iter_content from jedi.evaluate.helpers import get_module_names, evaluate_call_of_leaf @@ -34,6 +33,8 @@ from jedi.evaluate.sys_path import get_venv_path, dotted_path_in_sys_path from jedi.evaluate.iterable import unpack_tuple_to_dict from jedi.evaluate.filters import TreeNameDefinition from jedi.evaluate.syntax_tree import tree_name_to_contexts +from jedi.evaluate.context.module import ModuleContext +from jedi.evaluate.context.module import ModuleName # Jedi uses lots and lots of recursion. By setting this a little bit higher, we # can remove some "maximum recursion depth" errors. @@ -128,7 +129,7 @@ class Script(object): @cache.memoize_method def _get_module(self): - module = er.ModuleContext( + module = ModuleContext( self._evaluator, self._get_module_node(), self.path @@ -208,7 +209,7 @@ class Script(object): names = self._goto() if follow_imports: def check(name): - if isinstance(name, er.ModuleName): + if isinstance(name, ModuleName): return False return name.api_type == 'module' else: diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index e4b5dbb6..96507176 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -2,7 +2,7 @@ TODO Some parts of this module are still not well documented. """ -from jedi.evaluate.representation import ModuleContext +from jedi.evaluate.context.module import ModuleContext from jedi.evaluate import compiled from jedi.evaluate.compiled import mixed from jedi.evaluate.context import Context diff --git a/jedi/api/usages.py b/jedi/api/usages.py index 47b100ed..4d119e3c 100644 --- a/jedi/api/usages.py +++ b/jedi/api/usages.py @@ -2,7 +2,7 @@ from jedi.api import classes from parso.python import tree from jedi.evaluate import imports from jedi.evaluate.filters import TreeNameDefinition -from jedi.evaluate.representation import ModuleContext +from jedi.evaluate.context.module import ModuleContext def compare_contexts(c1, c2): diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index cdd00232..d4fa6a1b 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -92,7 +92,7 @@ def _check_for_setattr(instance): """ Check if there's any setattr method inside an instance. If so, return True. """ - from jedi.evaluate.representation import ModuleContext + from jedi.evaluate.context.module import ModuleContext module = instance.get_root_context() if not isinstance(module, ModuleContext): return False diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index be7fa34c..dd2a07cf 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -206,7 +206,7 @@ def _create(evaluator, obj, parent_context=None, *args): if parent_context.tree_node.get_root_node() == module_node: module_context = parent_context.get_root_context() else: - from jedi.evaluate.representation import ModuleContext + from jedi.evaluate.context.module import ModuleContext module_context = ModuleContext(evaluator, module_node, path=path) # TODO this __name__ is probably wrong. name = compiled_object.get_root_context().py__name__() diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py new file mode 100644 index 00000000..fc0d291b --- /dev/null +++ b/jedi/evaluate/context/module.py @@ -0,0 +1,212 @@ +import pkgutil +import imp +import re +import os + +from parso import python_bytes_to_unicode + +from jedi._compatibility import use_metaclass +from jedi.evaluate.cache import CachedMetaClass, evaluator_method_cache +from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \ + AbstractNameDefinition, ParserTreeFilter, DictFilter +from jedi.evaluate import compiled +from jedi.evaluate.context import TreeContext +from jedi.evaluate.imports import SubModuleName, infer_import + +class _ModuleAttributeName(AbstractNameDefinition): + """ + For module attributes like __file__, __str__ and so on. + """ + api_type = 'instance' + + def __init__(self, parent_module, string_name): + self.parent_context = parent_module + self.string_name = string_name + + def infer(self): + return compiled.create(self.parent_context.evaluator, str).execute_evaluated() + + +class ModuleName(ContextNameMixin, AbstractNameDefinition): + start_pos = 1, 0 + + def __init__(self, context, name): + self._context = context + self._name = name + + @property + def string_name(self): + return self._name + + +class ModuleContext(use_metaclass(CachedMetaClass, TreeContext)): + api_type = 'module' + parent_context = None + + def __init__(self, evaluator, module_node, path): + super(ModuleContext, self).__init__(evaluator, parent_context=None) + self.tree_node = module_node + self._path = path + + def get_filters(self, search_global, until_position=None, origin_scope=None): + yield ParserTreeFilter( + self.evaluator, + context=self, + until_position=until_position, + origin_scope=origin_scope + ) + yield GlobalNameFilter(self, self.tree_node) + yield DictFilter(self._sub_modules_dict()) + yield DictFilter(self._module_attributes_dict()) + for star_module in self.star_imports(): + yield next(star_module.get_filters(search_global)) + + # I'm not sure if the star import cache is really that effective anymore + # with all the other really fast import caches. Recheck. Also we would need + # to push the star imports into Evaluator.modules, if we reenable this. + @evaluator_method_cache([]) + def star_imports(self): + modules = [] + for i in self.tree_node.iter_imports(): + if i.is_star_import(): + name = i.get_paths()[-1][-1] + new = infer_import(self, name) + for module in new: + if isinstance(module, ModuleContext): + modules += module.star_imports() + modules += new + return modules + + @evaluator_method_cache() + def _module_attributes_dict(self): + names = ['__file__', '__package__', '__doc__', '__name__'] + # All the additional module attributes are strings. + return dict((n, _ModuleAttributeName(self, n)) for n in names) + + @property + def _string_name(self): + """ This is used for the goto functions. """ + if self._path is None: + return '' # no path -> empty name + else: + sep = (re.escape(os.path.sep),) * 2 + r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path) + # Remove PEP 3149 names + return re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) + + @property + @evaluator_method_cache() + def name(self): + return ModuleName(self, self._string_name) + + def _get_init_directory(self): + """ + :return: The path to the directory of a package. None in case it's not + a package. + """ + for suffix, _, _ in imp.get_suffixes(): + ending = '__init__' + suffix + py__file__ = self.py__file__() + if py__file__ is not None and py__file__.endswith(ending): + # Remove the ending, including the separator. + return self.py__file__()[:-len(ending) - 1] + return None + + def py__name__(self): + for name, module in self.evaluator.modules.items(): + if module == self and name != '': + return name + + return '__main__' + + def py__file__(self): + """ + In contrast to Python's __file__ can be None. + """ + if self._path is None: + return None + + return os.path.abspath(self._path) + + def py__package__(self): + if self._get_init_directory() is None: + return re.sub(r'\.?[^\.]+$', '', self.py__name__()) + else: + return self.py__name__() + + def _py__path__(self): + search_path = self.evaluator.sys_path + init_path = self.py__file__() + if os.path.basename(init_path) == '__init__.py': + with open(init_path, 'rb') as f: + content = python_bytes_to_unicode(f.read(), errors='replace') + # these are strings that need to be used for namespace packages, + # the first one is ``pkgutil``, the second ``pkg_resources``. + options = ('declare_namespace(__name__)', 'extend_path(__path__') + if options[0] in content or options[1] in content: + # It is a namespace, now try to find the rest of the + # modules on sys_path or whatever the search_path is. + paths = set() + for s in search_path: + other = os.path.join(s, self.name.string_name) + if os.path.isdir(other): + paths.add(other) + if paths: + return list(paths) + # TODO I'm not sure if this is how nested namespace + # packages work. The tests are not really good enough to + # show that. + # Default to this. + return [self._get_init_directory()] + + @property + def py__path__(self): + """ + Not seen here, since it's a property. The callback actually uses a + variable, so use it like:: + + foo.py__path__(sys_path) + + In case of a package, this returns Python's __path__ attribute, which + is a list of paths (strings). + Raises an AttributeError if the module is not a package. + """ + path = self._get_init_directory() + + if path is None: + raise AttributeError('Only packages have __path__ attributes.') + else: + return self._py__path__ + + @evaluator_method_cache() + def _sub_modules_dict(self): + """ + Lists modules in the directory of this module (if this module is a + package). + """ + path = self._path + names = {} + if path is not None and path.endswith(os.path.sep + '__init__.py'): + mods = pkgutil.iter_modules([os.path.dirname(path)]) + for module_loader, name, is_pkg in mods: + # It's obviously a relative import to the current module. + names[name] = SubModuleName(self, name) + + # TODO add something like this in the future, its cleaner than the + # import hacks. + # ``os.path`` is a hardcoded exception, because it's a + # ``sys.modules`` modification. + # if str(self.name) == 'os': + # names.append(Name('path', parent_context=self)) + + return names + + def py__class__(self): + return compiled.get_special_object(self.evaluator, 'MODULE_CLASS') + + def __repr__(self): + return "<%s: %s@%s-%s>" % ( + self.__class__.__name__, self._string_name, + self.tree_node.start_pos[0], self.tree_node.end_pos[0]) + + diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index f5b4cd0d..623d7461 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -27,6 +27,8 @@ from jedi.evaluate.helpers import is_stdlib_path from jedi.evaluate.utils import to_list from jedi.evaluate.context import ContextSet from jedi.parser_utils import get_parent_scope +from jedi.evaluate.context.module import ModuleContext + MAX_PARAM_SEARCHES = 20 @@ -99,8 +101,6 @@ def _search_function_executions(evaluator, module_context, funcdef): """ Returns a list of param names. """ - from jedi.evaluate import representation as er - func_string_name = funcdef.name.value compare_node = funcdef if func_string_name == '__init__': @@ -113,7 +113,7 @@ def _search_function_executions(evaluator, module_context, funcdef): i = 0 for for_mod_context in imports.get_modules_containing_name( evaluator, [module_context], func_string_name): - if not isinstance(module_context, er.ModuleContext): + if not isinstance(module_context, ModuleContext): return for name, trailer in _get_possible_nodes(for_mod_context, func_string_name): i += 1 diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index f06d3b66..0478cd4b 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -420,7 +420,8 @@ class Importer(object): :param only_modules: Indicates wheter it's possible to import a definition that is not defined in a module. """ - from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext + from jedi.evaluate.context.module import ModuleContext + from jedi.evaluate.representation import ImplicitNamespaceContext names = [] if self.import_path: # flask @@ -489,7 +490,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N code=code, path=path, cache=True, diff_cache=True, cache_path=settings.cache_directory) - from jedi.evaluate.representation import ModuleContext + from jedi.evaluate.context.module import ModuleContext return ModuleContext(evaluator, module_node, path=path) else: return compiled.load_module(evaluator, path) @@ -508,7 +509,7 @@ def get_modules_containing_name(evaluator, modules, name): """ Search a name in the directories of modules. """ - from jedi.evaluate import representation as er + from jedi.evaluate.context.module import ModuleContext def check_python_file(path): try: @@ -521,7 +522,7 @@ def get_modules_containing_name(evaluator, modules, name): return None else: module_node = node_cache_item.node - return er.ModuleContext(evaluator, module_node, path=path) + return ModuleContext(evaluator, module_node, path=path) def check_fs(path): with open(path, 'rb') as f: diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 869cf8a8..eec0e562 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -28,6 +28,7 @@ from parso.python import tree from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate import compiled from jedi.evaluate.context import LazyTreeContext, NO_CONTEXTS, ContextSet +from jedi.evaluate.context.module import ModuleContext from jedi import debug from jedi import _compatibility from jedi import parser_utils @@ -150,7 +151,6 @@ def py__getitem__(context, typ, node): # check for the instance typing._Optional (Python 3.6). return context.eval_node(nodes[0]) - from jedi.evaluate.representation import ModuleContext typing = ModuleContext( context.evaluator, module_node=_get_typing_replacement_module(context.evaluator.latest_grammar), diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index a89e4875..2276e9ee 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -38,13 +38,9 @@ py__doc__(include_call_signature: Returns the docstring for a context. """ import os -import pkgutil -import imp -import re from itertools import chain from parso.python import tree -from parso import python_bytes_to_unicode from jedi._compatibility import use_metaclass from jedi import debug @@ -59,9 +55,8 @@ from jedi.evaluate import imports from jedi.evaluate import helpers from jedi.evaluate import iterable from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \ - GlobalNameFilter, DictFilter, ContextName, AbstractNameDefinition, \ - ParamName, AnonymousInstanceParamName, TreeNameDefinition, \ - ContextNameMixin + DictFilter, ContextName, AbstractNameDefinition, \ + ParamName, AnonymousInstanceParamName, TreeNameDefinition from jedi.evaluate import context from jedi.evaluate.context import ContextualizedNode, NO_CONTEXTS, \ ContextSet, iterator_to_context_set @@ -423,203 +418,6 @@ class FunctionExecutionContext(context.TreeContext): return self.var_args.get_params(self) -class ModuleAttributeName(AbstractNameDefinition): - """ - For module attributes like __file__, __str__ and so on. - """ - api_type = 'instance' - - def __init__(self, parent_module, string_name): - self.parent_context = parent_module - self.string_name = string_name - - def infer(self): - return compiled.create(self.parent_context.evaluator, str).execute_evaluated() - - -class ModuleName(ContextNameMixin, AbstractNameDefinition): - start_pos = 1, 0 - - def __init__(self, context, name): - self._context = context - self._name = name - - @property - def string_name(self): - return self._name - - -class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)): - api_type = 'module' - parent_context = None - - def __init__(self, evaluator, module_node, path): - super(ModuleContext, self).__init__(evaluator, parent_context=None) - self.tree_node = module_node - self._path = path - - def get_filters(self, search_global, until_position=None, origin_scope=None): - yield ParserTreeFilter( - self.evaluator, - context=self, - until_position=until_position, - origin_scope=origin_scope - ) - yield GlobalNameFilter(self, self.tree_node) - yield DictFilter(self._sub_modules_dict()) - yield DictFilter(self._module_attributes_dict()) - for star_module in self.star_imports(): - yield next(star_module.get_filters(search_global)) - - # I'm not sure if the star import cache is really that effective anymore - # with all the other really fast import caches. Recheck. Also we would need - # to push the star imports into Evaluator.modules, if we reenable this. - @evaluator_method_cache([]) - def star_imports(self): - modules = [] - for i in self.tree_node.iter_imports(): - if i.is_star_import(): - name = i.get_paths()[-1][-1] - new = imports.infer_import(self, name) - for module in new: - if isinstance(module, ModuleContext): - modules += module.star_imports() - modules += new - return modules - - @evaluator_method_cache() - def _module_attributes_dict(self): - names = ['__file__', '__package__', '__doc__', '__name__'] - # All the additional module attributes are strings. - return dict((n, ModuleAttributeName(self, n)) for n in names) - - @property - def _string_name(self): - """ This is used for the goto functions. """ - if self._path is None: - return '' # no path -> empty name - else: - sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path) - # Remove PEP 3149 names - return re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) - - @property - @evaluator_method_cache() - def name(self): - return ModuleName(self, self._string_name) - - def _get_init_directory(self): - """ - :return: The path to the directory of a package. None in case it's not - a package. - """ - for suffix, _, _ in imp.get_suffixes(): - ending = '__init__' + suffix - py__file__ = self.py__file__() - if py__file__ is not None and py__file__.endswith(ending): - # Remove the ending, including the separator. - return self.py__file__()[:-len(ending) - 1] - return None - - def py__name__(self): - for name, module in self.evaluator.modules.items(): - if module == self and name != '': - return name - - return '__main__' - - def py__file__(self): - """ - In contrast to Python's __file__ can be None. - """ - if self._path is None: - return None - - return os.path.abspath(self._path) - - def py__package__(self): - if self._get_init_directory() is None: - return re.sub(r'\.?[^\.]+$', '', self.py__name__()) - else: - return self.py__name__() - - def _py__path__(self): - search_path = self.evaluator.sys_path - init_path = self.py__file__() - if os.path.basename(init_path) == '__init__.py': - with open(init_path, 'rb') as f: - content = python_bytes_to_unicode(f.read(), errors='replace') - # these are strings that need to be used for namespace packages, - # the first one is ``pkgutil``, the second ``pkg_resources``. - options = ('declare_namespace(__name__)', 'extend_path(__path__') - if options[0] in content or options[1] in content: - # It is a namespace, now try to find the rest of the - # modules on sys_path or whatever the search_path is. - paths = set() - for s in search_path: - other = os.path.join(s, self.name.string_name) - if os.path.isdir(other): - paths.add(other) - if paths: - return list(paths) - # TODO I'm not sure if this is how nested namespace - # packages work. The tests are not really good enough to - # show that. - # Default to this. - return [self._get_init_directory()] - - @property - def py__path__(self): - """ - Not seen here, since it's a property. The callback actually uses a - variable, so use it like:: - - foo.py__path__(sys_path) - - In case of a package, this returns Python's __path__ attribute, which - is a list of paths (strings). - Raises an AttributeError if the module is not a package. - """ - path = self._get_init_directory() - - if path is None: - raise AttributeError('Only packages have __path__ attributes.') - else: - return self._py__path__ - - @evaluator_method_cache() - def _sub_modules_dict(self): - """ - Lists modules in the directory of this module (if this module is a - package). - """ - path = self._path - names = {} - if path is not None and path.endswith(os.path.sep + '__init__.py'): - mods = pkgutil.iter_modules([os.path.dirname(path)]) - for module_loader, name, is_pkg in mods: - # It's obviously a relative import to the current module. - names[name] = imports.SubModuleName(self, name) - - # TODO add something like this in the future, its cleaner than the - # import hacks. - # ``os.path`` is a hardcoded exception, because it's a - # ``sys.modules`` modification. - # if str(self.name) == 'os': - # names.append(Name('path', parent_context=self)) - - return names - - def py__class__(self): - return compiled.get_special_object(self.evaluator, 'MODULE_CLASS') - - def __repr__(self): - return "<%s: %s@%s-%s>" % ( - self.__class__.__name__, self._string_name, - self.tree_node.start_pos[0], self.tree_node.end_pos[0]) - - class ImplicitNSName(AbstractNameDefinition): """ Accessing names for implicit namespace packages should infer to nothing. diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 24cc3342..3230c857 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -23,6 +23,7 @@ from jedi.evaluate import param from jedi.evaluate import analysis from jedi.evaluate.context import LazyTreeContext, ContextualizedNode, \ NO_CONTEXTS, ContextSet +from jedi.evaluate.context.module import ModuleContext from jedi.evaluate.syntax_tree import is_string # Now this is all part of fake tuples in Jedi. However super doesn't work on @@ -58,7 +59,7 @@ def execute(evaluator, obj, arguments): else: if obj.parent_context == evaluator.BUILTINS: module_name = 'builtins' - elif isinstance(obj.parent_context, er.ModuleContext): + elif isinstance(obj.parent_context, ModuleContext): module_name = obj.parent_context.name.string_name else: module_name = '' @@ -293,7 +294,7 @@ def collections_namedtuple(evaluator, obj, arguments): # Parse source module = evaluator.grammar.parse(source) generated_class = next(module.iter_classdefs()) - parent_context = er.ModuleContext(evaluator, module, '') + parent_context = ModuleContext(evaluator, module, '') return ContextSet(er.ClassContext(evaluator, parent_context, generated_class)) diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 23e06e39..7eeaf1f4 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -219,7 +219,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script_path): debug.warning('Error trying to read buildout_script: %s', buildout_script_path) return - from jedi.evaluate.representation import ModuleContext + from jedi.evaluate.context.module import ModuleContext for path in _check_module(ModuleContext(evaluator, module_node, buildout_script_path)): yield path diff --git a/test/test_evaluate/test_buildout_detection.py b/test/test_evaluate/test_buildout_detection.py index 08e53099..04ede288 100644 --- a/test/test_evaluate/test_buildout_detection.py +++ b/test/test_evaluate/test_buildout_detection.py @@ -9,7 +9,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file, sys_path_with_modifications, _check_module) from jedi.evaluate import Evaluator -from jedi.evaluate.representation import ModuleContext +from jedi.evaluate.context.module import ModuleContext from ..helpers import cwd_at