diff --git a/AUTHORS.txt b/AUTHORS.txt index 6d0abbc5..48cb385f 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -41,6 +41,7 @@ Dmytro Sadovnychyi (@sadovnychyi) Cristi Burcă (@scribu) bstaint (@bstaint) Mathias Rav (@Mortal) +Daniel Fiterman (@dfit99) Note: (@user) means a github user name. diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index ebce85aa..65e08855 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -34,8 +34,29 @@ class DummyFile(object): del self.loader -def find_module_py33(string, path=None): - loader = importlib.machinery.PathFinder.find_module(string, path) +def find_module_py34(string, path=None, fullname=None): + implicit_namespace_pkg = False + spec = None + loader = None + + spec = importlib.machinery.PathFinder.find_spec(string, path) + if hasattr(spec, 'origin'): + origin = spec.origin + implicit_namespace_pkg = origin == 'namespace' + + # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs + if implicit_namespace_pkg: + fullname = string if not path else fullname + implicit_ns_info = ImplicitNSInfo(fullname, spec.submodule_search_locations._path) + return None, implicit_ns_info, False + + # we have found the tail end of the dotted path + if hasattr(spec, 'loader'): + loader = spec.loader + return find_module_py33(string, path, loader) + +def find_module_py33(string, path=None, loader=None, fullname=None): + loader = loader or importlib.machinery.PathFinder.find_module(string, path) if loader is None and path is None: # Fallback to find builtins try: @@ -81,7 +102,7 @@ def find_module_py33(string, path=None): return module_file, module_path, is_package -def find_module_pre_py33(string, path=None): +def find_module_pre_py33(string, path=None, fullname=None): try: module_file, module_path, description = imp.find_module(string, path) module_type = description[2] @@ -121,6 +142,7 @@ def find_module_pre_py33(string, path=None): find_module = find_module_py33 if is_py33 else find_module_pre_py33 +find_module = find_module_py34 if is_py34 else find_module find_module.__doc__ = """ Provides information about a module. @@ -132,6 +154,12 @@ if the module is contained in a package. """ +class ImplicitNSInfo(object): + """Stores information returned from an implicit namespace spec""" + def __init__(self, name, paths): + self.name = name + self.paths = paths + # unicode function try: unicode = unicode diff --git a/jedi/api/usages.py b/jedi/api/usages.py index 254f7e1c..f9dba6ec 100644 --- a/jedi/api/usages.py +++ b/jedi/api/usages.py @@ -14,8 +14,9 @@ def usages(evaluator, definition_names, mods): if name.api_type == 'module': found = False for context in name.infer(): - found = True - yield context.name + if isinstance(context, ModuleContext): + found = True + yield context.name if not found: yield name else: diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index fa6bf929..6fee92b8 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -16,7 +16,7 @@ import os import pkgutil import sys -from jedi._compatibility import find_module, unicode +from jedi._compatibility import find_module, unicode, ImplicitNSInfo from jedi import debug from jedi import settings from jedi.common import source_to_unicode, unite @@ -306,8 +306,10 @@ class Importer(object): # At the moment we are only using one path. So this is # not important to be correct. try: + if not isinstance(path, list): + path = [path] module_file, module_path, is_pkg = \ - find_module(import_parts[-1], [path]) + find_module(import_parts[-1], path, fullname=module_name) break except ImportError: module_path = None @@ -323,7 +325,7 @@ class Importer(object): sys.path, temp = sys_path, sys.path try: module_file, module_path, is_pkg = \ - find_module(import_parts[-1]) + find_module(import_parts[-1], fullname=module_name) finally: sys.path = temp except ImportError: @@ -343,7 +345,12 @@ class Importer(object): source = module_file.read() module_file.close() - if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')): + if isinstance(module_path, ImplicitNSInfo): + from jedi.evaluate.representation import ImplicitNamespaceContext + fullname, paths = module_path.name, module_path.paths + module = ImplicitNamespaceContext(self._evaluator, fullname=fullname) + module.paths = paths + elif module_file is None and not module_path.endswith(('.py', '.zip', '.egg')): module = compiled.load_module(self._evaluator, module_path) else: module = _load_module(self._evaluator, module_path, source, sys_path, parent_module) @@ -384,7 +391,7 @@ class Importer(object): :param only_modules: Indicates wheter it's possible to import a definition that is not defined in a module. """ - from jedi.evaluate.representation import ModuleContext + from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext names = [] if self.import_path: # flask @@ -405,13 +412,16 @@ class Importer(object): # Non-modules are not completable. if context.api_type != 'module': # not a module continue - # namespace packages - if isinstance(context, ModuleContext) and \ - context.py__file__().endswith('__init__.py'): + if isinstance(context, ModuleContext) and context.py__file__().endswith('__init__.py'): paths = context.py__path__() names += self._get_module_names(paths, in_module=context) + # implicit namespace packages + elif isinstance(context, ImplicitNamespaceContext): + paths = context.paths + names += self._get_module_names(paths) + if only_modules: # In the case of an import like `from x.` we don't need to # add all the variables. diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index ff8071b6..bb21e93d 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -39,6 +39,7 @@ import os import pkgutil import imp import re +from itertools import chain from jedi._compatibility import use_metaclass from jedi.parser import tree @@ -557,3 +558,65 @@ class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)): def py__class__(self): return compiled.get_special_object(self.evaluator, 'MODULE_CLASS') + + +class ImplicitNSName(AbstractNameDefinition): + """ + Accessing names for implicit namespace packages should infer to nothing. + This object will prevent Jedi from raising exceptions + """ + def __init__(self, implicit_ns_context, string_name): + self.implicit_ns_context = implicit_ns_context + self.string_name = string_name + + def infer(self): + return [] + + def get_root_context(self): + return self.implicit_ns_context + + +class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, context.TreeContext)): + """ + Provides support for implicit namespace packages + """ + api_type = 'module' + parent_context = None + + def __init__(self, evaluator, fullname): + super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None) + self.evaluator = evaluator + self.fullname = fullname + + def get_filters(self, search_global, until_position=None, origin_scope=None): + yield DictFilter(self._sub_modules_dict()) + + @property + @memoize_default() + def name(self): + string_name = self.py__package__().rpartition('.')[-1] + return ImplicitNSName(self, string_name) + + def py__file__(self): + return None + + def py__package__(self): + """Return the fullname + """ + return self.fullname + + @property + def py__path__(self): + return lambda: [self.paths] + + @memoize_default() + def _sub_modules_dict(self): + names = {} + + paths = self.paths + file_names = chain.from_iterable(os.listdir(path) for path in paths) + mods = [file_name.rpartition('.')[0] if '.' in file_name else file_name for file_name in file_names] + + for name in mods: + names[name] = imports.SubModuleName(self, name) + return names \ No newline at end of file diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 1061a256..d54f698b 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -239,6 +239,8 @@ def generate_tokens(readline, use_exact_op_types=False): if not line: if contstr: yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix) + if contstr.endswith('\n'): + new_line = True break lnum += 1 diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 0618e4a9..f1bd952c 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -317,7 +317,7 @@ class Leaf(Base): @utf8_repr def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.value) + return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos) class LeafWithNewLines(Leaf): diff --git a/test/completion/usages.py b/test/completion/usages.py index b158e157..eabce568 100644 --- a/test/completion/usages.py +++ b/test/completion/usages.py @@ -83,17 +83,18 @@ import module_not_exists module_not_exists -#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 72, 16) +#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (11,17), (14,17), ('imports', 72, 16) from import_tree import rename1 -#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0) +#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (8,32), (5,0) rename1.abc -#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (4,32), (1,0) +#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (5,32), (2,0) from import_tree.rename1 import abc +#< (-5,8), (-2,32), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,0), (3,32) abc -#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 72, 16) +#< 20 ('rename1', 1,0), ('rename2', 4,5), (-11,24), (-8,0), (-5,17), (0,17), (3,17), ('imports', 72, 16) from import_tree.rename1 import abc #< (0, 32), diff --git a/test/test_evaluate/implicit_namespace_package/ns1/pkg/ns1_file.py b/test/test_evaluate/implicit_namespace_package/ns1/pkg/ns1_file.py new file mode 100644 index 00000000..940279f9 --- /dev/null +++ b/test/test_evaluate/implicit_namespace_package/ns1/pkg/ns1_file.py @@ -0,0 +1 @@ +foo = 'ns1_file!' diff --git a/test/test_evaluate/implicit_namespace_package/ns2/pkg/ns2_file.py b/test/test_evaluate/implicit_namespace_package/ns2/pkg/ns2_file.py new file mode 100644 index 00000000..e87d7d88 --- /dev/null +++ b/test/test_evaluate/implicit_namespace_package/ns2/pkg/ns2_file.py @@ -0,0 +1 @@ +foo = 'ns2_file!' diff --git a/test/test_evaluate/implicit_nested_namespaces/namespace/pkg/module.py b/test/test_evaluate/implicit_nested_namespaces/namespace/pkg/module.py new file mode 100644 index 00000000..3c378205 --- /dev/null +++ b/test/test_evaluate/implicit_nested_namespaces/namespace/pkg/module.py @@ -0,0 +1 @@ +CONST = 1 diff --git a/test/test_evaluate/test_implicit_namespace_package.py b/test/test_evaluate/test_implicit_namespace_package.py new file mode 100644 index 00000000..428a5d7c --- /dev/null +++ b/test/test_evaluate/test_implicit_namespace_package.py @@ -0,0 +1,58 @@ +from os.path import dirname, join + +import jedi +import pytest + + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_implicit_namespace_package(): + sys_path = [join(dirname(__file__), d) + for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']] + + def script_with_path(*args, **kwargs): + return jedi.Script(sys_path=sys_path, *args, **kwargs) + + # goto definition + assert script_with_path('from pkg import ns1_file').goto_definitions() + assert script_with_path('from pkg import ns2_file').goto_definitions() + assert not script_with_path('from pkg import ns3_file').goto_definitions() + + # goto assignment + tests = { + 'from pkg.ns2_file import foo': 'ns2_file!', + 'from pkg.ns1_file import foo': 'ns1_file!', + } + for source, solution in tests.items(): + ass = script_with_path(source).goto_assignments() + assert len(ass) == 1 + assert ass[0].description == "foo = '%s'" % solution + + # completion + completions = script_with_path('from pkg import ').completions() + names = [str(c.name) for c in completions] # str because of unicode + compare = ['ns1_file', 'ns2_file'] + # must at least contain these items, other items are not important + assert set(compare) == set(names) + + tests = { + 'from pkg import ns2_file as x': 'ns2_file!', + 'from pkg import ns1_file as x': 'ns1_file!' + } + for source, solution in tests.items(): + for c in script_with_path(source + '; x.').completions(): + if c.name == 'foo': + completion = c + solution = "foo = '%s'" % solution + assert completion.description == solution + +@pytest.mark.skipif('sys.version_info[:2] < (3,4)') +def test_implicit_nested_namespace_package(): + CODE = 'from implicit_nested_namespaces.namespace.pkg.module import CONST' + + sys_path = [dirname(__file__)] + + script = jedi.Script(sys_path=sys_path, source=CODE, line=1, column=61) + + result = script.goto_definitions() + + assert len(result) == 1 diff --git a/test/test_parser/test_old_fast_parser.py b/test/test_parser/test_old_fast_parser.py index e4553833..5ce6a1b9 100644 --- a/test/test_parser/test_old_fast_parser.py +++ b/test/test_parser/test_old_fast_parser.py @@ -257,7 +257,7 @@ def test_string_literals(): """) script = jedi.Script(dedent(source)) - script._get_module().tree_node.end_pos == (6, 0) + assert script._get_module().tree_node.end_pos == (6, 0) assert script.completions() diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 1ee880ea..72020a54 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -82,6 +82,39 @@ class TokenTest(unittest.TestCase): if value == 'if': assert prefix == ' ' + def test_tokenize_multiline_I(self): + # Make sure multiline string having newlines have the end marker on the + # next line + from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER + fundef = u('''""""\n''') + fundef_io = StringIO(fundef) + tokens = tokenize.generate_tokens(fundef_io.readline) + token_list = list(tokens) + assert token_list == [TokenInfo(ERRORTOKEN, '""""\n', (1, 0), ''), + TokenInfo(ENDMARKER , '', (2, 0), '')] + + def test_tokenize_multiline_II(self): + # Make sure multiline string having no newlines have the end marker on + # same line + from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER + fundef = u('''""""''') + fundef_io = StringIO(fundef) + tokens = tokenize.generate_tokens(fundef_io.readline) + token_list = list(tokens) + assert token_list == [TokenInfo(ERRORTOKEN, '""""', (1, 0), ''), + TokenInfo(ENDMARKER, '', (1, 4), '')] + + def test_tokenize_multiline_III(self): + # Make sure multiline string having newlines have the end marker on the + # next line even if several newline + from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER + fundef = u('''""""\n\n''') + fundef_io = StringIO(fundef) + tokens = tokenize.generate_tokens(fundef_io.readline) + token_list = list(tokens) + assert token_list == [TokenInfo(ERRORTOKEN, '""""\n\n', (1, 0), ''), + TokenInfo(ENDMARKER, '', (3, 0), '')] + def test_identifier_contains_unicode(self): fundef = dedent(u(''' def 我あφ():