Merge branch 'dev' of github.com:davidhalter/jedi into dev

AUTHORS.txt

@@ -41,6 +41,7 @@ Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
 Cristi Burcă (@scribu)
 bstaint (@bstaint)
 Mathias Rav (@Mortal) <rav@cs.au.dk>
+Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 
 
 Note: (@user) means a github user name.

jedi/_compatibility.py

@@ -34,8 +34,29 @@ class DummyFile(object):
         del self.loader
 
 
-def find_module_py33(string, path=None):
+def find_module_py34(string, path=None, fullname=None):
-    loader = importlib.machinery.PathFinder.find_module(string, path)
+    implicit_namespace_pkg = False
+    spec = None
+    loader = None
+
+    spec = importlib.machinery.PathFinder.find_spec(string, path)
+    if hasattr(spec, 'origin'):
+        origin = spec.origin
+        implicit_namespace_pkg = origin == 'namespace'
+
+    # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs
+    if implicit_namespace_pkg:
+        fullname = string if not path else fullname
+        implicit_ns_info = ImplicitNSInfo(fullname, spec.submodule_search_locations._path)
+        return None, implicit_ns_info, False
+
+    # we have found the tail end of the dotted path
+    if hasattr(spec, 'loader'):
+        loader = spec.loader
+    return find_module_py33(string, path, loader)
+
+def find_module_py33(string, path=None, loader=None, fullname=None):
+    loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
         try:
@@ -81,7 +102,7 @@ def find_module_py33(string, path=None):
     return module_file, module_path, is_package
 
 
-def find_module_pre_py33(string, path=None):
+def find_module_pre_py33(string, path=None, fullname=None):
     try:
         module_file, module_path, description = imp.find_module(string, path)
         module_type = description[2]
@@ -121,6 +142,7 @@ def find_module_pre_py33(string, path=None):
 
 
 find_module = find_module_py33 if is_py33 else find_module_pre_py33
+find_module = find_module_py34 if is_py34  else find_module
 find_module.__doc__ = """
 Provides information about a module.
 
@@ -132,6 +154,12 @@ if the module is contained in a package.
 """
 
 
+class ImplicitNSInfo(object):
+    """Stores information returned from an implicit namespace spec"""
+    def __init__(self, name, paths):
+        self.name = name
+        self.paths = paths
+
 # unicode function
 try:
     unicode = unicode

jedi/api/usages.py

@@ -14,6 +14,7 @@ def usages(evaluator, definition_names, mods):
             if name.api_type == 'module':
                 found = False
                 for context in name.infer():
+                    if isinstance(context, ModuleContext):
                         found = True
                         yield context.name
                 if not found:

jedi/evaluate/imports.py

@@ -16,7 +16,7 @@ import os
 import pkgutil
 import sys
 
-from jedi._compatibility import find_module, unicode
+from jedi._compatibility import find_module, unicode, ImplicitNSInfo
 from jedi import debug
 from jedi import settings
 from jedi.common import source_to_unicode, unite
@@ -306,8 +306,10 @@ class Importer(object):
                     # At the moment we are only using one path. So this is
                     # not important to be correct.
                     try:
+                        if not isinstance(path, list):
+                            path = [path]
                         module_file, module_path, is_pkg = \
-                            find_module(import_parts[-1], [path])
+                            find_module(import_parts[-1], path, fullname=module_name)
                         break
                     except ImportError:
                         module_path = None
@@ -323,7 +325,7 @@ class Importer(object):
                 sys.path, temp = sys_path, sys.path
                 try:
                     module_file, module_path, is_pkg = \
-                        find_module(import_parts[-1])
+                        find_module(import_parts[-1], fullname=module_name)
                 finally:
                     sys.path = temp
             except ImportError:
@@ -343,7 +345,12 @@ class Importer(object):
             source = module_file.read()
             module_file.close()
 
-        if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
+        if isinstance(module_path, ImplicitNSInfo):
+            from jedi.evaluate.representation import ImplicitNamespaceContext
+            fullname, paths = module_path.name, module_path.paths
+            module = ImplicitNamespaceContext(self._evaluator, fullname=fullname)
+            module.paths = paths
+        elif module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
             module = compiled.load_module(self._evaluator, module_path)
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path, parent_module)
@@ -384,7 +391,7 @@ class Importer(object):
         :param only_modules: Indicates wheter it's possible to import a
             definition that is not defined in a module.
         """
-        from jedi.evaluate.representation import ModuleContext
+        from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext
         names = []
         if self.import_path:
             # flask
@@ -405,13 +412,16 @@ class Importer(object):
                 # Non-modules are not completable.
                 if context.api_type != 'module':  # not a module
                     continue
-
                 # namespace packages
-                if isinstance(context, ModuleContext) and \
+                if isinstance(context, ModuleContext) and context.py__file__().endswith('__init__.py'):
-                        context.py__file__().endswith('__init__.py'):
                     paths = context.py__path__()
                     names += self._get_module_names(paths, in_module=context)
 
+                # implicit namespace packages
+                elif isinstance(context, ImplicitNamespaceContext):
+                    paths = context.paths
+                    names += self._get_module_names(paths)
+
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to
                     # add all the variables.

jedi/evaluate/representation.py

@@ -39,6 +39,7 @@ import os
 import pkgutil
 import imp
 import re
+from itertools import chain
 
 from jedi._compatibility import use_metaclass
 from jedi.parser import tree
@@ -557,3 +558,65 @@ class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)):
 
     def py__class__(self):
         return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')
+
+
+class ImplicitNSName(AbstractNameDefinition):
+    """
+    Accessing names for implicit namespace packages should infer to nothing.
+    This object will prevent Jedi from raising exceptions
+    """
+    def __init__(self, implicit_ns_context, string_name):
+        self.implicit_ns_context = implicit_ns_context
+        self.string_name = string_name
+
+    def infer(self):
+        return []
+
+    def get_root_context(self):
+        return self.implicit_ns_context
+
+
+class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, context.TreeContext)):
+    """
+    Provides support for implicit namespace packages
+    """
+    api_type = 'module'
+    parent_context = None
+
+    def __init__(self, evaluator, fullname):
+        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
+        self.evaluator = evaluator
+        self.fullname = fullname
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield DictFilter(self._sub_modules_dict())
+
+    @property
+    @memoize_default()
+    def name(self):
+        string_name = self.py__package__().rpartition('.')[-1]
+        return ImplicitNSName(self, string_name)
+
+    def py__file__(self):
+        return None
+
+    def py__package__(self):
+        """Return the fullname
+        """
+        return self.fullname
+
+    @property
+    def py__path__(self):
+        return lambda: [self.paths]
+
+    @memoize_default()
+    def _sub_modules_dict(self):
+        names = {}
+
+        paths = self.paths
+        file_names = chain.from_iterable(os.listdir(path) for path in paths)
+        mods = [file_name.rpartition('.')[0] if '.' in file_name else file_name for file_name in file_names]
+
+        for name in mods:
+            names[name] = imports.SubModuleName(self, name)
+        return names

jedi/parser/tokenize.py

@@ -239,6 +239,8 @@ def generate_tokens(readline, use_exact_op_types=False):
         if not line:
             if contstr:
                 yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix)
+                if contstr.endswith('\n'):
+                    new_line = True
             break
 
         lnum += 1

jedi/parser/tree.py

@@ -317,7 +317,7 @@ class Leaf(Base):
 
     @utf8_repr
     def __repr__(self):
-        return "<%s: %s>" % (type(self).__name__, self.value)
+        return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos)
 
 
 class LeafWithNewLines(Leaf):

test/completion/usages.py

@@ -83,17 +83,18 @@ import module_not_exists
 module_not_exists
 
 
-#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 72, 16)
+#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (11,17), (14,17), ('imports', 72, 16)
 from import_tree import rename1
 
-#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0)
+#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (8,32), (5,0)
 rename1.abc
 
-#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (4,32), (1,0)
+#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (5,32), (2,0)
 from import_tree.rename1 import abc
+#< (-5,8), (-2,32), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,0), (3,32)
 abc
 
-#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 72, 16)
+#< 20 ('rename1', 1,0), ('rename2', 4,5), (-11,24), (-8,0), (-5,17), (0,17), (3,17), ('imports', 72, 16)
 from import_tree.rename1 import abc
 
 #< (0, 32),

test/test_evaluate/implicit_namespace_package/ns1/pkg/ns1_file.py

@@ -0,0 +1 @@
+foo = 'ns1_file!'

test/test_evaluate/implicit_namespace_package/ns2/pkg/ns2_file.py

@@ -0,0 +1 @@
+foo = 'ns2_file!'

test/test_evaluate/implicit_nested_namespaces/namespace/pkg/module.py

@@ -0,0 +1 @@
+CONST = 1

test/test_evaluate/test_implicit_namespace_package.py

@@ -0,0 +1,58 @@
+from os.path import dirname, join
+
+import jedi
+import pytest
+
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_implicit_namespace_package():
+    sys_path = [join(dirname(__file__), d)
+                for d in ['implicit_namespace_package/ns1', 'implicit_namespace_package/ns2']]
+
+    def script_with_path(*args, **kwargs):
+        return jedi.Script(sys_path=sys_path, *args, **kwargs)
+
+    # goto definition
+    assert script_with_path('from pkg import ns1_file').goto_definitions()
+    assert script_with_path('from pkg import ns2_file').goto_definitions()
+    assert not script_with_path('from pkg import ns3_file').goto_definitions()
+
+    # goto assignment
+    tests = {
+        'from pkg.ns2_file import foo': 'ns2_file!',
+        'from pkg.ns1_file import foo': 'ns1_file!',
+    }
+    for source, solution in tests.items():
+        ass = script_with_path(source).goto_assignments()
+        assert len(ass) == 1
+        assert ass[0].description == "foo = '%s'" % solution
+
+    # completion
+    completions = script_with_path('from pkg import ').completions()
+    names = [str(c.name) for c in completions]  # str because of unicode
+    compare = ['ns1_file', 'ns2_file']
+    # must at least contain these items, other items are not important
+    assert set(compare) == set(names)
+
+    tests = {
+        'from pkg import ns2_file as x': 'ns2_file!',
+        'from pkg import ns1_file as x': 'ns1_file!'
+    }
+    for source, solution in tests.items():
+        for c in script_with_path(source + '; x.').completions():
+            if c.name == 'foo':
+                completion = c
+        solution = "foo = '%s'" % solution
+        assert completion.description == solution
+
+@pytest.mark.skipif('sys.version_info[:2] < (3,4)')
+def test_implicit_nested_namespace_package():
+    CODE = 'from implicit_nested_namespaces.namespace.pkg.module import CONST'
+
+    sys_path = [dirname(__file__)]
+
+    script = jedi.Script(sys_path=sys_path, source=CODE, line=1, column=61)
+
+    result = script.goto_definitions()
+
+    assert len(result) == 1

test/test_parser/test_old_fast_parser.py

@@ -257,7 +257,7 @@ def test_string_literals():
     """)
 
     script = jedi.Script(dedent(source))
-    script._get_module().tree_node.end_pos == (6, 0)
+    assert script._get_module().tree_node.end_pos == (6, 0)
     assert script.completions()
 
 

test/test_parser/test_tokenize.py

@@ -82,6 +82,39 @@ class TokenTest(unittest.TestCase):
             if value == 'if':
                 assert prefix == '    '
 
+    def test_tokenize_multiline_I(self):
+        # Make sure multiline string having newlines have the end marker on the
+        # next line
+        from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER
+        fundef = u('''""""\n''')
+        fundef_io = StringIO(fundef)
+        tokens = tokenize.generate_tokens(fundef_io.readline)
+        token_list = list(tokens)
+        assert token_list == [TokenInfo(ERRORTOKEN, '""""\n', (1, 0), ''),
+                              TokenInfo(ENDMARKER ,       '', (2, 0), '')]
+
+    def test_tokenize_multiline_II(self):
+        # Make sure multiline string having no newlines have the end marker on
+        # same line
+        from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER
+        fundef = u('''""""''')
+        fundef_io = StringIO(fundef)
+        tokens = tokenize.generate_tokens(fundef_io.readline)
+        token_list = list(tokens)
+        assert token_list == [TokenInfo(ERRORTOKEN, '""""', (1, 0), ''),
+                              TokenInfo(ENDMARKER,      '', (1, 4), '')]
+
+    def test_tokenize_multiline_III(self):
+        # Make sure multiline string having newlines have the end marker on the
+        # next line even if several newline
+        from jedi.parser.tokenize import TokenInfo, ERRORTOKEN, ENDMARKER
+        fundef = u('''""""\n\n''')
+        fundef_io = StringIO(fundef)
+        tokens = tokenize.generate_tokens(fundef_io.readline)
+        token_list = list(tokens)
+        assert token_list == [TokenInfo(ERRORTOKEN, '""""\n\n', (1, 0), ''),
+                              TokenInfo(ENDMARKER,          '', (3, 0), '')]
+
     def test_identifier_contains_unicode(self):
         fundef = dedent(u('''
         def 我あφ():