Merge branch 'dev' into get_code_fidelity

Conflicts:
	jedi/evaluate/docstrings.py
	jedi/parser/representation.py

Resolving merge problems:
* Introducing docstring for compiled.fake
* Partly fixing poor decision in TokenDocstring __init__

test/completion/generators.py

@@ -1,7 +1,6 @@
 # -----------------
 # yield statement
 # -----------------
-
 def gen():
     yield 1
     yield ""
@@ -13,6 +12,7 @@ next(gen_exe)
 #? int() str() list
 next(gen_exe, list)
 
+
 def gen_ret(value):
     yield value
 
@@ -22,6 +22,7 @@ next(gen_ret(1))
 #? []
 next(gen_ret())
 
+
 # -----------------
 # generators should not be indexable
 # -----------------
@@ -39,6 +40,7 @@ for a in get():
     #? int() str()
     a
 
+
 class Get():
     def __iter__(self):
         yield 1
@@ -63,6 +65,7 @@ g = iter([1.0])
 #? float()
 next(g)
 
+
 # -----------------
 # __next__
 # -----------------
@@ -90,6 +93,7 @@ for c in Counter(3, 8):
     #? int()
     print c
 
+
 # -----------------
 # tuples
 # -----------------
@@ -105,3 +109,30 @@ a, b = next(gen())
 a
 #? str() float()
 b
+
+# -----------------
+# More complicated access
+# -----------------
+
+# `close` is a method wrapper.
+#? ['__call__']
+gen().close.__call__
+
+#? 
+gen().throw()
+
+#? ['co_consts']
+gen().gi_code.co_consts
+
+#? []
+gen.gi_code.co_consts
+
+# `send` is also a method wrapper.
+#? ['__call__']
+gen().send.__call__
+
+#? tuple()
+gen().send()
+
+#? 
+gen()()

test/completion/sys_path.py

@@ -12,11 +12,11 @@ sys.path.append(sys.path[1] + '/thirdparty')
 # syntax err
 sys.path.append('a' +* '/thirdparty')
 
-#? ['evaluate', 'evaluate_representation']
+#? ['evaluate']
 import evaluate
 
-#? ['goto']
-evaluate.goto
+#? ['Evaluator']
+evaluate.Evaluator
 
 #? ['jedi_']
 import jedi_

test/completion/usages.py

@@ -136,9 +136,9 @@ class NestedClass():
     def __getattr__(self, name):
         return self
 
-# Shouldn't find a definition, because there's no name defined (used ``getattr``).
-
-#< (0, 14),
+# Shouldn't find a definition, because there's other `instance`.
+# TODO reenable that test
+##< (0, 14),
 NestedClass().instance
 
 

test/refactor.py

@@ -7,7 +7,7 @@ from __future__ import with_statement
 import os
 import re
 
-from jedi._compatibility import reduce
+from functools import reduce
 import jedi
 from jedi import refactoring
 

test/run.py

@@ -101,7 +101,8 @@ import re
 from ast import literal_eval
 
 import jedi
-from jedi._compatibility import unicode, reduce, StringIO, is_py3k
+from functools import reduce
+from jedi._compatibility import unicode, StringIO, is_py3
 
 
 TEST_COMPLETIONS = 0
@@ -216,7 +217,7 @@ def collect_file_tests(lines, lines_to_execute):
     test_type = None
     for line_nr, line in enumerate(lines):
         line_nr += 1  # py2.5 doesn't know about the additional enumerate param
-        if not is_py3k:
+        if not is_py3:
             line = unicode(line, 'UTF-8')
         if correct:
             r = re.match('^(\d+)\s*(.*)$', correct)
@@ -276,7 +277,6 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
                 yield case
 
 
-
 docoptstr = """
 Using run.py to make debugging easier with integration tests.
 
@@ -339,7 +339,7 @@ if __name__ == '__main__':
             return 0
         else:
             print("\ttest fail @%d, actual = %s, desired = %s"
-                    % (case.line_nr - 1, actual, desired))
+                  % (case.line_nr - 1, actual, desired))
             return 1
 
     import traceback

test/test_api.py

@@ -2,27 +2,25 @@
 Test all things related to the ``jedi.api`` module.
 """
 
-from jedi import common, api
+from jedi import api
 from pytest import raises
 
 
 def test_preload_modules():
     def check_loaded(*modules):
-        # + 1 for builtin, +1 for None module (currently used)
-        assert len(new) == len(modules) + 2
-        for i in modules + ('__builtin__',):
-            assert [i in k for k in new.keys() if k is not None]
+        # +1 for None module (currently used)
+        assert len(parser_cache) == len(modules) + 1
+        for i in modules:
+            assert [i in k for k in parser_cache.keys() if k is not None]
 
     from jedi import cache
     temp_cache, cache.parser_cache = cache.parser_cache, {}
-    new = cache.parser_cache
-    with common.ignored(KeyError): # performance of tests -> no reload
-        new['__builtin__'] = temp_cache['__builtin__']
+    parser_cache = cache.parser_cache
 
-    api.preload_module('datetime')
-    check_loaded('datetime')
+    api.preload_module('sys')
+    check_loaded()  # compiled (c_builtin) modules shouldn't be in the cache.
     api.preload_module('json', 'token')
-    check_loaded('datetime', 'json', 'token')
+    check_loaded('json', 'token')
 
     cache.parser_cache = temp_cache
 
@@ -30,6 +28,7 @@ def test_preload_modules():
 def test_empty_script():
     assert api.Script('')
 
+
 def test_line_number_errors():
     """
     Script should raise a ValueError if line/column numbers are not in a

test/test_api_classes.py

@@ -11,9 +11,10 @@ import jedi
 
 def test_is_keyword():
     results = Script('import ', 1, 1, None).goto_definitions()
-    assert len(results) == 1 and results[0].is_keyword == True
+    assert len(results) == 1 and results[0].is_keyword is True
     results = Script('str', 1, 1, None).goto_definitions()
-    assert len(results) == 1 and results[0].is_keyword == False
+    assert len(results) == 1 and results[0].is_keyword is False
+
 
 def make_definitions():
     """
@@ -71,6 +72,7 @@ def test_function_call_signature_in_doc():
     doc = defs[0].doc
     assert "f(x, y = 1, z = 'a')" in doc
 
+
 def test_class_call_signature():
     defs = Script("""
     class Foo:

test/test_builtin.py

@@ -1,13 +0,0 @@
-from jedi import builtin
-
-
-def test_parse_function_doc_illegal_docstr():
-
-    def test_func(a):
-        """
-        test_func(o
-
-        doesn't have a closing bracket.
-        """
-
-    assert ('', '') == builtin._parse_function_doc(test_func)

test/test_cache.py

@@ -8,15 +8,16 @@ import pytest
 
 import jedi
 from jedi import settings, cache
-from jedi.cache import ParserCacheItem, _ModulePickling
+from jedi.cache import ParserCacheItem, ParserPickling
 
 
-ModulePickling = _ModulePickling()
+ParserPicklingCls = type(ParserPickling)
+ParserPickling = ParserPicklingCls()
 
 
 def test_modulepickling_change_cache_dir(monkeypatch, tmpdir):
     """
-    ModulePickling should not save old cache when cache_directory is changed.
+    ParserPickling should not save old cache when cache_directory is changed.
 
     See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_
     """
@@ -29,19 +30,19 @@ def test_modulepickling_change_cache_dir(monkeypatch, tmpdir):
     path_2 = 'fake path 2'
 
     monkeypatch.setattr(settings, 'cache_directory', dir_1)
-    ModulePickling.save_module(path_1, item_1)
-    cached = load_stored_item(ModulePickling, path_1, item_1)
+    ParserPickling.save_parser(path_1, item_1)
+    cached = load_stored_item(ParserPickling, path_1, item_1)
     assert cached == item_1.parser
 
     monkeypatch.setattr(settings, 'cache_directory', dir_2)
-    ModulePickling.save_module(path_2, item_2)
-    cached = load_stored_item(ModulePickling, path_1, item_1)
+    ParserPickling.save_parser(path_2, item_2)
+    cached = load_stored_item(ParserPickling, path_1, item_1)
     assert cached is None
 
 
 def load_stored_item(cache, path, item):
     """Load `item` stored at `path` in `cache`."""
-    return cache.load_module(path, item.change_time - 1)
+    return cache.load_parser(path, item.change_time - 1)
 
 
 @pytest.mark.usefixtures("isolated_jedi_cache")
@@ -49,13 +50,13 @@ def test_modulepickling_delete_incompatible_cache():
     item = ParserCacheItem('fake parser')
     path = 'fake path'
 
-    cache1 = _ModulePickling()
+    cache1 = ParserPicklingCls()
     cache1.version = 1
-    cache1.save_module(path, item)
+    cache1.save_parser(path, item)
     cached1 = load_stored_item(cache1, path, item)
     assert cached1 == item.parser
 
-    cache2 = _ModulePickling()
+    cache2 = ParserPicklingCls()
     cache2.version = 2
     cached2 = load_stored_item(cache2, path, item)
     assert cached2 is None
@@ -66,7 +67,7 @@ def test_star_import_cache_duration():
     old, jedi.settings.star_import_cache_validity = \
             jedi.settings.star_import_cache_validity, new
 
-    cache.star_import_cache = {}  # first empty...
+    cache._star_import_cache = {}  # first empty...
     # path needs to be not-None (otherwise caching effects are not visible)
     jedi.Script('', 1, 0, '').completions()
     time.sleep(2 * new)
@@ -74,6 +75,6 @@ def test_star_import_cache_duration():
 
     # reset values
     jedi.settings.star_import_cache_validity = old
-    length = len(cache.star_import_cache)
-    cache.star_import_cache = {}
+    length = len(cache._star_import_cache)
+    cache._star_import_cache = {}
     assert length == 1

test/test_call_signatures.py

@@ -15,9 +15,11 @@ class TestCallSignatures(TestCase):
             assert signatures[0].call_name == expected_name
             assert signatures[0].index == expected_index
 
-    def test_call_signatures(self):
-        def run(source, name, index=0, column=None, line=1):
-            self._run(source, name, index, line, column)
+    def _run_simple(self, source, name, index=0, column=None, line=1):
+        self._run(source, name, index, line, column)
+
+    def test_simple(self):
+        run = self._run_simple
 
         # simple
         s1 = "abs(a, str("
@@ -60,19 +62,20 @@ class TestCallSignatures(TestCase):
 
         run("import time; abc = time; abc.sleep(", 'sleep', 0)
 
-        # jedi-vim #9
-        run("with open(", 'open', 0)
-
-        # jedi-vim #11
-        run("for sorted(", 'sorted', 0)
-        run("for s in sorted(", 'sorted', 0)
-
         # jedi #57
         s = "def func(alpha, beta): pass\n" \
             "func(alpha='101',"
         run(s, 'func', 0, column=13, line=2)
 
-    def test_function_definition_complex(self):
+    def test_flows(self):
+        # jedi-vim #9
+        self._run_simple("with open(", 'open', 0)
+
+        # jedi-vim #11
+        self._run_simple("for sorted(", 'sorted', 0)
+        self._run_simple("for s in sorted(", 'sorted', 0)
+
+    def test_complex(self):
         s = """
                 def abc(a,b):
                     pass
@@ -106,7 +109,7 @@ class TestCallSignatures(TestCase):
         # just don't throw an exception (if numpy doesn't exist, just ignore it)
         assert Script(s).call_signatures() == []
 
-    def test_function_definition_empty_paren_pre_space(self):
+    def test_call_signatures_empty_parentheses_pre_space(self):
         s = textwrap.dedent("""\
         def f(a, b):
             pass

test/test_compiled.py

@@ -0,0 +1,40 @@
+from jedi._compatibility import builtins
+from jedi.parser.representation import Function
+from jedi.evaluate import compiled
+from jedi.evaluate import Evaluator
+
+
+def test_simple():
+    e = Evaluator()
+    bltn = compiled.CompiledObject(builtins)
+    obj = compiled.CompiledObject('_str_', bltn)
+    upper = e.find_types(obj, 'upper')
+    assert len(upper) == 1
+    objs = list(e.execute(upper[0]))
+    assert len(objs) == 1
+    assert objs[0].obj is str
+
+
+def test_fake_loading():
+    assert isinstance(compiled.create(next), Function)
+
+    string = compiled.builtin.get_subscope_by_name('str')
+    from_name = compiled._create_from_name(
+        compiled.builtin,
+        string,
+        '__init__'
+    )
+    assert isinstance(from_name, Function)
+
+
+def test_fake_docstr():
+    assert compiled.create(next).docstr.as_string() == next.__doc__
+
+
+def test_parse_function_doc_illegal_docstr():
+    docstr = """
+    test_func(o
+
+    doesn't have a closing bracket.
+    """
+    assert ('', '') == compiled._parse_function_doc(docstr)

test/test_full_name.py

@@ -16,7 +16,8 @@ There are three kinds of test:
 import textwrap
 
 import jedi
-from jedi import api_classes
+from jedi.api import classes
+from jedi.evaluate import Evaluator
 from .helpers import TestCase
 
 
@@ -81,6 +82,6 @@ def test_keyword_full_name_should_be_none():
     """issue #94"""
     # Using `from jedi.keywords import Keyword` here does NOT work
     # in Python 3.  This is due to the import hack jedi using.
-    Keyword = api_classes.keywords.Keyword
-    d = api_classes.Definition(Keyword('(', (0, 0)))
+    Keyword = classes.keywords.Keyword
+    d = classes.Definition(Evaluator(), Keyword('(', (0, 0)))
     assert d.full_name is None

test/test_integration_import.py

@@ -7,6 +7,7 @@ import itertools
 
 from jedi import Script
 from .helpers import cwd_at
+from jedi._compatibility import is_py26
 
 
 def test_goto_definition_on_import():
@@ -21,7 +22,8 @@ def test_complete_on_empty_import():
     assert 10 < len(Script("from . import", 1, 5, '').completions()) < 30
     assert 10 < len(Script("from . import classes", 1, 5, '').completions()) < 30
     assert len(Script("import").completions()) == 0
-    assert len(Script("import import", path='').completions()) > 0
+    if not is_py26:  # python 2.6 doesn't always come with a library `import*`.
+        assert len(Script("import import", path='').completions()) > 0
 
     # 111
     assert Script("from datetime import").completions()[0].name == 'import'

test/test_parsing.py

@@ -1,4 +1,5 @@
 from jedi.parser import Parser
+from jedi.parser.user_context import UserContextParser
 from jedi.parser import representation as pr
 
 
@@ -8,7 +9,7 @@ def test_user_statement_on_import():
         "    time)"
 
     for pos in [(2, 1), (2, 4)]:
-        u = Parser(s, user_position=pos).user_stmt
+        u = UserContextParser(s, None, pos, None).user_stmt()
         assert isinstance(u, pr.Import)
         assert u.defunct is False
         assert [str(n) for n in u.get_defined_names()] == ['time']
@@ -17,7 +18,7 @@ def test_user_statement_on_import():
 class TestCallAndName():
     def get_call(self, source):
         stmt = Parser(source, no_docstr=True).module.statements[0]
-        return stmt.get_commands()[0]
+        return stmt.expression_list()[0]
 
     def test_name_and_call_positions(self):
         call = self.get_call('name\nsomething_else')
@@ -37,15 +38,15 @@ class TestCallAndName():
 
     def test_literal_type(self):
         literal = self.get_call('1.0')
-        assert isinstance(literal, pr.Number)
+        assert isinstance(literal, pr.Literal)
         assert type(literal.value) == float
 
         literal = self.get_call('1')
-        assert isinstance(literal, pr.Number)
+        assert isinstance(literal, pr.Literal)
         assert type(literal.value) == int
 
         literal = self.get_call('"hello"')
-        assert isinstance(literal, pr.String)
+        assert isinstance(literal, pr.Literal)
         assert literal.value == 'hello'
 
 

test/test_regression.py

@@ -8,9 +8,11 @@ import textwrap
 
 from .helpers import TestCase, cwd_at
 
+import pytest
 import jedi
 from jedi import Script
 from jedi import api
+from jedi.evaluate import imports
 from jedi.parser import Parser
 
 #jedi.set_debug_function()
@@ -73,15 +75,16 @@ class TestRegression(TestCase):
         s = Script("", 1, 0).completions()
         assert len(s) > 0
 
+    @pytest.mark.skip('Skip for now, test case is not really supported.')
     @cwd_at('jedi')
     def test_add_dynamic_mods(self):
-        api.settings.additional_dynamic_modules = ['dynamic.py']
+        fname = '__main__.py'
+        api.settings.additional_dynamic_modules = [fname]
         # Fictional module that defines a function.
-        src1 = "def ret(a): return a"
+        src1 = "def r(a): return a"
         # Other fictional modules in another place in the fs.
-        src2 = 'from .. import setup; setup.ret(1)'
-        # .parser to load the module
-        api.modules.Module(os.path.abspath('dynamic.py'), src2).parser
+        src2 = 'from .. import setup; setup.r(1)'
+        imports.load_module(os.path.abspath(fname), src2)
         result = Script(src1, path='../setup.py').goto_definitions()
         assert len(result) == 1
         assert result[0].description == 'class int'
@@ -118,7 +121,7 @@ class TestRegression(TestCase):
         s = "x()\nx( )\nx(  )\nx (  )"
         parser = Parser(s)
         for i, s in enumerate(parser.module.statements, 3):
-            for c in s.get_commands():
+            for c in s.expression_list():
                 self.assertEqual(c.execution.end_pos[1], i)
 
     def check_definition_by_marker(self, source, after_cursor, names):

test/test_utils.py

@@ -5,7 +5,7 @@ from .helpers import TestCase, cwd_at
 
 
 class TestSetupReadline(TestCase):
-    class NameSpace():
+    class NameSpace(object):
         pass
 
     def __init__(self, *args, **kwargs):
@@ -98,3 +98,7 @@ class TestSetupReadline(TestCase):
             assert self.completions('colorama')
             assert self.completions('colorama.Fore.BLACK') == ['colorama.Fore.BLACK']
             del self.namespace.colorama
+
+
+def test_version_info():
+    assert utils.version_info()[:2] > (0, 7)