Merge branch 'get_code_fidelity' of git://github.com/ganwell/jedi into ganwell

jedi/api/classes.py

@@ -241,6 +241,8 @@ class BaseDefinition(object):
 
         See :attr:`doc` for example.
         """
+        if isinstance(self._definition.docstr, pr.token_pr.TokenDocstring):
+            return unicode(self._definition.docstr.as_string())
         try:
             return unicode(self._definition.docstr)
         except AttributeError:

jedi/evaluate/compiled/fake.py

@@ -9,6 +9,7 @@ import inspect
 
 from jedi._compatibility import is_py3, builtins
 from jedi.parser import Parser
+from jedi.parser import token as token_pr
 from jedi.parser.representation import Class
 from jedi.evaluate.helpers import FakeName
 
@@ -102,7 +103,9 @@ def get_faked(module, obj, name=None):
     if not isinstance(result, Class) and result is not None:
         # Set the docstr which was previously not set (faked modules don't
         # contain it).
-        result.docstr = obj.__doc__ or ''
+        result.docstr = None
+        if obj.__doc__:
+            result.docstr = token_pr.TokenDocstring.fake_docstring(obj.__doc__)
         return result
 
 

jedi/evaluate/docstrings.py

@@ -36,7 +36,12 @@ REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
 def follow_param(evaluator, param):
     func = param.parent_function
     # print func, param, param.parent_function
-    param_str = _search_param_in_docstr(func.docstr, str(param.get_name()))
+    if not func.docstr:
+        return []
+    param_str = _search_param_in_docstr(
+        func.docstr.as_string(),
+        str(param.get_name())
+    )
     position = (1, 0)
 
     if param_str is not None:
@@ -112,7 +117,9 @@ def find_return_types(evaluator, func):
             if match:
                 return match.group(1)
 
-    type_str = search_return_in_docstr(func.docstr)
+    if not func.docstr:
+        return []
+    type_str = search_return_in_docstr(func.docstr.as_string())
     if not type_str:
         return []
 

jedi/parser/__init__.py

@@ -361,7 +361,9 @@ class Parser(object):
                 and first_tok.token_type == tokenize.STRING:
             # Normal docstring check
             if self.freshscope and not self.no_docstr:
-                self._scope.add_docstr(first_tok.token)
+                self._scope.add_docstr(
+                    token_pr.TokenDocstring(first_tok)
+                )
                 return None, tok
 
             # Attribute docstring (PEP 224) support (sphinx uses it, e.g.)
@@ -369,7 +371,9 @@ class Parser(object):
             elif first_tok.token_type == tokenize.STRING:
                 with common.ignored(IndexError, AttributeError):
                     # ...then set it as a docstring
-                    self._scope.statements[-1].add_docstr(first_tok.token)
+                    self._scope.statements[-1].add_docstr(
+                        token_pr.TokenDocstring(first_tok)
+                    )
                     return None, tok
 
         stmt = stmt_class(self.module, tok_list, first_pos, self.end_pos,

jedi/parser/representation.py

@@ -43,10 +43,21 @@ from jedi import common
 from jedi import debug
 from jedi import cache
 from jedi.parser import tokenize
+from jedi.parser import token as token_pr
+
 
 SCOPE_CONTENTS = ['asserts', 'subscopes', 'imports', 'statements', 'returns']
 
 
+
+class GetCodeState(object):
+    """A helper class for passing the state of get_code in a thread-safe
+    manner"""
+    __slots__ = ("last_pos")
+
+    def __init__(self):
+        self.last_pos = (0, 0)
+
 class Base(object):
     """
     This is just here to have an isinstance check, which is also used on
@@ -61,6 +72,29 @@ class Base(object):
     def isinstance(self, *cls):
         return isinstance(self, cls)
 
+    @property
+    def newline(self):
+        """Returns the newline type for the current code."""
+        #TODO: we need newline detection
+        return "\n"
+
+    @property
+    def whitespace(self):
+        """Returns the whitespace type for the current code: tab or space."""
+        #TODO: we need tab detection
+        return " "
+
+    def space(self, from_pos, to_pos):
+        """Return the space between two tokens"""
+        linecount = to_pos[0] - from_pos[0]
+        if linecount == 0:
+            return self.whitespace * (to_pos[1] - from_pos[1])
+        else:
+            return "%s%s" % (
+                self.newline * linecount,
+                self.whitespace * to_pos[1],
+            )
+
 
 class Simple(Base):
     """
@@ -150,7 +184,7 @@ class Scope(Simple, IsScope):
         self.subscopes = []
         self.imports = []
         self.statements = []
-        self.docstr = ''
+        self.docstr = None
         self.asserts = []
         # Needed here for fast_parser, because the fast_parser splits and
         # returns will be in "normal" modules.
@@ -176,9 +210,9 @@ class Scope(Simple, IsScope):
         self.statements.append(stmt)
         return stmt
 
-    def add_docstr(self, string):
+    def add_docstr(self, token):
         """ Clean up a docstring """
-        self.docstr = cleandoc(literal_eval(string))
+        self.docstr = token
 
     def add_import(self, imp):
         self.imports.append(imp)
@@ -192,14 +226,18 @@ class Scope(Simple, IsScope):
                 i += s.get_imports()
         return i
 
+    def get_code2(self, state=GetCodeState()):
+        string = []
+        return "".join(string)
+
     def get_code(self, first_indent=False, indention='    '):
         """
         :return: Returns the code of the current scope.
         :rtype: str
         """
         string = ""
-        if len(self.docstr) > 0:
+        if self.docstr:
-            string += '"""' + self.docstr + '"""\n'
+            string += '"""' + self.docstr.as_string() + '"""\n'
 
         objs = self.subscopes + self.imports + self.statements + self.returns
         for obj in sorted(objs, key=lambda x: x.start_pos):
@@ -431,12 +469,15 @@ class Class(Scope):
         """
         Return a document string including call signature of __init__.
         """
+        docstr = ""
+        if self.docstr:
+            docstr = self.docstr.as_string()
         for sub in self.subscopes:
             if sub.name.names[-1] == '__init__':
                 return '%s\n\n%s' % (
                     sub.get_call_signature(funcname=self.name.names[-1]),
-                    self.docstr)
+                    docstr)
-        return self.docstr
+        return docstr
 
 
 class Function(Scope):
@@ -516,7 +557,13 @@ class Function(Scope):
     @property
     def doc(self):
         """ Return a document string including call signature. """
-        return '%s\n\n%s' % (self.get_call_signature(), self.docstr)
+        docstr = ""
+        if self.docstr:
+            docstr = self.docstr.as_string()
+        return '%s\n\n%s' % (
+            self.get_call_signature(),
+            docstr,
+        )
 
 
 class Lambda(Function):
@@ -764,7 +811,7 @@ class Statement(Simple):
             for n in as_names:
                 n.parent = self.use_as_parent
         self.parent = parent
-        self.docstr = ''
+        self.docstr = None
         self._set_vars = None
         self.as_names = list(as_names)
 
@@ -772,9 +819,9 @@ class Statement(Simple):
         self._assignment_details = []
         # this is important for other scripts
 
-    def add_docstr(self, string):
+    def add_docstr(self, token):
         """ Clean up a docstring """
-        self.docstr = cleandoc(literal_eval(string))
+        self.docstr = token
 
     def get_code(self, new_line=True):
         def assemble(command_list, assignment=None):
@@ -787,7 +834,7 @@ class Statement(Simple):
         code = ''.join(assemble(*a) for a in self.assignment_details)
         code += assemble(self.expression_list())
         if self.docstr:
-            code += '\n"""%s"""' % self.docstr
+            code += '\n"""%s"""' % self.docstr.as_string()
 
         if new_line:
             return code + '\n'

jedi/parser/token.py

@@ -5,6 +5,8 @@ We want to have a token_list and start_position for everything the
 tokenizer returns. Therefore we need a memory efficient class. We
 found that a flat object with slots is the best.
 """
+from inspect import cleandoc
+from ast import literal_eval
 
 from jedi._compatibility import utf8, unicode
 
@@ -59,10 +61,16 @@ class Token(object):
 
     # Backward compatibility py2
     def __unicode__(self):
-        return unicode(self.token)
+        return self.as_string()
 
     # Backward compatibility py3
     def __str__(self):
+        return self.as_string()
+
+    def as_string(self):
+        """For backward compatibilty str(token) or unicode(token) will work.
+        BUT please use as_string() instead, because it is independant from the
+        python version."""
         return unicode(self.token)
 
     # Backward compatibility
@@ -93,7 +101,6 @@ class Token(object):
     def start_pos_col(self):
         return self._start_pos_col
 
-    # Backward compatibility
     @property
     def start_pos(self):
         return (self.start_pos_line, self.start_pos_col)
@@ -126,3 +133,35 @@ class Token(object):
         self._token          = state[1]
         self._start_pos_line = state[2]
         self._start_pos_col  = state[3]
+
+
+class TokenNoCompat(Token):
+    def __unicode__(self):
+        raise NotImplementedError("Compatibility only for basic token.")
+
+    def __str__(self):
+        raise NotImplementedError("Compatibility only for basic token.")
+
+    def __getitem__(self, key):
+        raise NotImplementedError("Compatibility only for basic token.")
+
+
+class TokenDocstring(TokenNoCompat):
+    """A string token that is a docstring.
+
+    as_string() will clean the token representing the docstring.
+    """
+    def __init__(self, token=None, state=None):
+        if token:
+            self.__setstate__(token.__getstate__())
+        else:
+            self.__setstate__(state)
+
+    @classmethod
+    def fake_docstring(cls, docstr):
+        # TODO: fixme when tests are up again
+        return TokenDocstring(state=(0, '"""\n%s\n"""' % docstr, 0, 0))
+
+    def as_string(self):
+        """Returns a literal cleaned version of the token"""
+        return cleandoc(literal_eval(self.token))

test/test_evaluate/test_compiled.py

@@ -19,12 +19,16 @@ def test_fake_loading():
     assert isinstance(compiled.create(next), Function)
 
     string = compiled.builtin.get_subscope_by_name('str')
-    from_name = compiled._create_from_name(compiled.builtin, string, '__init__')
+    from_name = compiled._create_from_name(
+        compiled.builtin,
+        string,
+        '__init__'
+    )
     assert isinstance(from_name, Function)
 
 
 def test_fake_docstr():
-    assert compiled.create(next).docstr == next.__doc__
+    assert compiled.create(next).docstr.as_string() == next.__doc__
 
 
 def test_parse_function_doc_illegal_docstr():

test/test_get_code.py

@@ -0,0 +1,46 @@
+import jedi.parser as parser
+import difflib
+
+code_basic_features = '''
+"""A mod docstring"""
+
+def a_function(a_argument, a_default = "default"):
+    """A func docstring"""
+
+    a_result = 3 * a_argument
+    print(a_result)  # a comment
+    b = """
+from
+to""" + "huhu"
+
+
+    if a_default == "default":
+        return str(a_result)
+    else
+        return None
+'''
+
+
+def diff_code_assert(a, b, n=4):
+    if a != b:
+        diff = "\n".join(difflib.unified_diff(
+            a.splitlines(),
+            b.splitlines(),
+            n=n,
+            lineterm=""
+        ))
+        assert False, "Code does not match:\n%s\n\ncreated code:\n%s" % (
+            diff,
+            b
+        )
+    pass
+
+
+def test_basic_parsing():
+    """Validate the parsing features"""
+
+    prs = parser.Parser(code_basic_features)
+#    diff_code_assert(
+#        code_basic_features,
+#        prs.top_module.get_code2()
+#    )