Progress in removing the docstring/call signature logic from the parser.

jedi/api/classes.py

@@ -17,6 +17,7 @@ from jedi.evaluate import imports
 from jedi.evaluate import compiled
 from jedi.evaluate.filters import ParamName
 from jedi.api.keywords import KeywordName
+from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature
 
 
 def _sort_names_by_start_pos(names):
@@ -244,10 +245,7 @@ class BaseDefinition(object):
             the ``foo.docstring(fast=False)`` on every object, because it
             parses all libraries starting with ``a``.
         """
-        if raw:
-            return _Help(self._name).raw(fast=fast)
-        else:
-            return _Help(self._name).full(fast=fast)
+        return _Help(self._name).docstring(fast=fast, raw=raw)
 
     @property
     def doc(self):
@@ -717,24 +715,20 @@ class _Help(object):
             return None
         return self._name.tree_name.get_definition()
 
-    def full(self, fast=True):
-        node = self._get_node(fast)
-        try:
-            return node.doc
-        except AttributeError:
-            return self.raw(fast)
-
-    def raw(self, fast=True):
+    def docstring(self, fast=True, raw=True):
         """
-        The raw docstring ``__doc__`` for any object.
+        The docstring ``__doc__`` for any object.
 
         See :attr:`doc` for example.
         """
         node = self._get_node(fast)
-        if node is None:
-            return ''
 
         try:
-            return node.raw_doc
+            node.get_doc_node
         except AttributeError:
             return ''
+        else:
+            if raw:
+                return clean_scope_docstring(node)
+            else:
+                return get_doc_with_call_signature(node)

jedi/evaluate/__init__.py

@@ -348,7 +348,8 @@ class Evaluator(object):
                 search_global=True
             )
         elif isinstance(atom, tree.Literal):
-            return set([compiled.create(self, atom.eval())])
+            string = parser_utils.safe_literal_eval(atom.value)
+            return set([compiled.create(self, string)])
         else:
             c = atom.children
             if c[0].type == 'string':

jedi/evaluate/docstrings.py

@@ -26,6 +26,7 @@ from jedi.parser.python import parse
 from jedi.parser.python.tree import search_ancestor
 from jedi.common import indent_block
 from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence
+from jedi.parser_utils import clean_scope_docstring
 
 
 DOCSTRING_PARAM_PATTERNS = [
@@ -196,11 +197,11 @@ def follow_param(module_context, param):
     if func.type == 'lambdef':
         return set()
 
-    types = eval_docstring(func.raw_doc)
+    types = eval_docstring(clean_scope_docstring(func))
     if func.name.value == '__init__':
         cls = search_ancestor(func, 'classdef')
         if cls is not None:
-            types |= eval_docstring(cls.raw_doc)
+            types |= eval_docstring(clean_scope_docstring(cls))
 
     return types
 
@@ -213,5 +214,5 @@ def find_return_types(module_context, func):
             if match:
                 return _strip_rst_role(match.group(1))
 
-    type_str = search_return_in_docstr(func.raw_doc)
+    type_str = search_return_in_docstr(clean_scope_docstring(func))
     return _evaluate_for_statement_string(module_context, type_str)

jedi/parser/python/tree.py

@@ -27,32 +27,12 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute
 See also :attr:`Scope.subscopes` and :attr:`Scope.statements`.
 """
 
-from inspect import cleandoc
 from itertools import chain
-import textwrap
 
-from jedi._compatibility import is_py3, utf8_repr, literal_eval, unicode
+from jedi._compatibility import utf8_repr, unicode
 from jedi.parser.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf
 
 
-def _safe_literal_eval(value):
-    first_two = value[:2].lower()
-    if first_two[0] == 'f' or first_two in ('fr', 'rf'):
-        # literal_eval is not able to resovle f literals. We have to do that
-        # manually in a later stage
-        return ''
-
-    try:
-        return literal_eval(value)
-    except SyntaxError:
-        # It's possible to create syntax errors with literals like rb'' in
-        # Python 2. This should not be possible and in that case just return an
-        # empty string.
-        # Before Python 3.3 there was a more strict definition in which order
-        # you could define literals.
-        return ''
-
-
 def search_ancestor(node, node_type_or_types):
     if not isinstance(node_type_or_types, (list, tuple)):
         node_type_or_types = (node_type_or_types,)
@@ -66,9 +46,7 @@ def search_ancestor(node, node_type_or_types):
 class DocstringMixin(object):
     __slots__ = ()
 
-    @property
-    def raw_doc(self):
-        """ Returns a cleaned version of the docstring token. """
+    def get_doc_node(self):
         if self.type == 'file_input':
             node = self.children[0]
         elif isinstance(self, ClassOrFunc):
@@ -80,25 +58,14 @@ class DocstringMixin(object):
             c = simple_stmt.parent.children
             index = c.index(simple_stmt)
             if not index:
-                return ''
+                return None
             node = c[index - 1]
 
         if node.type == 'simple_stmt':
             node = node.children[0]
-
         if node.type == 'string':
-            # TODO We have to check next leaves until there are no new
-            # leaves anymore that might be part of the docstring. A
-            # docstring can also look like this: ``'foo' 'bar'
-            # Returns a literal cleaned version of the ``Token``.
-            cleaned = cleandoc(_safe_literal_eval(node.value))
-            # Since we want the docstr output to be always unicode, just
-            # force it.
-            if is_py3 or isinstance(cleaned, unicode):
-                return cleaned
-            else:
-                return unicode(cleaned, 'UTF-8', 'replace')
-        return ''
+            return node
+        return None
 
 
 class PythonMixin(object):
@@ -229,9 +196,6 @@ class Name(_LeafWithoutNewlines):
 class Literal(PythonLeaf):
     __slots__ = ()
 
-    def eval(self):
-        return _safe_literal_eval(self.value)
-
 
 class Number(Literal):
     type = 'number'
@@ -447,18 +411,6 @@ class Class(ClassOrFunc):
             else:
                 return self.children[3]
 
-    @property
-    def doc(self):
-        """
-        Return a document string including call signature of __init__.
-        """
-        docstr = self.raw_doc
-        for sub in self.subscopes:
-            if sub.name.value == '__init__':
-                return '%s\n\n%s' % (
-                    sub.get_call_signature(call_string=self.name.value), docstr)
-        return docstr
-
 
 def _create_params(parent, argslist_list):
     """
@@ -556,35 +508,9 @@ class Function(ClassOrFunc):
         except IndexError:
             return None
 
-    def get_call_signature(self, width=72, call_string=None):
-        """
-        Generate call signature of this function.
-
-        :param width: Fold lines if a line is longer than this value.
-        :type width: int
-        :arg func_name: Override function name when given.
-        :type func_name: str
-
-        :rtype: str
-        """
-        # Lambdas have no name.
-        if call_string is None:
-            if self.type == 'lambdef':
-                call_string = '<lambda>'
-            else:
-                call_string = self.name.value
-        code = call_string + self._get_paramlist_code()
-        return '\n'.join(textwrap.wrap(code, width))
-
     def _get_paramlist_code(self):
         return self.children[2].get_code()
 
-    @property
-    def doc(self):
-        """ Return a document string including call signature. """
-        docstr = self.raw_doc
-        return '%s\n\n%s' % (self.get_call_signature(), docstr)
-
 
 class Lambda(Function):
     """

jedi/parser_utils.py

@@ -1,3 +1,7 @@
+import textwrap
+from inspect import cleandoc
+
+from jedi._compatibility import literal_eval, is_py3
 from jedi.parser.python import tree
 
 _EXECUTE_NODES = set([
@@ -95,3 +99,79 @@ def get_statement_of_position(node, pos):
                 except AttributeError:
                     pass  # Must be a non-scope
     return None
+
+
+def clean_scope_docstring(scope_node):
+    """ Returns a cleaned version of the docstring token. """
+    node = scope_node.get_doc_node()
+    if node is not None:
+        # TODO We have to check next leaves until there are no new
+        # leaves anymore that might be part of the docstring. A
+        # docstring can also look like this: ``'foo' 'bar'
+        # Returns a literal cleaned version of the ``Token``.
+        cleaned = cleandoc(safe_literal_eval(node.value))
+        # Since we want the docstr output to be always unicode, just
+        # force it.
+        if is_py3 or isinstance(cleaned, unicode):
+            return cleaned
+        else:
+            return unicode(cleaned, 'UTF-8', 'replace')
+    return ''
+
+
+def safe_literal_eval(value):
+    first_two = value[:2].lower()
+    if first_two[0] == 'f' or first_two in ('fr', 'rf'):
+        # literal_eval is not able to resovle f literals. We have to do that
+        # manually, but that's right now not implemented.
+        return ''
+
+    try:
+        return literal_eval(value)
+    except SyntaxError:
+        # It's possible to create syntax errors with literals like rb'' in
+        # Python 2. This should not be possible and in that case just return an
+        # empty string.
+        # Before Python 3.3 there was a more strict definition in which order
+        # you could define literals.
+        return ''
+
+
+def get_call_signature(funcdef, width=72, call_string=None):
+    """
+    Generate call signature of this function.
+
+    :param width: Fold lines if a line is longer than this value.
+    :type width: int
+    :arg func_name: Override function name when given.
+    :type func_name: str
+
+    :rtype: str
+    """
+    # Lambdas have no name.
+    if call_string is None:
+        if funcdef.type == 'lambdef':
+            call_string = '<lambda>'
+        else:
+            call_string = funcdef.name.value
+    code = call_string + funcdef._get_paramlist_code()
+    return '\n'.join(textwrap.wrap(code, width))
+
+
+def get_doc_with_call_signature(scope_node):
+    """
+    Return a document string including call signature.
+    """
+    call_signature = None
+    if scope_node.type == 'classdef':
+        for sub in scope_node.subscopes:
+            if sub.name.value == '__init__':
+                call_signature = \
+                    get_call_signature(sub, call_string=scope_node.name.value)
+    elif scope_node.type in ('funcdef', 'lambdef'):
+        call_signature = get_call_signature(scope_node)
+
+    doc = clean_scope_docstring(scope_node)
+    if call_signature is None:
+        return doc
+    return '%s\n\n%s' % (call_signature, doc)

test/test_evaluate/test_compiled.py

@@ -5,6 +5,7 @@ from jedi.parser.python import load_grammar
 from jedi.evaluate import compiled, instance
 from jedi.evaluate.representation import FunctionContext
 from jedi.evaluate import Evaluator
+from jedi.parser_utils import clean_scope_docstring
 from jedi import Script
 
 
@@ -33,7 +34,8 @@ def test_fake_loading():
 
 
 def test_fake_docstr():
-    assert compiled.create(_evaluator(), next).tree_node.raw_doc == next.__doc__
+    node = compiled.create(_evaluator(), next).tree_node
+    assert clean_scope_docstring(node) == next.__doc__
 
 
 def test_parse_function_doc_illegal_docstr():

test/test_parser/test_parser.py

@@ -9,7 +9,8 @@ from jedi._compatibility import u, is_py3
 from jedi.parser.python import parse, load_grammar
 from jedi.parser.python import tree
 from jedi.common import splitlines
-from jedi.parser_utils import get_statement_of_position
+from jedi.parser_utils import get_statement_of_position, \
+    clean_scope_docstring, safe_literal_eval
 
 
 def test_user_statement_on_import():
@@ -38,7 +39,7 @@ class TestCallAndName():
 
         leaf = self.get_call('1.0\n')
         assert leaf.value == '1.0'
-        assert leaf.eval() == 1.0
+        assert safe_literal_eval(leaf.value) == 1.0
         assert leaf.start_pos == (1, 0)
         assert leaf.end_pos == (1, 3)
 
@@ -49,15 +50,15 @@ class TestCallAndName():
     def test_literal_type(self):
         literal = self.get_call('1.0')
         assert isinstance(literal, tree.Literal)
-        assert type(literal.eval()) == float
+        assert type(safe_literal_eval(literal.value)) == float
 
         literal = self.get_call('1')
         assert isinstance(literal, tree.Literal)
-        assert type(literal.eval()) == int
+        assert type(safe_literal_eval(literal.value)) == int
 
         literal = self.get_call('"hello"')
         assert isinstance(literal, tree.Literal)
-        assert literal.eval() == 'hello'
+        assert safe_literal_eval(literal.value) == 'hello'
 
 
 class TestSubscopes():
@@ -131,7 +132,7 @@ def test_hex_values_in_docstring():
             return 1
         '''
 
-    doc = parse(source).subscopes[0].raw_doc
+    doc = clean_scope_docstring(parse(source).subscopes[0])
     if is_py3:
         assert doc == '\xff'
     else:

test/test_parser/test_parser_tree.py

@@ -6,6 +6,7 @@ import pytest
 
 from jedi.parser.python import parse
 from jedi.parser.python import tree
+from jedi.parser_utils import get_doc_with_call_signature, get_call_signature
 
 
 class TestsFunctionAndLambdaParsing(object):
@@ -63,7 +64,7 @@ class TestsFunctionAndLambdaParsing(object):
             assert node.annotation.value == expected_annotation
 
     def test_get_call_signature(self, node, expected):
-        assert node.get_call_signature() == expected['call_sig']
+        assert get_call_signature(node) == expected['call_sig']
 
     def test_doc(self, node, expected):
-        assert node.doc == expected.get('doc') or (expected['call_sig'] + '\n\n')
+        assert get_doc_with_call_signature(node) == (expected['call_sig'] + '\n\n')

test/test_parser/test_tokenize.py

@@ -8,6 +8,7 @@ from jedi.parser import tokenize
 from jedi.parser.python import parse
 from jedi.common import splitlines
 from jedi.parser.tokenize import TokenInfo
+from jedi.parser_utils import safe_literal_eval
 
 
 from ..helpers import unittest
@@ -141,7 +142,7 @@ class TokenTest(unittest.TestCase):
             string_tok = expr_stmt.children[2]
             assert string_tok.type == 'string'
             assert string_tok.value == s
-            assert string_tok.eval() == 'test'
+            assert safe_literal_eval(string_tok.value) == 'test'
 
 
 def test_tokenizer_with_string_literal_backslash():