diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 89fa0b9a..0839014c 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -17,6 +17,7 @@ from jedi.evaluate import imports from jedi.evaluate import compiled from jedi.evaluate.filters import ParamName from jedi.api.keywords import KeywordName +from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature def _sort_names_by_start_pos(names): @@ -244,10 +245,7 @@ class BaseDefinition(object): the ``foo.docstring(fast=False)`` on every object, because it parses all libraries starting with ``a``. """ - if raw: - return _Help(self._name).raw(fast=fast) - else: - return _Help(self._name).full(fast=fast) + return _Help(self._name).docstring(fast=fast, raw=raw) @property def doc(self): @@ -717,24 +715,20 @@ class _Help(object): return None return self._name.tree_name.get_definition() - def full(self, fast=True): - node = self._get_node(fast) - try: - return node.doc - except AttributeError: - return self.raw(fast) - - def raw(self, fast=True): + def docstring(self, fast=True, raw=True): """ - The raw docstring ``__doc__`` for any object. + The docstring ``__doc__`` for any object. See :attr:`doc` for example. """ node = self._get_node(fast) - if node is None: - return '' try: - return node.raw_doc + node.get_doc_node except AttributeError: return '' + else: + if raw: + return clean_scope_docstring(node) + else: + return get_doc_with_call_signature(node) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index eca81803..0fdd5202 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -348,7 +348,8 @@ class Evaluator(object): search_global=True ) elif isinstance(atom, tree.Literal): - return set([compiled.create(self, atom.eval())]) + string = parser_utils.safe_literal_eval(atom.value) + return set([compiled.create(self, string)]) else: c = atom.children if c[0].type == 'string': diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index ccc88348..1b91ae57 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -26,6 +26,7 @@ from jedi.parser.python import parse from jedi.parser.python.tree import search_ancestor from jedi.common import indent_block from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence +from jedi.parser_utils import clean_scope_docstring DOCSTRING_PARAM_PATTERNS = [ @@ -196,11 +197,11 @@ def follow_param(module_context, param): if func.type == 'lambdef': return set() - types = eval_docstring(func.raw_doc) + types = eval_docstring(clean_scope_docstring(func)) if func.name.value == '__init__': cls = search_ancestor(func, 'classdef') if cls is not None: - types |= eval_docstring(cls.raw_doc) + types |= eval_docstring(clean_scope_docstring(cls)) return types @@ -213,5 +214,5 @@ def find_return_types(module_context, func): if match: return _strip_rst_role(match.group(1)) - type_str = search_return_in_docstr(func.raw_doc) + type_str = search_return_in_docstr(clean_scope_docstring(func)) return _evaluate_for_statement_string(module_context, type_str) diff --git a/jedi/parser/python/tree.py b/jedi/parser/python/tree.py index 1bf77e24..360ff002 100644 --- a/jedi/parser/python/tree.py +++ b/jedi/parser/python/tree.py @@ -27,32 +27,12 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. """ -from inspect import cleandoc from itertools import chain -import textwrap -from jedi._compatibility import is_py3, utf8_repr, literal_eval, unicode +from jedi._compatibility import utf8_repr, unicode from jedi.parser.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf -def _safe_literal_eval(value): - first_two = value[:2].lower() - if first_two[0] == 'f' or first_two in ('fr', 'rf'): - # literal_eval is not able to resovle f literals. We have to do that - # manually in a later stage - return '' - - try: - return literal_eval(value) - except SyntaxError: - # It's possible to create syntax errors with literals like rb'' in - # Python 2. This should not be possible and in that case just return an - # empty string. - # Before Python 3.3 there was a more strict definition in which order - # you could define literals. - return '' - - def search_ancestor(node, node_type_or_types): if not isinstance(node_type_or_types, (list, tuple)): node_type_or_types = (node_type_or_types,) @@ -66,9 +46,7 @@ def search_ancestor(node, node_type_or_types): class DocstringMixin(object): __slots__ = () - @property - def raw_doc(self): - """ Returns a cleaned version of the docstring token. """ + def get_doc_node(self): if self.type == 'file_input': node = self.children[0] elif isinstance(self, ClassOrFunc): @@ -80,25 +58,14 @@ class DocstringMixin(object): c = simple_stmt.parent.children index = c.index(simple_stmt) if not index: - return '' + return None node = c[index - 1] if node.type == 'simple_stmt': node = node.children[0] - if node.type == 'string': - # TODO We have to check next leaves until there are no new - # leaves anymore that might be part of the docstring. A - # docstring can also look like this: ``'foo' 'bar' - # Returns a literal cleaned version of the ``Token``. - cleaned = cleandoc(_safe_literal_eval(node.value)) - # Since we want the docstr output to be always unicode, just - # force it. - if is_py3 or isinstance(cleaned, unicode): - return cleaned - else: - return unicode(cleaned, 'UTF-8', 'replace') - return '' + return node + return None class PythonMixin(object): @@ -229,9 +196,6 @@ class Name(_LeafWithoutNewlines): class Literal(PythonLeaf): __slots__ = () - def eval(self): - return _safe_literal_eval(self.value) - class Number(Literal): type = 'number' @@ -447,18 +411,6 @@ class Class(ClassOrFunc): else: return self.children[3] - @property - def doc(self): - """ - Return a document string including call signature of __init__. - """ - docstr = self.raw_doc - for sub in self.subscopes: - if sub.name.value == '__init__': - return '%s\n\n%s' % ( - sub.get_call_signature(call_string=self.name.value), docstr) - return docstr - def _create_params(parent, argslist_list): """ @@ -556,35 +508,9 @@ class Function(ClassOrFunc): except IndexError: return None - def get_call_signature(self, width=72, call_string=None): - """ - Generate call signature of this function. - - :param width: Fold lines if a line is longer than this value. - :type width: int - :arg func_name: Override function name when given. - :type func_name: str - - :rtype: str - """ - # Lambdas have no name. - if call_string is None: - if self.type == 'lambdef': - call_string = '' - else: - call_string = self.name.value - code = call_string + self._get_paramlist_code() - return '\n'.join(textwrap.wrap(code, width)) - def _get_paramlist_code(self): return self.children[2].get_code() - @property - def doc(self): - """ Return a document string including call signature. """ - docstr = self.raw_doc - return '%s\n\n%s' % (self.get_call_signature(), docstr) - class Lambda(Function): """ diff --git a/jedi/parser_utils.py b/jedi/parser_utils.py index de1e6652..5ea6f751 100644 --- a/jedi/parser_utils.py +++ b/jedi/parser_utils.py @@ -1,3 +1,7 @@ +import textwrap +from inspect import cleandoc + +from jedi._compatibility import literal_eval, is_py3 from jedi.parser.python import tree _EXECUTE_NODES = set([ @@ -95,3 +99,79 @@ def get_statement_of_position(node, pos): except AttributeError: pass # Must be a non-scope return None + + +def clean_scope_docstring(scope_node): + """ Returns a cleaned version of the docstring token. """ + node = scope_node.get_doc_node() + if node is not None: + # TODO We have to check next leaves until there are no new + # leaves anymore that might be part of the docstring. A + # docstring can also look like this: ``'foo' 'bar' + # Returns a literal cleaned version of the ``Token``. + cleaned = cleandoc(safe_literal_eval(node.value)) + # Since we want the docstr output to be always unicode, just + # force it. + if is_py3 or isinstance(cleaned, unicode): + return cleaned + else: + return unicode(cleaned, 'UTF-8', 'replace') + return '' + + +def safe_literal_eval(value): + first_two = value[:2].lower() + if first_two[0] == 'f' or first_two in ('fr', 'rf'): + # literal_eval is not able to resovle f literals. We have to do that + # manually, but that's right now not implemented. + return '' + + try: + return literal_eval(value) + except SyntaxError: + # It's possible to create syntax errors with literals like rb'' in + # Python 2. This should not be possible and in that case just return an + # empty string. + # Before Python 3.3 there was a more strict definition in which order + # you could define literals. + return '' + + +def get_call_signature(funcdef, width=72, call_string=None): + """ + Generate call signature of this function. + + :param width: Fold lines if a line is longer than this value. + :type width: int + :arg func_name: Override function name when given. + :type func_name: str + + :rtype: str + """ + # Lambdas have no name. + if call_string is None: + if funcdef.type == 'lambdef': + call_string = '' + else: + call_string = funcdef.name.value + code = call_string + funcdef._get_paramlist_code() + return '\n'.join(textwrap.wrap(code, width)) + + +def get_doc_with_call_signature(scope_node): + """ + Return a document string including call signature. + """ + call_signature = None + if scope_node.type == 'classdef': + for sub in scope_node.subscopes: + if sub.name.value == '__init__': + call_signature = \ + get_call_signature(sub, call_string=scope_node.name.value) + elif scope_node.type in ('funcdef', 'lambdef'): + call_signature = get_call_signature(scope_node) + + doc = clean_scope_docstring(scope_node) + if call_signature is None: + return doc + return '%s\n\n%s' % (call_signature, doc) diff --git a/test/test_evaluate/test_compiled.py b/test/test_evaluate/test_compiled.py index 899f56af..78ab8e3c 100644 --- a/test/test_evaluate/test_compiled.py +++ b/test/test_evaluate/test_compiled.py @@ -5,6 +5,7 @@ from jedi.parser.python import load_grammar from jedi.evaluate import compiled, instance from jedi.evaluate.representation import FunctionContext from jedi.evaluate import Evaluator +from jedi.parser_utils import clean_scope_docstring from jedi import Script @@ -33,7 +34,8 @@ def test_fake_loading(): def test_fake_docstr(): - assert compiled.create(_evaluator(), next).tree_node.raw_doc == next.__doc__ + node = compiled.create(_evaluator(), next).tree_node + assert clean_scope_docstring(node) == next.__doc__ def test_parse_function_doc_illegal_docstr(): diff --git a/test/test_parser/test_parser.py b/test/test_parser/test_parser.py index 61b11f54..b7ce75ef 100644 --- a/test/test_parser/test_parser.py +++ b/test/test_parser/test_parser.py @@ -9,7 +9,8 @@ from jedi._compatibility import u, is_py3 from jedi.parser.python import parse, load_grammar from jedi.parser.python import tree from jedi.common import splitlines -from jedi.parser_utils import get_statement_of_position +from jedi.parser_utils import get_statement_of_position, \ + clean_scope_docstring, safe_literal_eval def test_user_statement_on_import(): @@ -38,7 +39,7 @@ class TestCallAndName(): leaf = self.get_call('1.0\n') assert leaf.value == '1.0' - assert leaf.eval() == 1.0 + assert safe_literal_eval(leaf.value) == 1.0 assert leaf.start_pos == (1, 0) assert leaf.end_pos == (1, 3) @@ -49,15 +50,15 @@ class TestCallAndName(): def test_literal_type(self): literal = self.get_call('1.0') assert isinstance(literal, tree.Literal) - assert type(literal.eval()) == float + assert type(safe_literal_eval(literal.value)) == float literal = self.get_call('1') assert isinstance(literal, tree.Literal) - assert type(literal.eval()) == int + assert type(safe_literal_eval(literal.value)) == int literal = self.get_call('"hello"') assert isinstance(literal, tree.Literal) - assert literal.eval() == 'hello' + assert safe_literal_eval(literal.value) == 'hello' class TestSubscopes(): @@ -131,7 +132,7 @@ def test_hex_values_in_docstring(): return 1 ''' - doc = parse(source).subscopes[0].raw_doc + doc = clean_scope_docstring(parse(source).subscopes[0]) if is_py3: assert doc == '\xff' else: diff --git a/test/test_parser/test_parser_tree.py b/test/test_parser/test_parser_tree.py index 1e5ecdca..6ea5fdcd 100644 --- a/test/test_parser/test_parser_tree.py +++ b/test/test_parser/test_parser_tree.py @@ -6,6 +6,7 @@ import pytest from jedi.parser.python import parse from jedi.parser.python import tree +from jedi.parser_utils import get_doc_with_call_signature, get_call_signature class TestsFunctionAndLambdaParsing(object): @@ -63,7 +64,7 @@ class TestsFunctionAndLambdaParsing(object): assert node.annotation.value == expected_annotation def test_get_call_signature(self, node, expected): - assert node.get_call_signature() == expected['call_sig'] + assert get_call_signature(node) == expected['call_sig'] def test_doc(self, node, expected): - assert node.doc == expected.get('doc') or (expected['call_sig'] + '\n\n') + assert get_doc_with_call_signature(node) == (expected['call_sig'] + '\n\n') diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 3db51dd0..62617d85 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -8,6 +8,7 @@ from jedi.parser import tokenize from jedi.parser.python import parse from jedi.common import splitlines from jedi.parser.tokenize import TokenInfo +from jedi.parser_utils import safe_literal_eval from ..helpers import unittest @@ -141,7 +142,7 @@ class TokenTest(unittest.TestCase): string_tok = expr_stmt.children[2] assert string_tok.type == 'string' assert string_tok.value == s - assert string_tok.eval() == 'test' + assert safe_literal_eval(string_tok.value) == 'test' def test_tokenizer_with_string_literal_backslash():