diff --git a/jedi/api/classes.py b/jedi/api/classes.py index fed9fa8c..5e1d386c 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -241,6 +241,8 @@ class BaseDefinition(object): See :attr:`doc` for example. """ + if isinstance(self._definition.docstr, pr.token_pr.TokenDocstring): + return unicode(self._definition.docstr.as_string()) try: return unicode(self._definition.docstr) except AttributeError: diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 2c4c8150..f4e44c9a 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -9,6 +9,7 @@ import inspect from jedi._compatibility import is_py3, builtins from jedi.parser import Parser +from jedi.parser import token as token_pr from jedi.parser.representation import Class from jedi.evaluate.helpers import FakeName @@ -102,7 +103,9 @@ def get_faked(module, obj, name=None): if not isinstance(result, Class) and result is not None: # Set the docstr which was previously not set (faked modules don't # contain it). - result.docstr = obj.__doc__ or '' + result.docstr = None + if obj.__doc__: + result.docstr = token_pr.TokenDocstring.fake_docstring(obj.__doc__) return result diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 7911c399..111ba531 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -36,7 +36,12 @@ REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`') def follow_param(evaluator, param): func = param.parent_function # print func, param, param.parent_function - param_str = _search_param_in_docstr(func.docstr, str(param.get_name())) + if not func.docstr: + return [] + param_str = _search_param_in_docstr( + func.docstr.as_string(), + str(param.get_name()) + ) position = (1, 0) if param_str is not None: @@ -112,7 +117,9 @@ def find_return_types(evaluator, func): if match: return match.group(1) - type_str = search_return_in_docstr(func.docstr) + if not func.docstr: + return [] + type_str = search_return_in_docstr(func.docstr.as_string()) if not type_str: return [] diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 9a2c2835..d5ae2bac 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -361,7 +361,9 @@ class Parser(object): and first_tok.token_type == tokenize.STRING: # Normal docstring check if self.freshscope and not self.no_docstr: - self._scope.add_docstr(first_tok.token) + self._scope.add_docstr( + token_pr.TokenDocstring(first_tok) + ) return None, tok # Attribute docstring (PEP 224) support (sphinx uses it, e.g.) @@ -369,7 +371,9 @@ class Parser(object): elif first_tok.token_type == tokenize.STRING: with common.ignored(IndexError, AttributeError): # ...then set it as a docstring - self._scope.statements[-1].add_docstr(first_tok.token) + self._scope.statements[-1].add_docstr( + token_pr.TokenDocstring(first_tok) + ) return None, tok stmt = stmt_class(self.module, tok_list, first_pos, self.end_pos, diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index 73ce535e..a561aa61 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -43,10 +43,21 @@ from jedi import common from jedi import debug from jedi import cache from jedi.parser import tokenize +from jedi.parser import token as token_pr + SCOPE_CONTENTS = ['asserts', 'subscopes', 'imports', 'statements', 'returns'] + +class GetCodeState(object): + """A helper class for passing the state of get_code in a thread-safe + manner""" + __slots__ = ("last_pos") + + def __init__(self): + self.last_pos = (0, 0) + class Base(object): """ This is just here to have an isinstance check, which is also used on @@ -61,6 +72,29 @@ class Base(object): def isinstance(self, *cls): return isinstance(self, cls) + @property + def newline(self): + """Returns the newline type for the current code.""" + #TODO: we need newline detection + return "\n" + + @property + def whitespace(self): + """Returns the whitespace type for the current code: tab or space.""" + #TODO: we need tab detection + return " " + + def space(self, from_pos, to_pos): + """Return the space between two tokens""" + linecount = to_pos[0] - from_pos[0] + if linecount == 0: + return self.whitespace * (to_pos[1] - from_pos[1]) + else: + return "%s%s" % ( + self.newline * linecount, + self.whitespace * to_pos[1], + ) + class Simple(Base): """ @@ -150,7 +184,7 @@ class Scope(Simple, IsScope): self.subscopes = [] self.imports = [] self.statements = [] - self.docstr = '' + self.docstr = None self.asserts = [] # Needed here for fast_parser, because the fast_parser splits and # returns will be in "normal" modules. @@ -176,9 +210,9 @@ class Scope(Simple, IsScope): self.statements.append(stmt) return stmt - def add_docstr(self, string): + def add_docstr(self, token): """ Clean up a docstring """ - self.docstr = cleandoc(literal_eval(string)) + self.docstr = token def add_import(self, imp): self.imports.append(imp) @@ -192,14 +226,18 @@ class Scope(Simple, IsScope): i += s.get_imports() return i + def get_code2(self, state=GetCodeState()): + string = [] + return "".join(string) + def get_code(self, first_indent=False, indention=' '): """ :return: Returns the code of the current scope. :rtype: str """ string = "" - if len(self.docstr) > 0: - string += '"""' + self.docstr + '"""\n' + if self.docstr: + string += '"""' + self.docstr.as_string() + '"""\n' objs = self.subscopes + self.imports + self.statements + self.returns for obj in sorted(objs, key=lambda x: x.start_pos): @@ -431,12 +469,15 @@ class Class(Scope): """ Return a document string including call signature of __init__. """ + docstr = "" + if self.docstr: + docstr = self.docstr.as_string() for sub in self.subscopes: if sub.name.names[-1] == '__init__': return '%s\n\n%s' % ( sub.get_call_signature(funcname=self.name.names[-1]), - self.docstr) - return self.docstr + docstr) + return docstr class Function(Scope): @@ -516,7 +557,13 @@ class Function(Scope): @property def doc(self): """ Return a document string including call signature. """ - return '%s\n\n%s' % (self.get_call_signature(), self.docstr) + docstr = "" + if self.docstr: + docstr = self.docstr.as_string() + return '%s\n\n%s' % ( + self.get_call_signature(), + docstr, + ) class Lambda(Function): @@ -764,7 +811,7 @@ class Statement(Simple): for n in as_names: n.parent = self.use_as_parent self.parent = parent - self.docstr = '' + self.docstr = None self._set_vars = None self.as_names = list(as_names) @@ -772,9 +819,9 @@ class Statement(Simple): self._assignment_details = [] # this is important for other scripts - def add_docstr(self, string): + def add_docstr(self, token): """ Clean up a docstring """ - self.docstr = cleandoc(literal_eval(string)) + self.docstr = token def get_code(self, new_line=True): def assemble(command_list, assignment=None): @@ -787,7 +834,7 @@ class Statement(Simple): code = ''.join(assemble(*a) for a in self.assignment_details) code += assemble(self.expression_list()) if self.docstr: - code += '\n"""%s"""' % self.docstr + code += '\n"""%s"""' % self.docstr.as_string() if new_line: return code + '\n' diff --git a/jedi/parser/token.py b/jedi/parser/token.py index 40e60eee..108dcabc 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -5,6 +5,8 @@ We want to have a token_list and start_position for everything the tokenizer returns. Therefore we need a memory efficient class. We found that a flat object with slots is the best. """ +from inspect import cleandoc +from ast import literal_eval from jedi._compatibility import utf8, unicode @@ -59,10 +61,16 @@ class Token(object): # Backward compatibility py2 def __unicode__(self): - return unicode(self.token) + return self.as_string() # Backward compatibility py3 def __str__(self): + return self.as_string() + + def as_string(self): + """For backward compatibilty str(token) or unicode(token) will work. + BUT please use as_string() instead, because it is independant from the + python version.""" return unicode(self.token) # Backward compatibility @@ -93,7 +101,6 @@ class Token(object): def start_pos_col(self): return self._start_pos_col - # Backward compatibility @property def start_pos(self): return (self.start_pos_line, self.start_pos_col) @@ -126,3 +133,35 @@ class Token(object): self._token = state[1] self._start_pos_line = state[2] self._start_pos_col = state[3] + + +class TokenNoCompat(Token): + def __unicode__(self): + raise NotImplementedError("Compatibility only for basic token.") + + def __str__(self): + raise NotImplementedError("Compatibility only for basic token.") + + def __getitem__(self, key): + raise NotImplementedError("Compatibility only for basic token.") + + +class TokenDocstring(TokenNoCompat): + """A string token that is a docstring. + + as_string() will clean the token representing the docstring. + """ + def __init__(self, token=None, state=None): + if token: + self.__setstate__(token.__getstate__()) + else: + self.__setstate__(state) + + @classmethod + def fake_docstring(cls, docstr): + # TODO: fixme when tests are up again + return TokenDocstring(state=(0, '"""\n%s\n"""' % docstr, 0, 0)) + + def as_string(self): + """Returns a literal cleaned version of the token""" + return cleandoc(literal_eval(self.token)) diff --git a/test/test_evaluate/test_compiled.py b/test/test_evaluate/test_compiled.py index 1a0fc6e4..ceff5e95 100644 --- a/test/test_evaluate/test_compiled.py +++ b/test/test_evaluate/test_compiled.py @@ -19,12 +19,16 @@ def test_fake_loading(): assert isinstance(compiled.create(next), Function) string = compiled.builtin.get_subscope_by_name('str') - from_name = compiled._create_from_name(compiled.builtin, string, '__init__') + from_name = compiled._create_from_name( + compiled.builtin, + string, + '__init__' + ) assert isinstance(from_name, Function) def test_fake_docstr(): - assert compiled.create(next).docstr == next.__doc__ + assert compiled.create(next).docstr.as_string() == next.__doc__ def test_parse_function_doc_illegal_docstr(): diff --git a/test/test_get_code.py b/test/test_get_code.py new file mode 100644 index 00000000..506f030a --- /dev/null +++ b/test/test_get_code.py @@ -0,0 +1,46 @@ +import jedi.parser as parser +import difflib + +code_basic_features = ''' +"""A mod docstring""" + +def a_function(a_argument, a_default = "default"): + """A func docstring""" + + a_result = 3 * a_argument + print(a_result) # a comment + b = """ +from +to""" + "huhu" + + + if a_default == "default": + return str(a_result) + else + return None +''' + + +def diff_code_assert(a, b, n=4): + if a != b: + diff = "\n".join(difflib.unified_diff( + a.splitlines(), + b.splitlines(), + n=n, + lineterm="" + )) + assert False, "Code does not match:\n%s\n\ncreated code:\n%s" % ( + diff, + b + ) + pass + + +def test_basic_parsing(): + """Validate the parsing features""" + + prs = parser.Parser(code_basic_features) +# diff_code_assert( +# code_basic_features, +# prs.top_module.get_code2() +# )