remove token.py, docstrings are now just normal tokens until used

jedi/evaluate/compiled/fake.py

@@ -9,7 +9,7 @@ import inspect
 
 from jedi._compatibility import is_py3, builtins, unicode
 from jedi.parser import Parser
-from jedi.parser import token as token_pr
+from jedi.parser import tokenize
 from jedi.parser.representation import Class
 from jedi.evaluate.helpers import FakeName
 
@@ -105,7 +105,8 @@ def get_faked(module, obj, name=None):
         # contain it).
         result.docstr = None
         if obj.__doc__:
-            result.docstr = token_pr.TokenDocstring.fake_docstring(obj.__doc__)
+            doc = '''"""%s"""''' % obj.__doc__  # TODO need escapes.
+            result.docstr = tokenize.Token(tokenize.STRING, doc, (0, 0))
         return result
 
 

jedi/evaluate/docstrings.py

@@ -18,6 +18,7 @@ import re
 
 from jedi.evaluate.cache import memoize_default
 from jedi.parser import Parser
+from jedi.parser.representation import docstring_content
 
 DOCSTRING_PARAM_PATTERNS = [
     r'\s*:type\s+%s:\s*([^\n]+)',  # Sphinx
@@ -39,7 +40,8 @@ def follow_param(evaluator, param):
     if not func.docstr:
         return []
     param_str = _search_param_in_docstr(
-        func.docstr.as_string(),
+# TODO  this is ugly, no direct access?
+        docstring_content(func.docstr),
         str(param.get_name())
     )
     position = (1, 0)
@@ -119,7 +121,7 @@ def find_return_types(evaluator, func):
 
     if not func.docstr:
         return []
-    type_str = search_return_in_docstr(func.docstr.as_string())
+    type_str = search_return_in_docstr(docstring_content(func.docstr))
     if not type_str:
         return []