Fix issues with Python 3.6's f strings and underscores in numbers.

jedi/parser/tokenize.py

@@ -59,21 +59,32 @@ Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Name = r'\w+'
 
-Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
-Binnumber = r'0[bB](?:_?[01])+'
-if is_py3:
+if py_version >= 36:
+    Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
+    Binnumber = r'0[bB](?:_?[01])+'
     Octnumber = r'0[oO](?:_?[0-7])+'
+    Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
+    Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+    Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
+    Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
+                       r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
+    Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
+    Floatnumber = group(Pointfloat, Expfloat)
+    Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
 else:
-    Octnumber = '0[0-7]+'
-
-Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
-Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
-                   r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
-Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
-Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
+    Hexnumber = r'0[xX][0-9a-fA-F]+'
+    Binnumber = r'0[bB][01]+'
+    if is_py3:
+        Octnumber = r'0[oO][0-7]+'
+    else:
+        Octnumber = '0[0-7]+'
+    Decnumber = r'(?:0+|[1-9][0-9]*)'
+    Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+    Exponent = r'[eE][-+]?[0-9]+'
+    Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+    Expfloat = r'[0-9]+' + Exponent
+    Floatnumber = group(Pointfloat, Expfloat)
+    Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
 # Return the empty string, plus all of the valid string prefixes.

jedi/parser/tree.py

@@ -41,10 +41,27 @@ import abc
 
 from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
                                  literal_eval, use_metaclass, unicode)
-from jedi.parser import token
 from jedi.parser.utils import underscore_memoization
 
 
+def _safe_literal_eval(value):
+    first_two = value[:2].lower()
+    if first_two[0] == 'f' or first_two in ('fr', 'rf'):
+        # literal_eval is not able to resovle f literals. We have to do that
+        # manually in a later stage
+        return ''
+
+    try:
+        return literal_eval(value)
+    except SyntaxError:
+        # It's possible to create syntax errors with literals like rb'' in
+        # Python 2. This should not be possible and in that case just return an
+        # empty string.
+        # Before Python 3.3 there was a more strict definition in which order
+        # you could define literals.
+        return ''
+
+
 def is_node(node, *symbol_names):
     try:
         type = node.type
@@ -98,7 +115,7 @@ class DocstringMixin(object):
             # leaves anymore that might be part of the docstring. A
             # docstring can also look like this: ``'foo' 'bar'
             # Returns a literal cleaned version of the ``Token``.
-            cleaned = cleandoc(literal_eval(node.value))
+            cleaned = cleandoc(_safe_literal_eval(node.value))
             # Since we want the docstr output to be always unicode, just
             # force it.
             if is_py3 or isinstance(cleaned, unicode):
@@ -422,7 +439,7 @@ class Literal(LeafWithNewLines):
     __slots__ = ()
 
     def eval(self):
-        return literal_eval(self.value)
+        return _safe_literal_eval(self.value)
 
 
 class Number(Literal):