Fix issues with Python 3.6's f strings and underscores in numbers.

jedi/api/helpers.py

@@ -7,6 +7,7 @@ from collections import namedtuple
 from jedi._compatibility import u
 from jedi.evaluate.helpers import evaluate_call_of_leaf
 from jedi import parser
+from jedi.parser import tree
 from jedi.parser import tokenize
 from jedi.cache import time_cache
 from jedi import common
@@ -203,6 +204,8 @@ def evaluate_goto_definition(evaluator, context, leaf):
         return context.eval_node(leaf.parent)
     elif parent.type == 'trailer':
         return evaluate_call_of_leaf(context, leaf)
+    elif isinstance(leaf, tree.Literal):
+        return context.evaluator.eval_atom(context, leaf)
     return []
 
 

jedi/evaluate/__init__.py

@@ -270,7 +270,7 @@ class Evaluator(object):
         debug.dbg('eval_element %s@%s', element, element.start_pos)
         types = set()
         if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
-            types = self._eval_atom(context, element)
+            types = self.eval_atom(context, element)
         elif isinstance(element, tree.Keyword):
             # For False/True/None
             if element.value in ('False', 'True', 'None'):
@@ -283,7 +283,7 @@ class Evaluator(object):
         elif element.type in ('power', 'atom_expr'):
             first_child = element.children[0]
             if not (first_child.type == 'keyword' and first_child.value == 'await'):
-                types = self._eval_atom(context, first_child)
+                types = self.eval_atom(context, first_child)
                 for trailer in element.children[1:]:
                     if trailer == '**':  # has a power operation.
                         right = self.eval_element(context, element.children[2])
@@ -306,7 +306,7 @@ class Evaluator(object):
             assert element.value == '...'
             types = set([compiled.create(self, Ellipsis)])
         elif element.type == 'dotted_name':
-            types = self._eval_atom(context, element.children[0])
+            types = self.eval_atom(context, element.children[0])
             for next_name in element.children[2::2]:
                 # TODO add search_global=True?
                 types = unite(
@@ -325,7 +325,7 @@ class Evaluator(object):
         debug.dbg('eval_element result %s', types)
         return types
 
-    def _eval_atom(self, context, atom):
+    def eval_atom(self, context, atom):
         """
         Basically to process ``atom`` nodes. The parser sometimes doesn't
         generate the node (because it has just one child). In that case an atom
@@ -351,9 +351,9 @@ class Evaluator(object):
             c = atom.children
             if c[0].type == 'string':
                 # Will be one string.
-                types = self._eval_atom(context, c[0])
+                types = self.eval_atom(context, c[0])
                 for string in c[1:]:
-                    right = self._eval_atom(context, string)
+                    right = self.eval_atom(context, string)
                     types = precedence.calculate(self, context, types, '+', right)
                 return types
             # Parentheses without commas are not tuples.

jedi/parser/tokenize.py

@@ -59,21 +59,32 @@ Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Name = r'\w+'
 
-Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
+if py_version >= 36:
-Binnumber = r'0[bB](?:_?[01])+'
+    Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
-if is_py3:
+    Binnumber = r'0[bB](?:_?[01])+'
     Octnumber = r'0[oO](?:_?[0-7])+'
+    Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
+    Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+    Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
+    Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
+                       r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
+    Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
+    Floatnumber = group(Pointfloat, Expfloat)
+    Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
 else:
-    Octnumber = '0[0-7]+'
+    Hexnumber = r'0[xX][0-9a-fA-F]+'
-
+    Binnumber = r'0[bB][01]+'
-Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
+    if is_py3:
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+        Octnumber = r'0[oO][0-7]+'
-Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
+    else:
-Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
+        Octnumber = '0[0-7]+'
-                   r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
+    Decnumber = r'(?:0+|[1-9][0-9]*)'
-Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
+    Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Floatnumber = group(Pointfloat, Expfloat)
+    Exponent = r'[eE][-+]?[0-9]+'
-Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
+    Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+    Expfloat = r'[0-9]+' + Exponent
+    Floatnumber = group(Pointfloat, Expfloat)
+    Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
 # Return the empty string, plus all of the valid string prefixes.

jedi/parser/tree.py

@@ -41,10 +41,27 @@ import abc
 
 from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
                                  literal_eval, use_metaclass, unicode)
-from jedi.parser import token
 from jedi.parser.utils import underscore_memoization
 
 
+def _safe_literal_eval(value):
+    first_two = value[:2].lower()
+    if first_two[0] == 'f' or first_two in ('fr', 'rf'):
+        # literal_eval is not able to resovle f literals. We have to do that
+        # manually in a later stage
+        return ''
+
+    try:
+        return literal_eval(value)
+    except SyntaxError:
+        # It's possible to create syntax errors with literals like rb'' in
+        # Python 2. This should not be possible and in that case just return an
+        # empty string.
+        # Before Python 3.3 there was a more strict definition in which order
+        # you could define literals.
+        return ''
+
+
 def is_node(node, *symbol_names):
     try:
         type = node.type
@@ -98,7 +115,7 @@ class DocstringMixin(object):
             # leaves anymore that might be part of the docstring. A
             # docstring can also look like this: ``'foo' 'bar'
             # Returns a literal cleaned version of the ``Token``.
-            cleaned = cleandoc(literal_eval(node.value))
+            cleaned = cleandoc(_safe_literal_eval(node.value))
             # Since we want the docstr output to be always unicode, just
             # force it.
             if is_py3 or isinstance(cleaned, unicode):
@@ -422,7 +439,7 @@ class Literal(LeafWithNewLines):
     __slots__ = ()
 
     def eval(self):
-        return literal_eval(self.value)
+        return _safe_literal_eval(self.value)
 
 
 class Number(Literal):

test/completion/basic.py

@@ -19,6 +19,17 @@ str..
 #? []
 a(0):.
 
+# -----------------
+# _ separators (for older versions than 3.6, a = 1_2_3 will just be 1, the rest
+#               gets ignored.)
+# -----------------
+#? int()
+1_2_3
+#? int()
+123_345_345
+#? int()
+0x3_4
+
 # -----------------
 # if/else/elif
 # -----------------

test/test_evaluate/test_literals.py

@@ -0,0 +1,37 @@
+import pytest
+
+import jedi
+from jedi._compatibility import py_version
+
+
+def _eval_literal(value):
+    def_, = jedi.Script(value).goto_definitions()
+    return def_._name._context
+
+
+@pytest.mark.skipif('sys.version_info[:2] < (3, 6)')
+def test_f_strings():
+    """
+    f literals are not really supported in Jedi. They just get ignored and an
+    empty string is returned.
+    """
+    context = _eval_literal('f"asdf"')
+    assert context.obj == ''
+    context = _eval_literal('f"{asdf}"')
+    assert context.obj == ''
+    context = _eval_literal('F"{asdf}"')
+    assert context.obj == ''
+    context = _eval_literal('rF"{asdf}"')
+    assert context.obj == ''
+
+
+def test_rb_strings():
+    context = _eval_literal('br"asdf"')
+    assert context.obj == b'asdf'
+    context = _eval_literal('rb"asdf"')
+    if py_version < 33:
+        # Before Python 3.3 there was a more strict definition in which order
+        # you could define literals.
+        assert context.obj == ''
+    else:
+        assert context.obj == b'asdf'