diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 44e1ac70..dc6aa3b4 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -105,13 +105,17 @@ endprogs = {"'": _compile(single), '"': _compile(double), "r'''": single3prog, 'r"""': double3prog, "b'''": single3prog, 'b"""': double3prog, "u'''": single3prog, 'u"""': double3prog, - "br'''": single3prog, 'br"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "B'''": single3prog, 'B"""': double3prog, "U'''": single3prog, 'U"""': double3prog, + "br'''": single3prog, 'br"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "ur'''": single3prog, 'ur"""': double3prog, + "uR'''": single3prog, 'uR"""': double3prog, + "Ur'''": single3prog, 'Ur"""': double3prog, + "UR'''": single3prog, 'UR"""': double3prog, 'r': None, 'R': None, 'b': None, 'B': None} triple_quoted = {} @@ -120,7 +124,9 @@ for t in ("'''", '"""', "b'''", 'b"""', "B'''", 'B"""', "u'''", 'u"""', "U'''", 'U"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""'): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "ur'''", 'ur"""', "Ur'''", 'Ur"""', + "uR'''", 'uR"""', "UR'''", 'UR"""'): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', @@ -128,7 +134,9 @@ for t in ("'", '"', "b'", 'b"', "B'", 'B"', "u'", 'u"', "U'", 'U"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"'): + "bR'", 'bR"', "BR'", 'BR"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"'): single_quoted[t] = t del _compile diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 68144aaf..35024d7d 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -3,6 +3,8 @@ from io import StringIO from textwrap import dedent +import pytest + from jedi._compatibility import u, is_py3 from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT from jedi.parser import Parser, load_grammar, tokenize @@ -120,3 +122,31 @@ def test_tokenizer_with_string_literal_backslash(): import jedi c = jedi.Script("statement = u'foo\\\n'; statement").goto_definitions() assert c[0]._name.parent.obj == 'foo' + + +def test_ur_literals(): + """ + Decided to parse `u''` literals regardless of Python version. This makes + probably sense: + + - Python 3.2 doesn't support it and is still supported by Jedi, but might + not be. While this is incorrect, it's just incorrect for one "old" and in + the future not very important version. + - All the other Python versions work very well with it. + """ + def check(literal): + io = StringIO(literal) + tokens = tokenize.generate_tokens(io.readline) + token_list = list(tokens) + typ, result_literal, _, _ = token_list[0] + assert typ == STRING + assert result_literal == literal + + check('u""') + check('ur""') + check('Ur""') + check('UR""') + check('bR""') + # Must be in the right order. + with pytest.raises(AssertionError): + check('Rb""')