diff --git a/AUTHORS.txt b/AUTHORS.txt index fd604327..64aae316 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -32,5 +32,6 @@ Albertas Agejevas (@alga) Savor d'Isavano (@KenetJervet) Phillip Berndt (@phillipberndt) Ian Lee (@IanLee1521) +Farkhad Khatamov (@hatamov) Note: (@user) means a github user name. diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 707b82ba..82b02d66 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -68,7 +68,7 @@ double = r'[^"\\]*(?:\\.[^"\\]*)*"' single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""') +triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') # Single-line ' or " string. # Because of leftmost-then-longest match semantics, be sure to put the @@ -126,7 +126,7 @@ single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', "b'", 'b"', "B'", 'B"', - "u'", 'u""', "U'", 'U"', + "u'", 'u"', "U'", 'U"', "br'", 'br"', "Br'", 'Br"', "bR'", 'bR"', "BR'", 'BR"'): single_quoted[t] = t diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 980e913d..b4ad3cdc 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -6,6 +6,7 @@ from textwrap import dedent from jedi._compatibility import u, is_py3 from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT from jedi import parser +from token import STRING from ..helpers import unittest @@ -95,6 +96,24 @@ class TokenTest(unittest.TestCase): # They will be ignored in the parser, that's ok. assert unicode_token[0] == OP + def test_quoted_strings(self): + + string_tokens = [ + 'u"test"', + 'u"""test"""', + 'U"""test"""', + "u'''test'''", + "U'''test'''", + ] + + for s in string_tokens: + parsed = parser.Parser(u('''a = %s\n''' % s)) + tok_list = parsed.module.statements[0]._token_list + self.assertEqual(len(tok_list), 3) + tok = tok_list[2] + self.assertIsInstance(tok, parser.tokenize.Token) + self.assertEqual(tok.type, STRING) + def test_tokenizer_with_string_literal_backslash(): import jedi