From fa0bf4951c329a2ebdd60dbeabe070ce24f4d272 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 1 Sep 2017 00:20:24 +0200 Subject: [PATCH] Fix string prefixes for Python2. --- parso/python/tokenize.py | 20 ++++++++++++-------- test/test_pgen2.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index e5411c7..ecd2437 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -62,17 +62,18 @@ def maybe(*choices): # Return the empty string, plus all of the valid string prefixes. def _all_string_prefixes(version_info): + def different_case_versions(prefix): + for s in _itertools.product(*[(c, c.upper()) for c in prefix]): + yield ''.join(s) # The valid string prefixes. Only contain the lower case versions, # and don't contain any permuations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'br'] + _valid_string_prefixes = ['b', 'r', 'u'] + if version_info >= (3, 0): + _valid_string_prefixes.append('br') + if version_info >= (3, 6): _valid_string_prefixes += ['f', 'fr'] - if version_info <= (2, 7): - # TODO this is actually not 100% valid. ur is valid in Python 2.7, - # while ru is not. - # TODO rb is also not valid. - _valid_string_prefixes.append('ur') # if we add binary f-strings, add: ['fb', 'fbr'] result = set(['']) @@ -80,8 +81,11 @@ def _all_string_prefixes(version_info): for t in _itertools.permutations(prefix): # create a list with upper and lower versions of each # character - for s in _itertools.product(*[(c, c.upper()) for c in t]): - result.add(''.join(s)) + result.update(different_case_versions(t)) + if version_info <= (2, 7): + # In Python 2 the order cannot just be random. + result.update(different_case_versions('ur')) + result.update(different_case_versions('br')) return result diff --git a/test/test_pgen2.py b/test/test_pgen2.py index d559e5b..8c6e90f 100644 --- a/test/test_pgen2.py +++ b/test/test_pgen2.py @@ -254,3 +254,19 @@ def test_multiline_str_literals(each_version): def test_py2_backticks(works_in_py2): works_in_py2.parse("`1`") + + +def test_py2_string_prefixes(works_in_py2): + works_in_py2.parse("ur'1'") + works_in_py2.parse("Ur'1'") + works_in_py2.parse("UR'1'") + _invalid_syntax("ru'1'", works_in_py2.version) + + +def py_br(each_version): + _parse('br""', each_version) + + +def test_py3_rb(works_ge_py3): + works_ge_py3.parse("rb'1'") + works_ge_py3.parse("RB'1'")