From b618c99b08b399173d021184c02ea74425572134 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 12 Jul 2017 22:28:54 +0200 Subject: [PATCH] Octal fixes. --- parso/python/grammar27.txt | 12 +++--- parso/python/tokenize.py | 2 +- test/test_pgen2.py | 85 +++++++++++++++++++------------------- 3 files changed, 49 insertions(+), 50 deletions(-) diff --git a/parso/python/grammar27.txt b/parso/python/grammar27.txt index 9426c4a..f82ef2a 100644 --- a/parso/python/grammar27.txt +++ b/parso/python/grammar27.txt @@ -35,9 +35,8 @@ stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +expr_stmt: testlist (augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist))*) augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter @@ -98,7 +97,6 @@ and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr expr: xor_expr ('|' xor_expr)* xor_expr: and_expr ('^' and_expr)* and_expr: shift_expr ('&' shift_expr)* @@ -113,20 +111,20 @@ atom: ('(' [yield_expr|testlist_comp] ')' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') # Modification by David Halter, remove `testlist_gexp` and `listmaker` -testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +testlist_comp: test ( comp_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +exprlist: expr (',' expr)* [','] testlist: test (',' test)* [','] # Modification by David Halter, dictsetmaker -> dictorsetmaker (so that it's # the same as in the 3.4 grammar). dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | (test (comp_for | (',' test)* [','])) ) -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +classdef: 'class' NAME ['(' [testlist] ')'] ':' suite arglist: (argument ',')* (argument [','] |'*' test (',' argument)* [',' '**' test] diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 43f4547..e3a4fd8 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -125,7 +125,7 @@ def _create_token_collection(version_int): if version_int >= 30: Octnumber = r'0[oO][0-7]+' else: - Octnumber = '0[0-7]+' + Octnumber = '0[oO]?[0-7]+' Decnumber = r'(?:0+|[1-9][0-9]*)' Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r'[eE][-+]?[0-9]+' diff --git a/test/test_pgen2.py b/test/test_pgen2.py index bdf9ec4..bcdfb5a 100644 --- a/test/test_pgen2.py +++ b/test/test_pgen2.py @@ -200,68 +200,68 @@ def test_annotation_8(each_py3_version): _invalid_syntax(s, each_py3_version) -def test_except_new(): +def test_except_new(each_version): s = """ try: x except E as N: y""" - _parse(s) + _parse(s, each_version) -def test_except_old(): +def test_except_old(works_in_py2): s = """ try: x except E, N: y""" - _parse(s, version='2.7') + works_in_py2.parse(s) # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms -def test_set_literal_1(): - _parse("""x = {'one'}""") +def test_set_literal_1(each_version): + _parse("""x = {'one'}""", each_version) -def test_set_literal_2(): - _parse("""x = {'one', 1,}""") +def test_set_literal_2(each_version): + _parse("""x = {'one', 1,}""", each_version) -def test_set_literal_3(): - _parse("""x = {'one', 'two', 'three'}""") +def test_set_literal_3(each_version): + _parse("""x = {'one', 'two', 'three'}""", each_version) -def test_set_literal_4(): - _parse("""x = {2, 3, 4,}""") +def test_set_literal_4(each_version): + _parse("""x = {2, 3, 4,}""", each_version) -def test_new_octal_notation(): - code = """0o7777777777777""" - if py_version >= 30: - _parse(code) - else: - _invalid_syntax(code) - _invalid_syntax("""0o7324528887""") - -def test_new_binary_notation(): - _parse("""0b101010""") - _invalid_syntax("""0b0101021""") +def test_new_octal_notation(each_version): + _parse("""0o7777777777777""", each_version) + _invalid_syntax("""0o7324528887""", each_version) -def test_class_new_syntax(): - _parse("class B(t=7): pass") - _parse("class B(t, *args): pass") - _parse("class B(t, **kwargs): pass") - _parse("class B(t, *args, **kwargs): pass") - _parse("class B(t, y=9, *args, **kwargs): pass") +def test_old_octal_notation(works_in_py2): + works_in_py2.parse("07") -def test_parser_idempotency_extended_unpacking(): +def test_new_binary_notation(each_version): + _parse("""0b101010""", each_version) + _invalid_syntax("""0b0101021""", each_version) + + +def test_class_new_syntax(works_ge_py3): + works_ge_py3.parse("class B(t=7): pass") + works_ge_py3.parse("class B(t, *args): pass") + works_ge_py3.parse("class B(t, **kwargs): pass") + works_ge_py3.parse("class B(t, *args, **kwargs): pass") + works_ge_py3.parse("class B(t, y=9, *args, **kwargs): pass") + + +def test_parser_idempotency_extended_unpacking(works_ge_py3): """A cut-down version of pytree_idempotency.py.""" - _parse("a, *b, c = x\n") - _parse("[*a, b] = x\n") - _parse("(z, *y, w) = m\n") - _parse("for *z, m in d: pass\n") + works_ge_py3.parse("a, *b, c = x\n") + works_ge_py3.parse("[*a, b] = x\n") + works_ge_py3.parse("(z, *y, w) = m\n") + works_ge_py3.parse("for *z, m in d: pass\n") -@pytest.mark.skipif('sys.version_info[0] < 3') -def test_multiline_bytes_literals(): +def test_multiline_bytes_literals(each_version): """ It's not possible to get the same result when using \xaa in Python 2/3, because it's treated differently. @@ -272,23 +272,24 @@ def test_multiline_bytes_literals(): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") """ - _parse(s) + _parse(s, each_version) -def test_multiline_bytes_tripquote_literals(): + +def test_multiline_bytes_tripquote_literals(each_version): s = ''' b""" """ ''' - _parse(s) + _parse(s, each_version) -@pytest.mark.skipif('sys.version_info[0] < 3') -def test_multiline_str_literals(): + +def test_multiline_str_literals(each_version): s = """ md5test("\xaa" * 80, ("Test Using Larger Than Block-Size Key " "and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") """ - _parse(s) + _parse(s, each_version)