Release parso 0.8.1

Allow unparenthesized walrus in set literals, set comprehensions and indexes

CHANGELOG.rst

@@ -6,6 +6,11 @@ Changelog
 Unreleased
 ++++++++++
 
+0.8.1 (2020-12-10)
+++++++++++++++++++
+
+- Various small bugfixes
+
 0.8.0 (2020-08-05)
 ++++++++++++++++++
 

parso/__init__.py

@@ -43,7 +43,7 @@ from parso.grammar import Grammar, load_grammar
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.8.0'
+__version__ = '0.8.1'
 
 
 def parse(code=None, **kwargs):

parso/python/errors.py

@@ -957,7 +957,11 @@ class _FStringRule(SyntaxRule):
         if '\\' in expr.get_code():
             self.add_issue(expr, message=self.message_expr)
 
-        conversion = fstring_expr.children[2]
+        children_2 = fstring_expr.children[2]
+        if children_2.type == 'operator' and children_2.value == '=':
+            conversion = fstring_expr.children[3]
+        else:
+            conversion = children_2
         if conversion.type == 'fstring_conversion':
             name = conversion.children[1]
             if name.value not in ('s', 'r', 'a'):

parso/python/grammar310.txt

@@ -124,14 +124,14 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
 testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
-subscript: test | [test] ':' [test] [sliceop]
+subscript: test [':=' test] | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( ((test ':' test | '**' expr)
                    (comp_for | (',' (test ':' test | '**' expr))* [','])) |
-                  ((test | star_expr)
-                   (comp_for | (',' (test | star_expr))* [','])) )
+                  ((test [':=' test] | star_expr)
+                   (comp_for | (',' (test [':=' test] | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -167,5 +167,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' (testlist_comp | yield_expr) ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/grammar36.txt

@@ -154,5 +154,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist_comp [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' (testlist_comp | yield_expr) [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/grammar37.txt

@@ -152,5 +152,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' (testlist_comp | yield_expr) [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/grammar38.txt

@@ -167,5 +167,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' (testlist_comp | yield_expr) ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/grammar39.txt

@@ -130,8 +130,8 @@ exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( ((test ':' test | '**' expr)
                    (comp_for | (',' (test ':' test | '**' expr))* [','])) |
-                  ((test | star_expr)
-                   (comp_for | (',' (test | star_expr))* [','])) )
+                  ((test [':=' test] | star_expr)
+                   (comp_for | (',' (test [':=' test] | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -167,5 +167,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' (testlist_comp | yield_expr) ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/tokenize.py

@@ -110,9 +110,14 @@ def _get_token_collection(version_info):
             _create_token_collection(version_info)
         return result
 
-
-fstring_string_single_line = _compile(r'(?:\{\{|\}\}|\\(?:\r\n?|\n)|[^{}\r\n])+')
-fstring_string_multi_line = _compile(r'(?:[^{}]+|\{\{|\}\})+')
+unicode_character_name = r'[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*'
+fstring_string_single_line = _compile(
+    r'(?:\{\{|\}\}|\\N\{' + unicode_character_name +
+    r'\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+'
+)
+fstring_string_multi_line = _compile(
+    r'(?:\{\{|\}\}|\\N\{' + unicode_character_name + r'\}|\\[^N]|[^{}\\])+'
+)
 fstring_format_spec_single_line = _compile(r'(?:\\(?:\r\n?|\n)|[^{}\r\n])+')
 fstring_format_spec_multi_line = _compile(r'[^{}]+')
 

parso/python/tree.py

@@ -775,8 +775,8 @@ class WithStmt(Flow):
         return names
 
     def get_test_node_from_name(self, name):
-        node = name.parent
-        if node.type != 'with_item':
+        node = search_ancestor(name, "with_item")
+        if node is None:
             raise ValueError('The name is not actually part of a with statement.')
         return node.children[0]
 

test/failing_examples.py

@@ -356,4 +356,12 @@ if sys.version_info[:2] >= (3, 8):
         '(False := 1)',
         '(None := 1)',
         '(__debug__ := 1)',
+        # Unparenthesized walrus not allowed in dict literals, dict comprehensions and slices
+        '{a:="a": b:=1}',
+        '{y:=1: 2 for x in range(5)}',
+        'a[b:=0:1:2]',
+    ]
+    # f-string debugging syntax with invalid conversion character
+    FAILING_EXAMPLES += [
+        "f'{1=!b}'",
     ]

test/test_fstring.py

@@ -60,6 +60,24 @@ def grammar():
 
         # a line continuation inside of an format spec
         'f"{123:.2\\\nf}"',
+
+        # some unparenthesized syntactic structures
+        'f"{*x,}"',
+        'f"{*x, *y}"',
+        'f"{x, *y}"',
+        'f"{*x, y}"',
+        'f"{x for x in [1]}"',
+
+        # named unicode characters
+        'f"\\N{BULLET}"',
+        'f"\\N{FLEUR-DE-LIS}"',
+        'f"\\N{NO ENTRY}"',
+        'f"Combo {expr} and \\N{NO ENTRY}"',
+        'f"\\N{NO ENTRY} and {expr}"',
+        'f"\\N{no entry}"',
+        'f"\\N{SOYOMBO LETTER -A}"',
+        'f"\\N{DOMINO TILE HORIZONTAL-00-00}"',
+        'f"""\\N{NO ENTRY}"""',
     ]
 )
 def test_valid(code, grammar):
@@ -79,6 +97,7 @@ def test_valid(code, grammar):
 
         # invalid conversion characters
         'f"{1!{a}}"',
+        'f"{1=!{a}}"',
         'f"{!{a}}"',
 
         # The curly braces must contain an expression
@@ -96,6 +115,11 @@ def test_valid(code, grammar):
 
         # a newline without a line continuation inside a single-line string
         'f"abc\ndef"',
+
+        # various named unicode escapes that aren't name-shaped
+        'f"\\N{ BULLET }"',
+        'f"\\N{NO   ENTRY}"',
+        'f"""\\N{NO\nENTRY}"""',
     ]
 )
 def test_invalid(code, grammar):
@@ -114,6 +138,8 @@ def test_invalid(code, grammar):
                            (1, 10), (1, 11), (1, 12), (1, 13)]),
         ('f"""\n {\nfoo\n }"""', [(1, 0), (1, 4), (2, 1), (3, 0), (4, 1),
                                   (4, 2), (4, 5)]),
+        ('f"\\N{NO ENTRY} and {expr}"', [(1, 0), (1, 2), (1, 19), (1, 20),
+                                         (1, 24), (1, 25), (1, 26)]),
     ]
 )
 def test_tokenize_start_pos(code, positions):

test/test_parser_tree.py

@@ -229,3 +229,13 @@ def test_iter_funcdefs():
     module = parse(code, version='3.8')
     func_names = [f.name.value for f in module.iter_funcdefs()]
     assert func_names == ['normal', 'asyn', 'dec_normal', 'dec_async']
+
+
+def test_with_stmt_get_test_node_from_name():
+    code = "with A as X.Y, B as (Z), C as Q[0], D as Q['foo']: pass"
+    with_stmt = parse(code, version='3').children[0]
+    tests = [
+        with_stmt.get_test_node_from_name(name).value
+        for name in with_stmt.get_defined_names(include_setitem=True)
+    ]
+    assert tests == ["A", "B", "C", "D"]

test/test_python_errors.py

@@ -274,6 +274,8 @@ def test_paren_kwarg():
         r'fr"\""',
         r'fr"\\\""',
         r"print(f'Some {x:.2f} and some {y}')",
+        # Unparenthesized yield expression
+        'def foo(): return f"{yield 1}"',
     ]
 )
 def test_valid_fstrings(code):
@@ -287,12 +289,37 @@ def test_valid_fstrings(code):
         '[total := total + v for v in range(10)]',
         'while chunk := file.read(2):\n pass',
         'numbers = [y := math.factorial(x), y**2, y**3]',
+        '{(a:="a"): (b:=1)}',
+        '{(y:=1): 2 for x in range(5)}',
+        'a[(b:=0)]',
+        'a[(b:=0, c:=0)]',
+        'a[(b:=0):1:2]',
     ]
 )
 def test_valid_namedexpr(code):
     assert not _get_error_list(code, version='3.8')
 
 
+@pytest.mark.parametrize(
+    'code', [
+        '{x := 1, 2, 3}',
+        '{x4 := x ** 5 for x in range(7)}',
+    ]
+)
+def test_valid_namedexpr_set(code):
+    assert not _get_error_list(code, version='3.9')
+
+
+@pytest.mark.parametrize(
+    'code', [
+        'a[b:=0]',
+        'a[b:=0, c:=0]',
+    ]
+)
+def test_valid_namedexpr_index(code):
+    assert not _get_error_list(code, version='3.10')
+
+
 @pytest.mark.parametrize(
     ('code', 'message'), [
         ("f'{1+}'", ('invalid syntax')),