From 8cae7ed526ea21d3b84cbf9ff2119e587384e18e Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sun, 1 Dec 2019 15:43:17 +0800 Subject: [PATCH 1/4] Fixing davidhalter/parso#89 [all changes are in parso/python/errors.py] * utility function (`_get_namedexpr`) extracting all assignment expression (`namedexpr_test`) nodes * add `is_namedexpr` parameter to `_CheckAssignmentRule._check_assignment` and special error message for assignment expression related assignment issues (*cannot use named assignment with xxx*) * add assignment expression check to `_CompForRule` (*assignment expression cannot be used in a comprehension iterable expression*) * add `_NamedExprRule` for special assignment expression checks - *cannot use named assignment with lambda* - *cannot use named assignment with subscript* - *cannot use named assignment with attribute* - and fallback general checks in `_CheckAssignmentRule._check_assignment` * add `_ComprehensionRule` for special checks on assignment expression in a comprehension - *assignment expression within a comprehension cannot be used in a class body* - *assignment expression cannot rebind comprehension iteration variable 'xxx'* --- parso/python/errors.py | 105 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 6 deletions(-) diff --git a/parso/python/errors.py b/parso/python/errors.py index 6bf0d17..f48c8aa 100644 --- a/parso/python/errors.py +++ b/parso/python/errors.py @@ -125,6 +125,20 @@ def _get_for_stmt_definition_exprs(for_stmt): return list(_iter_definition_exprs_from_lists(exprlist)) +def _get_namedexpr(node): + """Get assignment expression if node contains.""" + namedexpr_list = list() + + def fetch_namedexpr(node): + if node.type == 'namedexpr_test': + namedexpr_list.append(node) + if hasattr(node, 'children'): + [fetch_namedexpr(child) for child in node.children] + + fetch_namedexpr(node) + return namedexpr_list + + class _Context(object): def __init__(self, node, add_syntax_error, parent_context=None): self.node = node @@ -883,7 +897,7 @@ class _FStringRule(SyntaxRule): class _CheckAssignmentRule(SyntaxRule): - def _check_assignment(self, node, is_deletion=False): + def _check_assignment(self, node, is_deletion=False, is_namedexpr=False): error = None type_ = node.type if type_ == 'lambdef': @@ -907,9 +921,9 @@ class _CheckAssignmentRule(SyntaxRule): # This is not a comprehension, they were handled # further above. for child in second.children[::2]: - self._check_assignment(child, is_deletion) + self._check_assignment(child, is_deletion, is_namedexpr) else: # Everything handled, must be useless brackets. - self._check_assignment(second, is_deletion) + self._check_assignment(second, is_deletion, is_namedexpr) elif type_ == 'keyword': if self._normalizer.version < (3, 8): error = 'keyword' @@ -941,15 +955,18 @@ class _CheckAssignmentRule(SyntaxRule): error = 'function call' elif type_ in ('testlist_star_expr', 'exprlist', 'testlist'): for child in node.children[::2]: - self._check_assignment(child, is_deletion) + self._check_assignment(child, is_deletion, is_namedexpr) elif ('expr' in type_ and type_ != 'star_expr' # is a substring or '_test' in type_ or type_ in ('term', 'factor')): error = 'operator' if error is not None: - cannot = "can't" if self._normalizer.version < (3, 8) else "cannot" - message = ' '.join([cannot, "delete" if is_deletion else "assign to", error]) + if is_namedexpr: + message = 'cannot use named assignment with %s' % error + else: + cannot = "can't" if self._normalizer.version < (3, 8) else "cannot" + message = ' '.join([cannot, "delete" if is_deletion else "assign to", error]) self.add_issue(node, message=message) @@ -962,6 +979,14 @@ class _CompForRule(_CheckAssignmentRule): if expr_list.type != 'expr_list': # Already handled. self._check_assignment(expr_list) + or_test = node.children[3] + expr_list = _get_namedexpr(or_test) + for expr in expr_list: + # [i+1 for i in (i := range(5))] + # [i+1 for i in (j := range(5))] + # [i+1 for i in (lambda: (j := range(5)))()] + self.add_issue(expr, message='assignment expression cannot be used in a comprehension iterable expression') + return node.parent.children[0] == 'async' \ and not self._normalizer.context.is_async_funcdef() @@ -1008,3 +1033,71 @@ class _ForStmtRule(_CheckAssignmentRule): expr_list = for_stmt.children[1] if expr_list.type != 'expr_list': # Already handled. self._check_assignment(expr_list) + + +@ErrorFinder.register_rule(type='namedexpr_test') +class _NamedExprRule(_CheckAssignmentRule): + # namedexpr_test: test [':=' test] + + def is_issue(self, namedexpr_test): + first = namedexpr_test.children[0] + if first.type == 'lambdef': + # (lambda: x := 1) + self.add_issue(namedexpr_test, message='cannot use named assignment with lambda') + elif first.type == 'atom_expr': + for child in first.children: + if child.type != 'trailer': + continue + first_child = child.children[0] + if first_child.type == 'operator': + if first_child.value == '[': + # (a[i] := x) + self.add_issue(namedexpr_test, message='cannot use named assignment with subscript') + elif first_child.value == '.': + # (a.b := c) + self.add_issue(namedexpr_test, message='cannot use named assignment with attribute') + else: + self._check_assignment(first, is_namedexpr=True) + + +@ErrorFinder.register_rule(type='testlist_comp') +@ErrorFinder.register_rule(type='dictorsetmaker') +class _ComprehensionRule(SyntaxRule): + # testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) + # dictorsetmaker: ( ((test ':' test | '**' expr) + # (comp_for | (',' (test ':' test | '**' expr)) * [','])) | + # ((test | star_expr) + # (comp_for | (',' (test | star_expr)) * [',']))) + + def is_issue(self, node): + exprlist = None + for child in node.children: + if child.type in _COMP_FOR_TYPES: + if child.type == 'sync_comp_for': + exprlist = _get_for_stmt_definition_exprs(child) + elif child.type == 'comp_for': + exprlist = _get_for_stmt_definition_exprs(child.children[1]) + break + + # not a comprehension + if exprlist is None: + return + + # in class body + in_class = self._normalizer.context.node.type == 'classdef' + + namelist = [expr.value for expr in exprlist] + namedexpr_list = _get_namedexpr(node) + for expr in namedexpr_list: + if in_class: + # class Example: + # [(j := i) for i in range(5)] + self.add_issue(expr, message='assignment expression within a comprehension cannot be used in a class body') + + first = expr.children[0] + if first.type == 'name' and first.value in namelist: + # [i := 0 for i, j in range(5)] + # [[(i := i) for j in range(5)] for i in range(5)] + # [i for i, j in range(5) if True or (i := 1)] + # [False and (i := 0) for i, j in range(5)] + self.add_issue(expr, message='assignment expression cannot rebind comprehension iteration variable %r' % first.value) From 76fe4792e74c71f8533648654205f2f25208244f Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sun, 1 Dec 2019 16:23:18 +0800 Subject: [PATCH 2/4] Deal with nested comprehension e.g. `[i for i, j in range(5) for k in range (10) if True or (i := 1)]` --- parso/python/errors.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/parso/python/errors.py b/parso/python/errors.py index f48c8aa..913af98 100644 --- a/parso/python/errors.py +++ b/parso/python/errors.py @@ -1070,14 +1070,26 @@ class _ComprehensionRule(SyntaxRule): # (comp_for | (',' (test | star_expr)) * [',']))) def is_issue(self, node): - exprlist = None + exprlist = list() + namedexpr_list = list() + + def process_comp(comp_for): + if comp_for.type in _COMP_FOR_TYPES: + if comp_for.type == 'sync_comp_for': + comp = comp_for + elif comp_for.type == 'comp_for': + comp = comp_for.children[1] + exprlist.extend(_get_for_stmt_definition_exprs(comp)) + + if len(comp.children) > 4: + comp_iter = comp.children[4] + process_comp(comp_iter) + else: + # skip assignment expressions in comp_for + namedexpr_list.extend(_get_namedexpr(comp_for)) + for child in node.children: - if child.type in _COMP_FOR_TYPES: - if child.type == 'sync_comp_for': - exprlist = _get_for_stmt_definition_exprs(child) - elif child.type == 'comp_for': - exprlist = _get_for_stmt_definition_exprs(child.children[1]) - break + process_comp(child) # not a comprehension if exprlist is None: @@ -1087,7 +1099,6 @@ class _ComprehensionRule(SyntaxRule): in_class = self._normalizer.context.node.type == 'classdef' namelist = [expr.value for expr in exprlist] - namedexpr_list = _get_namedexpr(node) for expr in namedexpr_list: if in_class: # class Example: From 776e151370b76df994f79eb6840823457b832c08 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 13 Dec 2019 11:55:53 +0800 Subject: [PATCH 3/4] Revised implementation * search ancestors of namedexpr_test directly for comprehensions * added test samples for invalid namedexpr_test syntax --- parso/python/errors.py | 94 +++++++++++++++++----------------------- test/failing_examples.py | 22 ++++++++++ 2 files changed, 62 insertions(+), 54 deletions(-) diff --git a/parso/python/errors.py b/parso/python/errors.py index 913af98..9ae5329 100644 --- a/parso/python/errors.py +++ b/parso/python/errors.py @@ -1041,6 +1041,46 @@ class _NamedExprRule(_CheckAssignmentRule): def is_issue(self, namedexpr_test): first = namedexpr_test.children[0] + + # defined names + exprlist = list() + + def process_comp_for(comp_for): + if comp_for.type == 'sync_comp_for': + comp = comp_for + elif comp_for.type == 'comp_for': + comp = comp_for.children[1] + exprlist.extend(_get_for_stmt_definition_exprs(comp)) + + def search_all_comp_ancestors(node): + ancestors = list() + while True: + node = node.parent + if node is None: + return ancestors + if node.type in ['testlist_comp', 'dictorsetmaker']: + for child in node.children: + if child.type in _COMP_FOR_TYPES: + process_comp_for(child) + ancestors.append(node) + break + + # check assignment expressions in comprehensions + search_all = search_all_comp_ancestors(namedexpr_test) + if search_all: + if self._normalizer.context.node.type == 'classdef': + self.add_issue( + namedexpr_test, message='assignment expression within a comprehension cannot be used in a class body') + + namelist = [expr.value for expr in exprlist] + if first.type == 'name' and first.value in namelist: + # [i := 0 for i, j in range(5)] + # [[(i := i) for j in range(5)] for i in range(5)] + # [i for i, j in range(5) if True or (i := 1)] + # [False and (i := 0) for i, j in range(5)] + self.add_issue( + namedexpr_test, message='assignment expression cannot rebind comprehension iteration variable %r' % first.value) + if first.type == 'lambdef': # (lambda: x := 1) self.add_issue(namedexpr_test, message='cannot use named assignment with lambda') @@ -1058,57 +1098,3 @@ class _NamedExprRule(_CheckAssignmentRule): self.add_issue(namedexpr_test, message='cannot use named assignment with attribute') else: self._check_assignment(first, is_namedexpr=True) - - -@ErrorFinder.register_rule(type='testlist_comp') -@ErrorFinder.register_rule(type='dictorsetmaker') -class _ComprehensionRule(SyntaxRule): - # testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) - # dictorsetmaker: ( ((test ':' test | '**' expr) - # (comp_for | (',' (test ':' test | '**' expr)) * [','])) | - # ((test | star_expr) - # (comp_for | (',' (test | star_expr)) * [',']))) - - def is_issue(self, node): - exprlist = list() - namedexpr_list = list() - - def process_comp(comp_for): - if comp_for.type in _COMP_FOR_TYPES: - if comp_for.type == 'sync_comp_for': - comp = comp_for - elif comp_for.type == 'comp_for': - comp = comp_for.children[1] - exprlist.extend(_get_for_stmt_definition_exprs(comp)) - - if len(comp.children) > 4: - comp_iter = comp.children[4] - process_comp(comp_iter) - else: - # skip assignment expressions in comp_for - namedexpr_list.extend(_get_namedexpr(comp_for)) - - for child in node.children: - process_comp(child) - - # not a comprehension - if exprlist is None: - return - - # in class body - in_class = self._normalizer.context.node.type == 'classdef' - - namelist = [expr.value for expr in exprlist] - for expr in namedexpr_list: - if in_class: - # class Example: - # [(j := i) for i in range(5)] - self.add_issue(expr, message='assignment expression within a comprehension cannot be used in a class body') - - first = expr.children[0] - if first.type == 'name' and first.value in namelist: - # [i := 0 for i, j in range(5)] - # [[(i := i) for j in range(5)] for i in range(5)] - # [i for i, j in range(5) if True or (i := 1)] - # [False and (i := 0) for i, j in range(5)] - self.add_issue(expr, message='assignment expression cannot rebind comprehension iteration variable %r' % first.value) diff --git a/test/failing_examples.py b/test/failing_examples.py index c15cbf8..c4f247a 100644 --- a/test/failing_examples.py +++ b/test/failing_examples.py @@ -319,3 +319,25 @@ if sys.version_info[:2] < (3, 8): continue '''), # 'continue' not supported inside 'finally' clause" ] + +if sys.version_info[:2] >= (3, 8): + # assignment expressions from issue#89 + FAILING_EXAMPLES += [ + # Case 2 + '(lambda: x := 1)', + # Case 3 + '(a[i] := x)', + # Case 4 + '(a.b := c)', + # Case 5 + '[i:= 0 for i, j in range(5)]', + '[[(i:= i) for j in range(5)] for i in range(5)]', + '[i for i, j in range(5) if True or (i:= 1)]', + '[False and (i:= 0) for i, j in range(5)]', + # Case 6 + '[i+1 for i in (i:= range(5))]', + '[i+1 for i in (j:= range(5))]', + '[i+1 for i in (lambda: (j:= range(5)))()]', + # Case 7 + 'class Example:\n [(j := i) for i in range(5)]', + ] From 89c4d959e9a9fba4559eafd8fd57d92c82ce1d83 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 14 Dec 2019 09:37:16 +0100 Subject: [PATCH 4/4] * moved all namedexpr_test related rules to `_NamedExprRule` * added valid examples --- parso/python/errors.py | 37 ++++++++++++++++--------------------- test/test_python_errors.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/parso/python/errors.py b/parso/python/errors.py index 9ae5329..71c0dad 100644 --- a/parso/python/errors.py +++ b/parso/python/errors.py @@ -125,19 +125,6 @@ def _get_for_stmt_definition_exprs(for_stmt): return list(_iter_definition_exprs_from_lists(exprlist)) -def _get_namedexpr(node): - """Get assignment expression if node contains.""" - namedexpr_list = list() - - def fetch_namedexpr(node): - if node.type == 'namedexpr_test': - namedexpr_list.append(node) - if hasattr(node, 'children'): - [fetch_namedexpr(child) for child in node.children] - - fetch_namedexpr(node) - return namedexpr_list - class _Context(object): def __init__(self, node, add_syntax_error, parent_context=None): @@ -979,14 +966,6 @@ class _CompForRule(_CheckAssignmentRule): if expr_list.type != 'expr_list': # Already handled. self._check_assignment(expr_list) - or_test = node.children[3] - expr_list = _get_namedexpr(or_test) - for expr in expr_list: - # [i+1 for i in (i := range(5))] - # [i+1 for i in (j := range(5))] - # [i+1 for i in (lambda: (j := range(5)))()] - self.add_issue(expr, message='assignment expression cannot be used in a comprehension iterable expression') - return node.parent.children[0] == 'async' \ and not self._normalizer.context.is_async_funcdef() @@ -1040,8 +1019,24 @@ class _NamedExprRule(_CheckAssignmentRule): # namedexpr_test: test [':=' test] def is_issue(self, namedexpr_test): + # assigned name first = namedexpr_test.children[0] + def search_namedexpr_in_comp_for(node): + while True: + parent = node.parent + if parent is None: + return parent + if parent.type == 'sync_comp_for' and parent.children[3] == node: + return parent + node = parent + + if search_namedexpr_in_comp_for(namedexpr_test): + # [i+1 for i in (i := range(5))] + # [i+1 for i in (j := range(5))] + # [i+1 for i in (lambda: (j := range(5)))()] + self.add_issue(namedexpr_test, message='assignment expression cannot be used in a comprehension iterable expression') + # defined names exprlist = list() diff --git a/test/test_python_errors.py b/test/test_python_errors.py index 71a67eb..ce5bcb3 100644 --- a/test/test_python_errors.py +++ b/test/test_python_errors.py @@ -293,6 +293,19 @@ def test_valid_fstrings(code): assert not _get_error_list(code, version='3.6') +@pytest.mark.parametrize( + 'code', [ + 'a = (b := 1)', + '[x4 := x ** 5 for x in range(7)]', + '[total := total + v for v in range(10)]', + 'while chunk := file.read(2):\n pass', + 'numbers = [y := math.factorial(x), y**2, y**3]', + ] +) +def test_valid_namedexpr(code): + assert not _get_error_list(code, version='3.8') + + @pytest.mark.parametrize( ('code', 'message'), [ ("f'{1+}'", ('invalid syntax')),