Prepare 0.8.4

ENH: add grammar file from py313

.github/workflows/build.yml

@@ -22,18 +22,15 @@ jobs:
       # Ignore F401, which are unused imports. flake8 is a primitive tool and is sometimes wrong.
       run: flake8 --extend-ignore F401 parso test/*.py setup.py scripts/
     - name: Run Mypy
-      run: mypy parso
+      run: mypy parso setup.py
   test:
     runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
         experimental: [false]
-        # include:
-        #   - python-version: '3.10-dev'
-        #     experimental: true
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

CHANGELOG.rst

@@ -6,6 +6,11 @@ Changelog
 Unreleased
 ++++++++++
 
+0.8.4 (2024-04-05)
+++++++++++++++++++
+
+- Add basic support for Python 3.13
+
 0.8.3 (2021-11-30)
 ++++++++++++++++++
 

parso/__init__.py

@@ -43,7 +43,7 @@ from parso.grammar import Grammar, load_grammar
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.8.3'
+__version__ = '0.8.4'
 
 
 def parse(code=None, **kwargs):

parso/grammar.py

@@ -106,14 +106,14 @@ class Grammar(Generic[_NodeT]):
 
         if file_io is None:
             if code is None:
-                file_io = FileIO(path)  # type: ignore
+                file_io = FileIO(path)  # type: ignore[arg-type]
             else:
                 file_io = KnownContentFileIO(path, code)
 
         if cache and file_io.path is not None:
             module_node = load_module(self._hashed, file_io, cache_path=cache_path)
             if module_node is not None:
-                return module_node  # type: ignore
+                return module_node  # type: ignore[no-any-return]
 
         if code is None:
             code = file_io.read()
@@ -132,7 +132,7 @@ class Grammar(Generic[_NodeT]):
                 module_node = module_cache_item.node
                 old_lines = module_cache_item.lines
                 if old_lines == lines:
-                    return module_node  # type: ignore
+                    return module_node  # type: ignore[no-any-return]
 
                 new_node = self._diff_parser(
                     self._pgen_grammar, self._tokenizer, module_node
@@ -144,7 +144,7 @@ class Grammar(Generic[_NodeT]):
                                    # Never pickle in pypy, it's slow as hell.
                                    pickling=cache and not is_pypy,
                                    cache_path=cache_path)
-                return new_node  # type: ignore
+                return new_node  # type: ignore[no-any-return]
 
         tokens = self._tokenizer(lines)
 
@@ -160,7 +160,7 @@ class Grammar(Generic[_NodeT]):
                                # Never pickle in pypy, it's slow as hell.
                                pickling=cache and not is_pypy,
                                cache_path=cache_path)
-        return root_node  # type: ignore
+        return root_node  # type: ignore[no-any-return]
 
     def _get_token_namespace(self):
         ns = self._token_namespace

parso/pgen2/generator.py

@@ -276,7 +276,7 @@ def generate_grammar(bnf_grammar: str, token_namespace) -> Grammar:
                     dfa_state.transitions[transition] = DFAPlan(next_dfa)
 
     _calculate_tree_traversal(rule_to_dfas)
-    return Grammar(start_nonterminal, rule_to_dfas, reserved_strings)  # type: ignore
+    return Grammar(start_nonterminal, rule_to_dfas, reserved_strings)  # type: ignore[arg-type]
 
 
 def _make_transition(token_namespace, reserved_syntax_strings, label):

parso/python/errors.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import codecs
+import sys
 import warnings
 import re
 from contextlib import contextmanager
@@ -33,7 +34,10 @@ def _get_rhs_name(node, version):
                 return "literal"
             else:
                 if second.children[1] == ":" or second.children[0] == "**":
-                    return "dict display"
+                    if version < (3, 10):
+                        return "dict display"
+                    else:
+                        return "dict literal"
                 else:
                     return "set display"
         elif (
@@ -47,7 +51,10 @@ def _get_rhs_name(node, version):
         elif first == "[":
             return "list"
         elif first == "{" and second == "}":
-            return "dict display"
+            if version < (3, 10):
+                return "dict display"
+            else:
+                return "dict literal"
         elif first == "{" and len(node.children) > 2:
             return "set display"
     elif type_ == "keyword":
@@ -58,7 +65,10 @@ def _get_rhs_name(node, version):
         else:
             return str(node.value)
     elif type_ == "operator" and node.value == "...":
-        return "Ellipsis"
+        if version < (3, 10):
+            return "Ellipsis"
+        else:
+            return "ellipsis"
     elif type_ == "comparison":
         return "comparison"
     elif type_ in ("string", "number", "strings"):
@@ -83,7 +93,10 @@ def _get_rhs_name(node, version):
         or "_test" in type_
         or type_ in ("term", "factor")
     ):
-        return "operator"
+        if version < (3, 10):
+            return "operator"
+        else:
+            return "expression"
     elif type_ == "star_expr":
         return "starred"
     elif type_ == "testlist_star_expr":
@@ -610,7 +623,10 @@ class _NameChecks(SyntaxRule):
 
 @ErrorFinder.register_rule(type='string')
 class _StringChecks(SyntaxRule):
-    message = "bytes can only contain ASCII literal characters."
+    if sys.version_info < (3, 10):
+        message = "bytes can only contain ASCII literal characters."
+    else:
+        message = "bytes can only contain ASCII literal characters"
 
     def is_issue(self, leaf):
         string_prefix = leaf.string_prefix.lower()
@@ -1043,14 +1059,20 @@ class _CheckAssignmentRule(SyntaxRule):
                         error = 'literal'
                     else:
                         if second.children[1] == ':':
-                            error = 'dict display'
+                            if self._normalizer.version < (3, 10):
+                                error = 'dict display'
+                            else:
+                                error = 'dict literal'
                         else:
                             error = 'set display'
                 elif first == "{" and second == "}":
                     if self._normalizer.version < (3, 8):
                         error = 'literal'
                     else:
-                        error = "dict display"
+                        if self._normalizer.version < (3, 10):
+                            error = "dict display"
+                        else:
+                            error = "dict literal"
                 elif first == "{" and len(node.children) > 2:
                     if self._normalizer.version < (3, 8):
                         error = 'literal'
@@ -1083,7 +1105,10 @@ class _CheckAssignmentRule(SyntaxRule):
                 error = str(node.value)
         elif type_ == 'operator':
             if node.value == '...':
-                error = 'Ellipsis'
+                if self._normalizer.version < (3, 10):
+                    error = 'Ellipsis'
+                else:
+                    error = 'ellipsis'
         elif type_ == 'comparison':
             error = 'comparison'
         elif type_ in ('string', 'number', 'strings'):
@@ -1098,7 +1123,10 @@ class _CheckAssignmentRule(SyntaxRule):
             if node.children[0] == 'await':
                 error = 'await expression'
             elif node.children[-2] == '**':
-                error = 'operator'
+                if self._normalizer.version < (3, 10):
+                    error = 'operator'
+                else:
+                    error = 'expression'
             else:
                 # Has a trailer
                 trailer = node.children[-1]
@@ -1120,7 +1148,10 @@ class _CheckAssignmentRule(SyntaxRule):
         elif ('expr' in type_ and type_ != 'star_expr'  # is a substring
               or '_test' in type_
               or type_ in ('term', 'factor')):
-            error = 'operator'
+            if self._normalizer.version < (3, 10):
+                error = 'operator'
+            else:
+                error = 'expression'
         elif type_ == "star_expr":
             if is_deletion:
                 if self._normalizer.version >= (3, 9):

parso/python/grammar313.txt

@@ -0,0 +1,169 @@
+# Grammar for Python
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://devguide.python.org/grammar/
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: stmt* ENDMARKER
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' namedexpr_test NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]]
+  | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+  | '**' tfpdef [','])
+)
+tfpdef: NAME [':' test]
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']
+)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt | NEWLINE
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal and annotated assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist_star_expr]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+
+namedexpr_test: test [':=' test]
+test: or_test ['if' or_test 'else' test] | lambdef
+lambdef: 'lambda' [varargslist] ':' test
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test [':=' test] | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test [':=' test] | star_expr)
+                   (comp_for | (',' (test [':=' test] | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguments are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test ':=' test |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
+comp_if: 'if' or_test [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist_star_expr
+
+strings: (STRING | fstring)+
+fstring: FSTRING_START fstring_content* FSTRING_END
+fstring_content: FSTRING_STRING | fstring_expr
+fstring_conversion: '!' NAME
+fstring_expr: '{' (testlist_comp | yield_expr) ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_format_spec: ':' fstring_content*

parso/python/tree.py

@@ -295,6 +295,8 @@ class FStringEnd(PythonLeaf):
 
 
 class _StringComparisonMixin:
+    __slots__ = ()
+
     def __eq__(self, other):
         """
         Make comparisons with strings easy.
@@ -544,6 +546,7 @@ class Function(ClassOrFunc):
         4. annotation (if present)
     """
     type = 'funcdef'
+    __slots__ = ()
 
     def __init__(self, children):
         super().__init__(children)

setup.cfg

@@ -13,6 +13,9 @@ ignore =
 
 
 [mypy]
+show_error_codes = true
+enable_error_code = ignore-without-code
+
 disallow_subclassing_any = True
 
 # Avoid creating future gotchas emerging from bad typing
@@ -23,3 +26,4 @@ warn_unused_configs = True
 warn_unreachable = True
 
 strict_equality = True
+no_implicit_optional = False

setup.py

@@ -47,12 +47,16 @@ setup(
     ],
     extras_require={
         'testing': [
-            'pytest<6.0.0',
+            'pytest',
             'docopt',
         ],
         'qa': [
-            'flake8==3.8.3',
-            'mypy==0.782',
+            # Latest version which supports Python 3.6
+            'flake8==5.0.4',
+            # Latest version which supports Python 3.6
+            'mypy==0.971',
+            # Arbitrary pins, latest at the time of pinning
+            'types-setuptools==67.2.0.1',
         ],
     },
 )

test/test_python_errors.py

@@ -1,6 +1,7 @@
 """
 Testing if parso finds syntax errors and indentation errors.
 """
+import re
 import sys
 import warnings
 
@@ -136,6 +137,28 @@ def _get_actual_exception(code):
         wanted = 'SyntaxError: invalid syntax'
     elif wanted == "SyntaxError: f-string: single '}' is not allowed":
         wanted = 'SyntaxError: invalid syntax'
+    elif "Maybe you meant '==' instead of '='?" in wanted:
+        wanted = wanted.removesuffix(" here. Maybe you meant '==' instead of '='?")
+    elif re.match(
+        r"SyntaxError: unterminated string literal \(detected at line \d+\)", wanted
+    ):
+        wanted = "SyntaxError: EOL while scanning string literal"
+    elif re.match(
+        r"SyntaxError: unterminated triple-quoted string literal \(detected at line \d+\)",
+        wanted,
+    ):
+        wanted = 'SyntaxError: EOF while scanning triple-quoted string literal'
+    elif wanted == 'SyntaxError: cannot use starred expression here':
+        wanted = "SyntaxError: can't use starred expression here"
+    elif wanted == 'SyntaxError: f-string: cannot use starred expression here':
+        wanted = "SyntaxError: f-string: can't use starred expression here"
+    elif re.match(
+        r"IndentationError: expected an indented block after '[^']*' statement on line \d",
+        wanted,
+    ):
+        wanted = 'IndentationError: expected an indented block'
+    elif wanted == 'SyntaxError: unterminated string literal':
+        wanted = 'SyntaxError: EOL while scanning string literal'
     return [wanted], line_nr