From 19f4550cedfc5cf519ebee4f3aa0b1addf082992 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 24 Jul 2020 17:39:49 +0200 Subject: [PATCH] Use enum instead of our own logic --- parso/parser.py | 2 +- parso/python/token.py | 31 ++++++++++++++++--------------- test/test_pgen2.py | 2 +- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/parso/parser.py b/parso/parser.py index 66f5443..2bfa89f 100644 --- a/parso/parser.py +++ b/parso/parser.py @@ -86,7 +86,7 @@ class StackNode(object): def _token_to_transition(grammar, type_, value): # Map from token to label - if type_.contains_syntax: + if type_.value.contains_syntax: # Check for reserved words (keywords) try: return grammar.reserved_syntax_strings[value] diff --git a/parso/python/token.py b/parso/python/token.py index bb86ec9..be2b13c 100644 --- a/parso/python/token.py +++ b/parso/python/token.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +from enum import Enum + class TokenType(object): def __init__(self, name, contains_syntax=False): @@ -10,18 +12,17 @@ class TokenType(object): return '%s(%s)' % (self.__class__.__name__, self.name) -class TokenTypes(object): - """ - Basically an enum, but Python 2 doesn't have enums in the standard library. - """ - def __init__(self, names, contains_syntax): - for name in names: - setattr(self, name, TokenType(name, contains_syntax=name in contains_syntax)) - - -PythonTokenTypes = TokenTypes(( - 'STRING', 'NUMBER', 'NAME', 'ERRORTOKEN', 'NEWLINE', 'INDENT', 'DEDENT', - 'ERROR_DEDENT', 'FSTRING_STRING', 'FSTRING_START', 'FSTRING_END', 'OP', - 'ENDMARKER'), - contains_syntax=('NAME', 'OP'), -) +class PythonTokenTypes(Enum): + STRING = TokenType('STRING') + NUMBER = TokenType('NUMBER') + NAME = TokenType('NAME', contains_syntax=True) + ERRORTOKEN = TokenType('ERRORTOKEN') + NEWLINE = TokenType('NEWLINE') + INDENT = TokenType('INDENT') + DEDENT = TokenType('DEDENT') + ERROR_DEDENT = TokenType('ERROR_DEDENT') + FSTRING_STRING = TokenType('FSTRING_STRING') + FSTRING_START = TokenType('FSTRING_START') + FSTRING_END = TokenType('FSTRING_END') + OP = TokenType('OP', contains_syntax=True) + ENDMARKER = TokenType('ENDMARKER') diff --git a/test/test_pgen2.py b/test/test_pgen2.py index 1b82baa..2166cdc 100644 --- a/test/test_pgen2.py +++ b/test/test_pgen2.py @@ -351,7 +351,7 @@ def test_left_recursion(): @pytest.mark.parametrize( 'grammar, error_match', [ ['foo: bar | baz\nbar: NAME\nbaz: NAME\n', - r"foo is ambiguous.*given a TokenType\(NAME\).*bar or baz"], + r"foo is ambiguous.*given a PythonTokenTypes\.NAME.*bar or baz"], ['''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''', r"foo is ambiguous.*given a ReservedString\(x\).*bar or baz"], ['''foo: bar | 'x'\nbar: 'x'\n''',