Use enum instead of our own logic

This commit is contained in:
Dave Halter
2020-07-24 17:39:49 +02:00
parent a0662b3b3b
commit 19f4550ced
3 changed files with 18 additions and 17 deletions

View File

@@ -86,7 +86,7 @@ class StackNode(object):
def _token_to_transition(grammar, type_, value):
# Map from token to label
if type_.contains_syntax:
if type_.value.contains_syntax:
# Check for reserved words (keywords)
try:
return grammar.reserved_syntax_strings[value]

View File

@@ -1,5 +1,7 @@
from __future__ import absolute_import
from enum import Enum
class TokenType(object):
def __init__(self, name, contains_syntax=False):
@@ -10,18 +12,17 @@ class TokenType(object):
return '%s(%s)' % (self.__class__.__name__, self.name)
class TokenTypes(object):
"""
Basically an enum, but Python 2 doesn't have enums in the standard library.
"""
def __init__(self, names, contains_syntax):
for name in names:
setattr(self, name, TokenType(name, contains_syntax=name in contains_syntax))
PythonTokenTypes = TokenTypes((
'STRING', 'NUMBER', 'NAME', 'ERRORTOKEN', 'NEWLINE', 'INDENT', 'DEDENT',
'ERROR_DEDENT', 'FSTRING_STRING', 'FSTRING_START', 'FSTRING_END', 'OP',
'ENDMARKER'),
contains_syntax=('NAME', 'OP'),
)
class PythonTokenTypes(Enum):
STRING = TokenType('STRING')
NUMBER = TokenType('NUMBER')
NAME = TokenType('NAME', contains_syntax=True)
ERRORTOKEN = TokenType('ERRORTOKEN')
NEWLINE = TokenType('NEWLINE')
INDENT = TokenType('INDENT')
DEDENT = TokenType('DEDENT')
ERROR_DEDENT = TokenType('ERROR_DEDENT')
FSTRING_STRING = TokenType('FSTRING_STRING')
FSTRING_START = TokenType('FSTRING_START')
FSTRING_END = TokenType('FSTRING_END')
OP = TokenType('OP', contains_syntax=True)
ENDMARKER = TokenType('ENDMARKER')

View File

@@ -351,7 +351,7 @@ def test_left_recursion():
@pytest.mark.parametrize(
'grammar, error_match', [
['foo: bar | baz\nbar: NAME\nbaz: NAME\n',
r"foo is ambiguous.*given a TokenType\(NAME\).*bar or baz"],
r"foo is ambiguous.*given a PythonTokenTypes\.NAME.*bar or baz"],
['''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''',
r"foo is ambiguous.*given a ReservedString\(x\).*bar or baz"],
['''foo: bar | 'x'\nbar: 'x'\n''',