From 6098d891504421583437d10dc12b2c2fa33ef77e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 24 Jun 2018 13:03:07 +0200 Subject: [PATCH] Add PythonTokens to get rid of a lot of the token module eventually --- parso/python/token.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/parso/python/token.py b/parso/python/token.py index 6f7ad5a..9571364 100644 --- a/parso/python/token.py +++ b/parso/python/token.py @@ -113,3 +113,30 @@ def generate_token_id(string): except KeyError: pass return globals()[string] + + +class Token(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.name) + + +class Tokens(object): + """ + Basically an enum, but Python 2 doesn't have enums in the standard library. + """ + def __init__(self, names, contains_syntax): + for name in names: + setattr(self, name, Token(name)) + + self.contains_syntax = [getattr(self, name) for name in contains_syntax] + + +PythonTokens = Tokens(( + 'STRING', 'NUMBER', 'NAME', 'ERRORTOKEN', 'NEWLINE', 'INDENT', 'DEDENT', + 'ERROR_DEDENT', 'FSTRING_STRING', 'FSTRING_START', 'FSTRING_END', 'OP', + 'ENDMARKER'), + contains_syntax=('NAME', 'OP'), +)