remove encoding stuff from tokenizer - encoding is always unicode
This commit is contained in:
1 parent
1a3cca6edb
commit
e152939791
1 file changed
-2
@@ -31,8 +31,6 @@ namechars = string.ascii_letters + '_'
|
||||
|
||||
COMMENT = N_TOKENS
|
||||
tok_name[COMMENT] = 'COMMENT'
|
||||
ENCODING = N_TOKENS + 1
|
||||
tok_name[ENCODING] = 'ENCODING'
|
||||
|
||||
|
||||
class Token(object):
|
||||
|
||||
Reference in new issue
Block a user