remove encoding stuff from tokenizer - encoding is always unicode

This commit is contained in:
Dave Halter
2014-02-26 12:55:32 +01:00
parent 1a3cca6edb
commit e152939791

View File

@@ -31,8 +31,6 @@ namechars = string.ascii_letters + '_'
COMMENT = N_TOKENS
tok_name[COMMENT] = 'COMMENT'
ENCODING = N_TOKENS + 1
tok_name[ENCODING] = 'ENCODING'
class Token(object):