Replace the tokenizer's output with a tuple (switching back from a Token class).

This commit is contained in:
Dave Halter committed 2014-12-16 00:10:07 +01:00
1 parent 680fdd574b
commit eaace104dd
5 files changed
+50 -136

No files matched your search

+2 -5
View File
@@ -291,13 +291,10 @@ class Parser(object):
self._scope = self.module
"""
new_scope = False
for token in tokenizer:
typ = token.type
value = token.value
for typ, value, start_pos, prefix in tokenizer:
if typ == tokenize.OP:
typ = grammar.opmap[value]
yield typ, value, token.prefix, token.start_pos
yield typ, value, prefix, start_pos
def __repr__(self):
return "<%s: %s>" % (type(self).__name__, self.module)