remove end_pos stuff from tokenizer, the tokens can do that themselves

This commit is contained in:
Dave Halter committed 2014-02-25 13:29:27 +01:00
1 parent 2252271bf5
commit 5b84f0b27f
3 files changed
+26 -43

No files matched your search

+1 -1
View File
@@ -616,7 +616,7 @@ class PushBackTokenizer(object):
def __init__(self, tokenizer):
self._tokenizer = tokenizer
self._push_backs = []
self.current = tokenize.TokenInfo(None, '', (0, 0), (0, 0))
self.current = tokenize.TokenInfo(None, '', (0, 0))
def push_last_back(self):
self._push_backs.append(self.current)