Remove some old unused tokenize stuff.

This commit is contained in:
Dave Halter committed 2014-12-15 17:44:40 +01:00
1 parent 955f125c0d
commit 680fdd574b
2 files changed
+1 -11

No files matched your search

+1 -1
View File
@@ -122,7 +122,7 @@ class Parser(object):
p = PgenParser(grammar, self.convert_node, self.convert_leaf,
self.error_recovery)
tokenizer = tokenizer or tokenize.source_tokens(source)
self.module = p.parse(p.tokenize(self._tokenize(tokenizer)))
self.module = p.parse(self._tokenize(tokenizer))
self.module.used_names = self.used_names
self.module.path = module_path