a basic approach to the new fast parser

This commit is contained in:
David Halter committed 2013-04-10 22:33:49 +04:30
1 parent a99d9541bd
commit 5dd05eff1a
4 files changed
+158 -114

No files matched your search

+1 -2
View File
@@ -126,7 +126,6 @@ class NoErrorTokenizer(object):
else:
c[2] = self.offset[0] + c[2][0], c[2][1]
c[3] = self.offset[0] + c[3][0], c[3][1]
print 'h', c, tokenize.tok_name[c[0]], self.current[2:4]
self.current = c
def close():
@@ -139,7 +138,7 @@ class NoErrorTokenizer(object):
tokenize.NEWLINE, tokenize.DEDENT) \
and c[0] not in (tokenize.COMMENT, tokenize.INDENT,
tokenize.NL, tokenize.NEWLINE, tokenize.DEDENT):
print c, tokenize.tok_name[c[0]]
#print c, tokenize.tok_name[c[0]]
tok = c[1]
indent = c[2][1]