Fix some endmarker prefix issues in the fast parser.

This commit is contained in:
Dave Halter
2015-02-03 22:09:55 +01:00
parent 6cdfecb541
commit 66dfa59286
3 changed files with 42 additions and 16 deletions
+2 -2
View File
@@ -252,7 +252,7 @@ class Parser(object):
#print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index)
self._stack_removal(grammar, stack, index + 1, value, start_pos)
if typ == token.INDENT:
# For every deleted INDENT we got to delete a DEDENT as well.
# For every deleted INDENT we have to delete a DEDENT as well.
# Otherwise the parser will get into trouble and DEDENT too early.
self._omit_dedent += 1
@@ -307,7 +307,7 @@ class Parser(object):
def _tokenize(self, tokenizer):
for typ, value, start_pos, prefix in tokenizer:
# print(token.tok_name[typ], repr(value), start_pos, repr(prefix))
print(token.tok_name[typ], repr(value), start_pos, repr(prefix))
if self._omit_dedent and typ == token.DEDENT:
self._omit_dedent -= 1
continue