Errortokens should also make the parser fail in the normal parser.

This commit is contained in:
Dave Halter
2015-12-20 23:11:52 +01:00
parent 5791860861
commit c15551ccc1
3 changed files with 5 additions and 2 deletions
+3 -1
View File
@@ -156,7 +156,9 @@ class Parser(object):
def _tokenize(self, tokenizer):
for typ, value, start_pos, prefix in tokenizer:
if typ == OP:
if typ == ERRORTOKEN:
raise Parser.ParserError
elif typ == OP:
typ = token.opmap[value]
yield typ, value, prefix, start_pos