Fix tokenizer error tokens

This commit is contained in:
Dave Halter committed 2020-04-07 09:55:28 +02:00
1 parent f45941226f
commit 7b14a86e0a
2 files changed
+26 -1

No files matched your search

+1 -1
View File
@@ -532,7 +532,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first
if not pseudomatch: # scan for tokens
match = whitespace.match(line, pos)
if pos == 0 and paren_level == 0 and not fstring_stack:
if new_line and paren_level == 0 and not fstring_stack:
for t in dedent_if_necessary(match.end()):
yield t
pos = match.end()