From 57320af6eb9ea6522ada667eebb9dd2a5cb8be2e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 9 Jan 2019 00:55:54 +0100 Subject: [PATCH] Fix another tokenizer issue --- parso/python/tokenize.py | 5 +++-- test/test_diff_parser.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 2b72cb4..3a59643 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -457,10 +457,11 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)): pseudomatch = pseudo_token.match(line, pos) if not pseudomatch: # scan for tokens match = whitespace.match(line, pos) + if pos == 0: + for t in dedent_if_necessary(match.end()): + yield t pos = match.end() new_line = False - for t in dedent_if_necessary(pos): - yield t yield PythonToken( ERRORTOKEN, line[pos], (lnum, pos), additional_prefix + match.group(0) diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 3c5e2d1..4e80bd2 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -961,4 +961,8 @@ def test_random_unicode_characters(differ): differ.parse('\r\r', parsers=1) differ.parse("˟Ę\x05À\r rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True) differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1) + s = ' if not (self, "_fi\x02\x0e\x08\n\nle"):' + differ.parse(s, parsers=1, expect_error_leaves=True) + differ.parse('') + differ.parse(s + '\n', parsers=1, expect_error_leaves=True) differ.parse('')