VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-07 05:14:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix another tokenizer issue

Browse Source
This commit is contained in:
Dave Halter
2019-01-09 00:55:54 +01:00
parent 574e1c63e8
commit 57320af6eb
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
parso/python/tokenize.py
View File

@@ -457,10 +457,11 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
pseudomatch = pseudo_token.match(line, pos) pseudomatch = pseudo_token.match(line, pos)
if not pseudomatch: # scan for tokens if not pseudomatch: # scan for tokens
match = whitespace.match(line, pos) match = whitespace.match(line, pos)
if pos == 0:
for t in dedent_if_necessary(match.end()):
yield t
pos = match.end() pos = match.end()
new_line = False new_line = False
for t in dedent_if_necessary(pos):
yield t
yield PythonToken( yield PythonToken(
ERRORTOKEN, line[pos], (lnum, pos), ERRORTOKEN, line[pos], (lnum, pos),
additional_prefix + match.group(0) additional_prefix + match.group(0)

4
test/test_diff_parser.py
View File

@@ -961,4 +961,8 @@ def test_random_unicode_characters(differ):
differ.parse('\r\r', parsers=1) differ.parse('\r\r', parsers=1)
differ.parse("˟Ę\x05À\r rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True) differ.parse("˟Ę\x05À\r rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True)
differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1) differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1)
s = ' if not (self, "_fi\x02\x0e\x08\n\nle"):'
differ.parse(s, parsers=1, expect_error_leaves=True)
differ.parse('')
differ.parse(s + '\n', parsers=1, expect_error_leaves=True)
differ.parse('') differ.parse('')