VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-08-10 18:02:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix positioning for names that are interleaved with error tokens

Browse Source
This commit is contained in:
Dave Halter
2019-07-13 12:34:49 +02:00
parent 99dd4a84d4
commit 2b8544021f
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parso/python/tokenize.py
+6 -4
View File
@@ -629,19 +629,19 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
def _split_illegal_unicode_name(token, start_pos, prefix): def _split_illegal_unicode_name(token, start_pos, prefix):
def create_token(): def create_token():
return PythonToken( return PythonToken(ERRORTOKEN if is_illegal else NAME, found, pos, prefix)
ERRORTOKEN if is_illegal else NAME, found,
(start_pos[0], start_pos[1] + i), prefix
)
found = '' found = ''
is_illegal = False is_illegal = False
pos = start_pos
for i, char in enumerate(token): for i, char in enumerate(token):
if is_illegal: if is_illegal:
if is_identifier(char): if is_identifier(char):
yield create_token() yield create_token()
found = char found = char
is_illegal = False is_illegal = False
prefix = ''
pos = start_pos[0], start_pos[1] + i
else: else:
found += char found += char
else: else:
@@ -651,6 +651,8 @@ def _split_illegal_unicode_name(token, start_pos, prefix):
else: else:
if found: if found:
yield create_token() yield create_token()
prefix = ''
pos = start_pos[0], start_pos[1] + i
found = char found = char
is_illegal = True is_illegal = True