VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-06 21:04:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix tokenizer: Carriage returns after backslashes were not properly handled

Browse Source
This commit is contained in:
Dave Halter
2019-01-14 01:49:09 +01:00
parent d3d28480ed
commit d7171ae927
2 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parso/python/tokenize.py
View File

@@ -573,7 +573,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
indents.append(indent) indents.append(indent)
break break
yield PythonToken(NAME, token, spos, prefix) yield PythonToken(NAME, token, spos, prefix)
elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\r'): # continued stmt elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\\r'): # continued stmt
additional_prefix += prefix + line[start:] additional_prefix += prefix + line[start:]
break break
else: else:

5
test/test_tokenize.py
View File

@@ -307,3 +307,8 @@ def test_form_feed():
assert error_token.prefix == '\f' assert error_token.prefix == '\f'
assert error_token.string == '"""' assert error_token.string == '"""'
assert endmarker.prefix == '' assert endmarker.prefix == ''
def test_carriage_return():
lst = _get_token_list(' =\\\rclass')
assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER]