mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-06 12:54:29 +08:00
Tokenizer: It should not be possible to break out of backslashes on the next line, even if it was an error
This commit is contained in:
@@ -549,7 +549,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first
|
||||
or (initial == '.' and token != '.' and token != '...')):
|
||||
yield PythonToken(NUMBER, token, spos, prefix)
|
||||
elif pseudomatch.group(3) is not None: # ordinary name
|
||||
if token in always_break_tokens:
|
||||
if token in always_break_tokens and (fstring_stack or paren_level):
|
||||
fstring_stack[:] = []
|
||||
paren_level = 0
|
||||
# We only want to dedent if the token is on a new line.
|
||||
|
||||
@@ -1585,3 +1585,27 @@ def test_byte_order_mark(differ):
|
||||
''')
|
||||
differ.initialize('\n')
|
||||
differ.parse(code3, parsers=2, expect_error_leaves=True)
|
||||
|
||||
|
||||
def test_backslash_insertion(differ):
|
||||
code1 = dedent('''
|
||||
def f():
|
||||
x
|
||||
def g():
|
||||
base = "" \\
|
||||
""
|
||||
return
|
||||
''')
|
||||
code2 = dedent('''
|
||||
def f():
|
||||
x
|
||||
def g():
|
||||
base = "" \\
|
||||
def h():
|
||||
""
|
||||
return
|
||||
''')
|
||||
|
||||
differ.initialize(code1)
|
||||
differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
|
||||
differ.parse(code1, parsers=2, copies=1)
|
||||
|
||||
@@ -263,6 +263,7 @@ xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Pyth
|
||||
(' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
|
||||
('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
|
||||
NEWLINE, DEDENT, NAME, DEDENT]),
|
||||
(' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]),
|
||||
]
|
||||
)
|
||||
def test_token_types(code, types):
|
||||
@@ -343,7 +344,7 @@ def test_form_feed():
|
||||
|
||||
def test_carriage_return():
|
||||
lst = _get_token_list(' =\\\rclass')
|
||||
assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER]
|
||||
assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER]
|
||||
|
||||
|
||||
def test_backslash():
|
||||
|
||||
Reference in New Issue
Block a user