Tokenizer: It should not be possible to break out of backslashes on the next line, even if it was an error

This commit is contained in:
Dave Halter
2020-04-06 01:25:06 +02:00
parent b12dd498bb
commit 556ce86cde
3 changed files with 27 additions and 2 deletions

View File

@@ -549,7 +549,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first
or (initial == '.' and token != '.' and token != '...')): or (initial == '.' and token != '.' and token != '...')):
yield PythonToken(NUMBER, token, spos, prefix) yield PythonToken(NUMBER, token, spos, prefix)
elif pseudomatch.group(3) is not None: # ordinary name elif pseudomatch.group(3) is not None: # ordinary name
if token in always_break_tokens: if token in always_break_tokens and (fstring_stack or paren_level):
fstring_stack[:] = [] fstring_stack[:] = []
paren_level = 0 paren_level = 0
# We only want to dedent if the token is on a new line. # We only want to dedent if the token is on a new line.

View File

@@ -1585,3 +1585,27 @@ def test_byte_order_mark(differ):
''') ''')
differ.initialize('\n') differ.initialize('\n')
differ.parse(code3, parsers=2, expect_error_leaves=True) differ.parse(code3, parsers=2, expect_error_leaves=True)
def test_backslash_insertion(differ):
code1 = dedent('''
def f():
x
def g():
base = "" \\
""
return
''')
code2 = dedent('''
def f():
x
def g():
base = "" \\
def h():
""
return
''')
differ.initialize(code1)
differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
differ.parse(code1, parsers=2, copies=1)

View File

@@ -263,6 +263,7 @@ xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Pyth
(' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]), (' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP, ('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
NEWLINE, DEDENT, NAME, DEDENT]), NEWLINE, DEDENT, NAME, DEDENT]),
(' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]),
] ]
) )
def test_token_types(code, types): def test_token_types(code, types):
@@ -343,7 +344,7 @@ def test_form_feed():
def test_carriage_return(): def test_carriage_return():
lst = _get_token_list(' =\\\rclass') lst = _get_token_list(' =\\\rclass')
assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER] assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER]
def test_backslash(): def test_backslash():