mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-07 05:14:29 +08:00
Tokenizer: It should not be possible to break out of backslashes on the next line, even if it was an error
This commit is contained in:
@@ -549,7 +549,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first
|
|||||||
or (initial == '.' and token != '.' and token != '...')):
|
or (initial == '.' and token != '.' and token != '...')):
|
||||||
yield PythonToken(NUMBER, token, spos, prefix)
|
yield PythonToken(NUMBER, token, spos, prefix)
|
||||||
elif pseudomatch.group(3) is not None: # ordinary name
|
elif pseudomatch.group(3) is not None: # ordinary name
|
||||||
if token in always_break_tokens:
|
if token in always_break_tokens and (fstring_stack or paren_level):
|
||||||
fstring_stack[:] = []
|
fstring_stack[:] = []
|
||||||
paren_level = 0
|
paren_level = 0
|
||||||
# We only want to dedent if the token is on a new line.
|
# We only want to dedent if the token is on a new line.
|
||||||
|
|||||||
@@ -1585,3 +1585,27 @@ def test_byte_order_mark(differ):
|
|||||||
''')
|
''')
|
||||||
differ.initialize('\n')
|
differ.initialize('\n')
|
||||||
differ.parse(code3, parsers=2, expect_error_leaves=True)
|
differ.parse(code3, parsers=2, expect_error_leaves=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_backslash_insertion(differ):
|
||||||
|
code1 = dedent('''
|
||||||
|
def f():
|
||||||
|
x
|
||||||
|
def g():
|
||||||
|
base = "" \\
|
||||||
|
""
|
||||||
|
return
|
||||||
|
''')
|
||||||
|
code2 = dedent('''
|
||||||
|
def f():
|
||||||
|
x
|
||||||
|
def g():
|
||||||
|
base = "" \\
|
||||||
|
def h():
|
||||||
|
""
|
||||||
|
return
|
||||||
|
''')
|
||||||
|
|
||||||
|
differ.initialize(code1)
|
||||||
|
differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
|
||||||
|
differ.parse(code1, parsers=2, copies=1)
|
||||||
|
|||||||
@@ -263,6 +263,7 @@ xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Pyth
|
|||||||
(' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
|
(' )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
|
||||||
('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
|
('a\n b\n )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
|
||||||
NEWLINE, DEDENT, NAME, DEDENT]),
|
NEWLINE, DEDENT, NAME, DEDENT]),
|
||||||
|
(' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
def test_token_types(code, types):
|
def test_token_types(code, types):
|
||||||
@@ -343,7 +344,7 @@ def test_form_feed():
|
|||||||
|
|
||||||
def test_carriage_return():
|
def test_carriage_return():
|
||||||
lst = _get_token_list(' =\\\rclass')
|
lst = _get_token_list(' =\\\rclass')
|
||||||
assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER]
|
assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER]
|
||||||
|
|
||||||
|
|
||||||
def test_backslash():
|
def test_backslash():
|
||||||
|
|||||||
Reference in New Issue
Block a user