diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 4e4f62b..40ad694 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -487,14 +487,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)): new_line = False if paren_level == 0 and not fstring_stack: i = 0 + indent_start = start while line[i] == '\f': i += 1 # TODO don't we need to change spos as well? - start -= 1 - if start > indents[-1]: + indent_start -= 1 + if indent_start > indents[-1]: yield PythonToken(INDENT, '', spos, '') - indents.append(start) - for t in dedent_if_necessary(start): + indents.append(indent_start) + for t in dedent_if_necessary(indent_start): yield t if fstring_stack: diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index a5eb4b9..4d68b9c 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -1016,20 +1016,20 @@ def test_special_no_newline_ending(differ): def test_random_character_insertion(differ): code1 = dedent('''\ - def create(self): - 1 - if self.path is not None: - return - # 3 - # 4 + def create(self): + 1 + if self.path is not None: + return + # 3 + # 4 ''') code2 = dedent('''\ - def create(self): - 1 - if 2: - x return - # 3 - # 4 + def create(self): + 1 + if 2: + x return + # 3 + # 4 ''') differ.initialize(code1) differ.parse(code2, copies=1, parsers=3, expect_error_leaves=True) diff --git a/test/test_tokenize.py b/test/test_tokenize.py index 397adb4..a3e1a88 100644 --- a/test/test_tokenize.py +++ b/test/test_tokenize.py @@ -299,3 +299,11 @@ def test_brackets_no_indentation(): """) lst = _get_token_list(code) assert [t.type for t in lst] == [OP, NEWLINE, OP, OP, NEWLINE, ENDMARKER] + + +def test_form_feed(): + error_token, endmarker = _get_token_list(dedent('''\ + \f"""''')) + assert error_token.prefix == '\f' + assert error_token.string == '"""' + assert endmarker.prefix == ''