From 7ae1efe5c751d6d845eae3e821e4d9dba446a39e Mon Sep 17 00:00:00 2001
From: Dave Halter <davidhalter88@gmail.com>
Date: Sun, 13 Jan 2019 23:16:09 +0100
Subject: [PATCH] Fix tokenizer: Form feeds and multiline docstrings didn't
 work together

---
 parso/python/tokenize.py |  9 +++++----
 test/test_diff_parser.py | 24 ++++++++++++------------
 test/test_tokenize.py    |  8 ++++++++
 3 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py
index 4e4f62b..40ad694 100644
--- a/parso/python/tokenize.py
+++ b/parso/python/tokenize.py
@@ -487,14 +487,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 new_line = False
                 if paren_level == 0 and not fstring_stack:
                     i = 0
+                    indent_start = start
                     while line[i] == '\f':
                         i += 1
                         # TODO don't we need to change spos as well?
-                        start -= 1
-                    if start > indents[-1]:
+                        indent_start -= 1
+                    if indent_start > indents[-1]:
                         yield PythonToken(INDENT, '', spos, '')
-                        indents.append(start)
-                    for t in dedent_if_necessary(start):
+                        indents.append(indent_start)
+                    for t in dedent_if_necessary(indent_start):
                         yield t
 
             if fstring_stack:
diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py
index a5eb4b9..4d68b9c 100644
--- a/test/test_diff_parser.py
+++ b/test/test_diff_parser.py
@@ -1016,20 +1016,20 @@ def test_special_no_newline_ending(differ):
 
 def test_random_character_insertion(differ):
     code1 = dedent('''\
-            def create(self):
-                1
-                if self.path is not None:
-                    return
-                # 3
-                # 4
+        def create(self):
+            1
+            if self.path is not None:
+                return
+            # 3
+            # 4
         ''')
     code2 = dedent('''\
-            def create(self):
-                1
-                if 2:
-             x       return
-                # 3
-                # 4
+        def create(self):
+            1
+            if 2:
+         x       return
+            # 3
+            # 4
         ''')
     differ.initialize(code1)
     differ.parse(code2, copies=1, parsers=3, expect_error_leaves=True)
diff --git a/test/test_tokenize.py b/test/test_tokenize.py
index 397adb4..a3e1a88 100644
--- a/test/test_tokenize.py
+++ b/test/test_tokenize.py
@@ -299,3 +299,11 @@ def test_brackets_no_indentation():
         """)
     lst = _get_token_list(code)
     assert [t.type for t in lst] == [OP, NEWLINE, OP, OP, NEWLINE, ENDMARKER]
+
+
+def test_form_feed():
+    error_token, endmarker = _get_token_list(dedent('''\
+        \f"""'''))
+    assert error_token.prefix == '\f'
+    assert error_token.string == '"""'
+    assert endmarker.prefix == ''