Fix tokenizer: Dedents should only happen after newlines

parso/python/tokenize.py

@@ -554,13 +554,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 if token in always_break_tokens:
                     fstring_stack[:] = []
                     paren_level = 0
-                    while True:
+                    # We only want to dedent if the token is on a new line.
-                        indent = indents.pop()
+                    if re.match(r'[ \f\t]*$', line[:start]):
-                        if indent > start:
+                        while True:
-                            yield PythonToken(DEDENT, '', spos, '')
+                            indent = indents.pop()
-                        else:
+                            if indent > start:
-                            indents.append(indent)
+                                yield PythonToken(DEDENT, '', spos, '')
-                            break
+                            else:
+                                indents.append(indent)
+                                break
                 yield PythonToken(NAME, token, spos, prefix)
             elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'):  # continued stmt
                 additional_prefix += prefix + line[start:]

test/test_diff_parser.py

@@ -722,3 +722,26 @@ def test_paren_in_strange_position(differ):
     differ.initialize(code1)
     differ.parse(code2, parsers=1, copies=2, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=1)
+
+
+def insert_line_into_code(code, index, line):
+    lines = split_lines(code, keepends=True)
+    lines.insert(index, line)
+    return ''.join(lines)
+
+
+def test_paren_before_docstring(differ):
+    code1 = dedent('''\
+        # comment
+        """
+        The
+        """
+        from parso import tree
+        from parso import python
+        ''')
+
+    code2 = insert_line_into_code(code1, 1, ' ' * 16 + 'raise InternalParseError(\n')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=2, copies=1)

test/test_tokenize.py

@@ -246,3 +246,25 @@ def test_error_string():
     assert t1.string == '"'
     assert endmarker.prefix == '\n'
     assert endmarker.string == ''
+
+
+def test_indent_error_recovery():
+    code = dedent("""\
+                        str(
+        from x import a
+        def
+        """)
+    lst = _get_token_list(code)
+    expected = [
+        # `str(`
+        INDENT, NAME, OP,
+        # `from parso`
+        NAME, NAME,
+        # `import a` on same line as the previous from parso
+        NAME, NAME, NEWLINE,
+        # Dedent happens, because there's an import now and the import
+        # statement "breaks" out of the opening paren on the first line.
+        DEDENT,
+        # `b`
+        NAME, NEWLINE, ENDMARKER]
+    assert [t.type for t in lst] == expected