Fix tokenizer: Dedents before error tokens are properly done, now.

test/test_diff_parser.py

@@ -935,5 +935,20 @@ def test_with_and_funcdef_in_call(differ, prefix):
     code2 = insert_line_into_code(code1, 3, 'def y(self, args):\n')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=3, copies=0, expect_error_leaves=True)
-    differ.parse(code1, parsers=1, copies=0)
+    differ.parse(code2, parsers=3, expect_error_leaves=True)
+    differ.parse(code1, parsers=1)
+
+
+def test_wrong_backslash(differ):
+    code1 = dedent('''\
+        def y():
+            1
+            for x in y:
+                continue
+        ''')
+
+    code2 = insert_line_into_code(code1, 3, '\.whl$\n')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=2, expect_error_leaves=True)
+    differ.parse(code1, parsers=1, copies=1)

test/test_tokenize.py

@@ -197,11 +197,12 @@ def test_ur_literals():
 
 
 def test_error_literal():
-    error_token, endmarker = _get_token_list('"\n')
+    error_token, newline, endmarker = _get_token_list('"\n')
     assert error_token.type == ERRORTOKEN
     assert error_token.string == '"'
+    assert newline.type == NEWLINE
     assert endmarker.type == ENDMARKER
-    assert endmarker.prefix == '\n'
+    assert endmarker.prefix == ''
 
     bracket, error_token, endmarker = _get_token_list('( """')
     assert error_token.type == ERRORTOKEN
@@ -240,11 +241,12 @@ def test_indentation(code, types):
 
 
 def test_error_string():
-    t1, endmarker = _get_token_list(' "\n')
+    t1, newline, endmarker = _get_token_list(' "\n')
     assert t1.type == ERRORTOKEN
     assert t1.prefix == ' '
     assert t1.string == '"'
-    assert endmarker.prefix == '\n'
+    assert newline.type == NEWLINE
+    assert endmarker.prefix == ''
     assert endmarker.string == ''
 
 
@@ -268,3 +270,18 @@ def test_indent_error_recovery():
         # `b`
         NAME, NEWLINE, ENDMARKER]
     assert [t.type for t in lst] == expected
+
+
+def test_error_token_after_dedent():
+    code = dedent("""\
+        class C:
+            pass
+        $foo
+        """)
+    lst = _get_token_list(code)
+    expected = [
+        NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE, DEDENT,
+        # $foo\n
+        ERRORTOKEN, NAME, NEWLINE, ENDMARKER
+    ]
+    assert [t.type for t in lst] == expected