Fix a nasty issue in the tokenizer. Fixes #836.

At the same time there was a related issue of not cleaning up newlines properly.

test/test_parser/test_diff_parser.py

@@ -206,6 +206,11 @@ def test_open_parentheses(differ):
     differ.parse('isinstance()\n' + func, parsers=2, copies=1)
 
 
+def test_open_parentheses_at_end(differ):
+    code = "a['"
+    differ.initialize(code)
+    differ.parse(code, parsers=1, expect_error_leaves=True)
+
 def test_backslash(differ):
     src = dedent(r"""
     a = 1\