From e62a88b190799677fbdb4cc0bc8db10e75bf5537 Mon Sep 17 00:00:00 2001
From: Dave Halter <davidhalter88@gmail.com>
Date: Thu, 3 Aug 2017 21:48:04 +0200
Subject: [PATCH] Use a bit better warnings for tokenizer errors.

---
 parso/python/normalizer.py | 24 +++++++++++++++++++-----
 test/test_python_errors.py | 10 ++++++----
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/parso/python/normalizer.py b/parso/python/normalizer.py
index 39172a9..4d9e5c4 100644
--- a/parso/python/normalizer.py
+++ b/parso/python/normalizer.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import codecs
+import re
 from contextlib import contextmanager
 
 from parso.normalizer import Normalizer, NormalizerConfig, Issue
@@ -512,7 +513,16 @@ class ErrorFinder(Normalizer):
                     message = 'unindent does not match any outer indentation level'
                 self._add_indentation_error(message, spacing)
             else:
-                self._add_syntax_error('invalid syntax', leaf)
+                match = re.match('\\w{,2}("{1,3}|\'{1,3})', leaf.value)
+                if match is None:
+                    message = 'invalid syntax'
+                else:
+                    if len(match.group(1)) == 1:
+                        print(match.group(1))
+                        message = 'EOL while scanning string literal'
+                    else:
+                        message = 'EOF while scanning triple-quoted string literal'
+                self._add_syntax_error(message, leaf, overwrite=True)
         elif leaf.type == 'name':
             if leaf.value == '__debug__' and leaf.is_definition():
                 if self._version < (3, 0):
@@ -672,13 +682,17 @@ class ErrorFinder(Normalizer):
     def _add_indentation_error(self, message, spacing):
         self._add_error(903, "IndentationError: " + message, spacing)
 
-    def _add_syntax_error(self, message, node):
-        self._add_error(901, "SyntaxError: " + message, node)
+    def _add_syntax_error(self, message, node, overwrite=False):
+        self._add_error(901, "SyntaxError: " + message, node, overwrite)
 
-    def _add_error(self, code, message, node):
+    def _add_error(self, code, message, node, overwrite=False):
         # Check if the issues are on the same line.
         line = node.start_pos[0]
-        self._error_dict.setdefault(line, (code, message, node))
+        args = (code, message, node)
+        if overwrite:
+            self._error_dict[line] = args
+        else:
+            self._error_dict.setdefault(line, args)
 
     def finalize(self):
         self._context.finalize()
diff --git a/test/test_python_errors.py b/test/test_python_errors.py
index 91268df..e6130b9 100644
--- a/test/test_python_errors.py
+++ b/test/test_python_errors.py
@@ -106,6 +106,12 @@ FAILING_EXAMPLES = [
     r'b"\x"',
     r'b"\"',
 
+    # Parser/tokenize.c
+    r'"""',
+    r'"',
+    r"'''",
+    r"'",
+
     # SyntaxErrors from Python/symtable.c
     'def f(x, x): pass',
     'nonlocal a',
@@ -319,10 +325,6 @@ def _get_actual_exception(code):
     if sys.version_info[:2] == (2, 6) and wanted == 'SyntaxError: unexpected EOF while parsing':
         wanted = 'SyntaxError: invalid syntax'
 
-    if wanted == 'SyntaxError: EOL while scanning string literal':
-        # TODO This is not what we want in the future. Remove this.
-        wanted = 'SyntaxError: invalid syntax'
-
     if wanted == 'SyntaxError: non-keyword arg after keyword arg':
         # The python 3.5+ way, a bit nicer.
         wanted = 'SyntaxError: positional argument follows keyword argument'