Add syntax errors to the parser.

jedi/parser/__init__.py

@@ -22,7 +22,7 @@ from jedi.parser import tree as pt
 from jedi.parser import tokenize
 from jedi.parser import token
 from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER,
-                               STRING, OP)
+                               STRING, OP, ERRORTOKEN)
 from jedi.parser.pgen2.pgen import generate_grammar
 from jedi.parser.pgen2.parse import PgenParser
 
@@ -74,6 +74,12 @@ class ErrorStatement(object):
         return first_type
 
 
+class ParserSyntaxError(object):
+    def __init__(self, message, position):
+        self.message = message
+        self.position = position
+
+
 class Parser(object):
     """
     This class is used to parse a Python file, it then divides them into a
@@ -115,6 +121,8 @@ class Parser(object):
             'lambdef_nocond': pt.Lambda,
         }
 
+        self.syntax_errors = []
+
         self._global_names = []
         self._omit_dedent_list = []
         self._indent_counter = 0
@@ -321,11 +329,17 @@ class Parser(object):
                 self._indent_counter -= 1
             elif typ == INDENT:
                 self._indent_counter += 1
+            elif typ == ERRORTOKEN:
+                self._add_syntax_error('Strange token', start_pos)
+                continue
 
             if typ == OP:
                 typ = token.opmap[value]
             yield typ, value, prefix, start_pos
 
+    def _add_syntax_error(self, message, position):
+        self.syntax_errors.append(ParserSyntaxError(message, position))
+
     def __repr__(self):
         return "<%s: %s>" % (type(self).__name__, self.module)
 

jedi/parser/pgen2/parse.py

@@ -117,12 +117,7 @@ class PgenParser(object):
     def addtoken(self, type, value, prefix, start_pos):
         """Add a token; return True iff this is the end of the program."""
         # Map from token to label
-        try:
+        ilabel = self.classify(type, value, start_pos)
-            ilabel = self.classify(type, value, start_pos)
-        except ParseError:
-            # Currently we ignore tokens like `?`.
-            print('invalid token', tokenize.tok_name[type], repr(value))
-            return
 
         # Loop until the token is shifted; may raise exceptions
         while True:
@@ -171,14 +166,12 @@ class PgenParser(object):
     def classify(self, type, value, start_pos):
         """Turn a token into a label.  (Internal)"""
         if type == tokenize.NAME:
-            # Check for reserved words
+            # Check for reserved words (keywords)
-            ilabel = self.grammar.keywords.get(value)
+            try:
-            if ilabel is not None:
+                return self.grammar.keywords[value]
-                return ilabel
+            except KeyError:
-        ilabel = self.grammar.tokens.get(type)
+                pass
-        if ilabel is None:
+        return self.grammar.tokens[type]
-            raise ParseError("bad token", type, value, start_pos)
-        return ilabel
 
     def shift(self, type, value, newstate, prefix, start_pos):
         """Shift a token.  (Internal)"""