Rework the parser so we can use arbitrary start nodes of the syntax.

This also includes a rework for error recovery in the parser. This is now just possible for file_input parsing, which means for full files.
Includes also a refactoring of the tokenizer. No more do we have to add an additional newline, because it now works correctly (removes certain confusion.

jedi/evaluate/compiled/fake.py

@@ -8,7 +8,7 @@ import os
 import inspect
 
 from jedi._compatibility import is_py3, builtins, unicode
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser import tree as pt
 from jedi.evaluate.helpers import FakeName
 
@@ -31,7 +31,7 @@ def _load_faked_module(module):
             modules[module_name] = None
             return
         grammar = load_grammar('grammar3.4')
-        module = Parser(grammar, unicode(source), module_name).module
+        module = ParserWithRecovery(grammar, unicode(source), module_name).module
         modules[module_name] = module
 
         if module_name == 'builtins' and not is_py3:

jedi/evaluate/docstrings.py

@@ -20,7 +20,7 @@ from itertools import chain
 from textwrap import dedent
 
 from jedi.evaluate.cache import memoize_default
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.common import indent_block
 from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated
 
@@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module):
     # Take the default grammar here, if we load the Python 2.7 grammar here, it
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
-    p = Parser(load_grammar(), code % indent_block(string))
+    p = ParserWithRecovery(load_grammar(), code % indent_block(string))
     try:
         pseudo_cls = p.module.subscopes[0]
         # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,

jedi/evaluate/finder.py

@@ -487,8 +487,8 @@ def global_names_dict_generator(evaluator, scope, position):
     the current scope is function:
 
     >>> from jedi._compatibility import u, no_unicode_pprint
-    >>> from jedi.parser import Parser, load_grammar
-    >>> parser = Parser(load_grammar(), u('''
+    >>> from jedi.parser import ParserWithRecovery, load_grammar
+    >>> parser = ParserWithRecovery(load_grammar(), u('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None

jedi/evaluate/pep0484.py

@@ -19,9 +19,11 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from
 """
 
 from itertools import chain
+
 from jedi.parser import Parser, load_grammar
 from jedi.evaluate.cache import memoize_default
 from jedi.evaluate.compiled import CompiledObject
+from jedi import debug
 
 
 def _evaluate_for_annotation(evaluator, annotation):
@@ -30,13 +32,13 @@ def _evaluate_for_annotation(evaluator, annotation):
         for definition in evaluator.eval_element(annotation):
             if (isinstance(definition, CompiledObject) and
                     isinstance(definition.obj, str)):
-                p = Parser(load_grammar(), definition.obj)
-                try:
-                    element = p.module.children[0].children[0]
-                except (AttributeError, IndexError):
-                    continue
-                element.parent = annotation.parent
-                definitions |= evaluator.eval_element(element)
+                p = Parser(load_grammar(), definition.obj, start='expr')
+                element = p.get_parsed_node()
+                if element is None:
+                    debug.warning('Annotation not parsed: %s' % definition.obj)
+                else:
+                    element.parent = annotation.parent
+                    definitions |= evaluator.eval_element(element)
             else:
                 definitions.add(definition)
         return list(chain.from_iterable(

jedi/evaluate/sys_path.py

@@ -5,7 +5,7 @@ from jedi.evaluate.site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.evaluate.cache import memoize_default
 from jedi import debug
 from jedi import common
@@ -209,7 +209,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
             debug.dbg('Error trying to read buildout_script: %s', buildout_script)
             return
 
-        p = Parser(evaluator.grammar, source, buildout_script)
+        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
         cache.save_parser(buildout_script, p)
         return p.module