Rework the parser so we can use arbitrary start nodes of the syntax.

This also includes a rework for error recovery in the parser. This is now just possible for file_input parsing, which means for full files.
Includes also a refactoring of the tokenizer. No more do we have to add an additional newline, because it now works correctly (removes certain confusion.

test/test_parser/test_get_code.py

@@ -3,7 +3,7 @@ import difflib
 import pytest
 
 from jedi._compatibility import u
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 code_basic_features = u('''
 """A mod docstring"""
@@ -44,7 +44,7 @@ def diff_code_assert(a, b, n=4):
 def test_basic_parsing():
     """Validate the parsing features"""
 
-    prs = Parser(load_grammar(), code_basic_features)
+    prs = ParserWithRecovery(load_grammar(), code_basic_features)
     diff_code_assert(
         code_basic_features,
         prs.module.get_code()
@@ -53,7 +53,7 @@ def test_basic_parsing():
 
 def test_operators():
     src = u('5  * 3')
-    prs = Parser(load_grammar(), src)
+    prs = ParserWithRecovery(load_grammar(), src)
     diff_code_assert(src, prs.module.get_code())
 
 
@@ -82,7 +82,7 @@ def method_with_docstring():
     """class docstr"""
     pass
 ''')
-    assert Parser(load_grammar(), s).module.get_code() == s
+    assert ParserWithRecovery(load_grammar(), s).module.get_code() == s
 
 
 def test_end_newlines():
@@ -92,7 +92,7 @@ def test_end_newlines():
     line the parser needs.
     """
     def test(source, end_pos):
-        module = Parser(load_grammar(), u(source)).module
+        module = ParserWithRecovery(load_grammar(), u(source)).module
         assert module.get_code() == source
         assert module.end_pos == end_pos