Rework the parser so we can use arbitrary start nodes of the syntax.

This also includes a rework for error recovery in the parser. This is now just possible for file_input parsing, which means for full files.
Includes also a refactoring of the tokenizer. No more do we have to add an additional newline, because it now works correctly (removes certain confusion.

test/test_evaluate/test_absolute_import.py

@@ -4,7 +4,7 @@ Python 2.X)
 """
 import jedi
 from jedi._compatibility import u
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from .. import helpers
 
 
@@ -12,7 +12,7 @@ def test_explicit_absolute_imports():
     """
     Detect modules with ``from __future__ import absolute_import``.
     """
-    parser = Parser(load_grammar(), u("from __future__ import absolute_import"), "test.py")
+    parser = ParserWithRecovery(load_grammar(), u("from __future__ import absolute_import"), "test.py")
     assert parser.module.has_explicit_absolute_import
 
 
@@ -20,7 +20,7 @@ def test_no_explicit_absolute_imports():
     """
      Detect modules without ``from __future__ import absolute_import``.
     """
-    parser = Parser(load_grammar(), u("1"), "test.py")
+    parser = ParserWithRecovery(load_grammar(), u("1"), "test.py")
     assert not parser.module.has_explicit_absolute_import
 
 
@@ -30,7 +30,7 @@ def test_dont_break_imports_without_namespaces():
     assume that all imports have non-``None`` namespaces.
     """
     src = u("from __future__ import absolute_import\nimport xyzzy")
-    parser = Parser(load_grammar(), src, "test.py")
+    parser = ParserWithRecovery(load_grammar(), src, "test.py")
     assert parser.module.has_explicit_absolute_import
 
 

test/test_evaluate/test_annotations.py

@@ -35,3 +35,20 @@ def test_simple_annotations():
     annot('')""")
 
     assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['int']
+
+
+@pytest.mark.skipif('sys.version_info[0] < 3')
+@pytest.mark.parametrize('reference', [
+    'assert 1',
+    '1',
+    'lambda: 3',
+    'def x(): pass',
+    '1, 2',
+    r'1\n'
+])
+def test_illegal_forward_references(reference):
+    source = """
+    def foo(bar: "%s"):
+        bar""" % reference
+
+    assert not jedi.Script(source).goto_definitions()

test/test_evaluate/test_buildout_detection.py

@@ -7,7 +7,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file,
                                     sys_path_with_modifications,
                                     _check_module)
 from jedi.evaluate import Evaluator
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 from ..helpers import cwd_at
 
@@ -37,7 +37,7 @@ def test_append_on_non_sys_path():
         d = Dummy()
         d.path.append('foo')"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert len(paths) > 0
     assert 'foo' not in paths
@@ -48,7 +48,7 @@ def test_path_from_invalid_sys_path_assignment():
         import sys
         sys.path = 'invalid'"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert len(paths) > 0
     assert 'invalid' not in paths
@@ -60,7 +60,7 @@ def test_sys_path_with_modifications():
         import os
     """))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
     paths = sys_path_with_modifications(Evaluator(grammar), p.module)
     assert '/tmp/.buildout/eggs/important_package.egg' in paths
@@ -83,7 +83,7 @@ def test_path_from_sys_path_assignment():
         if __name__ == '__main__':
             sys.exit(important_package.main())"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert 1 not in paths
     assert '/home/test/.buildout/eggs/important_package.egg' in paths

test/test_evaluate/test_sys_path.py

@@ -5,14 +5,14 @@ import sys
 import pytest
 
 from jedi._compatibility import unicode
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.evaluate import sys_path, Evaluator
 
 
 def test_paths_from_assignment():
     def paths(src):
         grammar = load_grammar()
-        stmt = Parser(grammar, unicode(src)).module.statements[0]
+        stmt = ParserWithRecovery(grammar, unicode(src)).module.statements[0]
         return set(sys_path._paths_from_assignment(Evaluator(grammar), stmt))
 
     assert paths('sys.path[0:0] = ["a"]') == set(['a'])