Merge with the linter branch (especially the changes of pep484.

test/test_parser/test_pgen2.py

@@ -9,6 +9,7 @@ test_grammar.py files from both Python 2 and Python 3.
 from textwrap import dedent
 
 
+from jedi._compatibility import unicode
 from jedi.parser import Parser, load_grammar, ParseError
 import pytest
 
@@ -18,7 +19,7 @@ from test.helpers import TestCase
 def parse(code, version='3.4'):
     code = dedent(code) + "\n\n"
     grammar = load_grammar(version=version)
-    return Parser(grammar, code, 'file_input').get_parsed_node()
+    return Parser(grammar, unicode(code), 'file_input').get_parsed_node()
 
 
 class TestDriver(TestCase):
@@ -45,8 +46,8 @@ class GrammarTest(TestCase):
 class TestMatrixMultiplication(GrammarTest):
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_matrix_multiplication_operator(self):
-        parse("a @ b")
-        parse("a @= b")
+        parse("a @ b", "3.5")
+        parse("a @= b", "3.5")
 
 
 class TestYieldFrom(GrammarTest):
@@ -61,7 +62,7 @@ class TestAsyncAwait(GrammarTest):
     def test_await_expr(self):
         parse("""async def foo():
                              await x
-                      """)
+                      """, "3.5")
 
         parse("""async def foo():
 
@@ -70,46 +71,56 @@ class TestAsyncAwait(GrammarTest):
             def foo(): pass
 
             await x
-        """)
+        """, "3.5")
 
-        parse("""async def foo(): return await a""")
+        parse("""async def foo(): return await a""", "3.5")
 
         parse("""def foo():
             def foo(): pass
             async def foo(): await x
-        """)
+        """, "3.5")
 
-        self.invalid_syntax("await x")
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_await_expr_invalid(self):
+        self.invalid_syntax("await x", version="3.5")
         self.invalid_syntax("""def foo():
-                                   await x""")
+                                   await x""", version="3.5")
 
         self.invalid_syntax("""def foo():
             def foo(): pass
             async def foo(): pass
             await x
-        """)
+        """, version="3.5")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
     def test_async_var(self):
-        parse("""async = 1""")
-        parse("""await = 1""")
-        parse("""def async(): pass""")
-
-    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
-    def test_async_with(self):
-        parse("""async def foo():
-                             async for a in b: pass""")
-
-        self.invalid_syntax("""def foo():
-                                   async for a in b: pass""")
+        parse("""async = 1""", "3.5")
+        parse("""await = 1""", "3.5")
+        parse("""def async(): pass""", "3.5")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_async_for(self):
         parse("""async def foo():
-                             async with a: pass""")
+                             async for a in b: pass""", "3.5")
 
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_for_invalid(self):
         self.invalid_syntax("""def foo():
-                                   async with a: pass""")
+                                   async for a in b: pass""", version="3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_async_with(self):
+        parse("""async def foo():
+                             async with a: pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_with_invalid(self):
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""", version="3.5")
 
 
 class TestRaiseChanges(GrammarTest):
@@ -232,6 +243,9 @@ class TestParserIdempotency(TestCase):
 
 
 class TestLiterals(GrammarTest):
+    # It's not possible to get the same result when using \xaa in Python 2/3,
+    # because it's treated differently.
+    @pytest.mark.skipif('sys.version_info[0] < 3')
     def test_multiline_bytes_literals(self):
         s = """
             md5test(b"\xaa" * 80,
@@ -250,6 +264,7 @@ class TestLiterals(GrammarTest):
             '''
         parse(s)
 
+    @pytest.mark.skipif('sys.version_info[0] < 3')
     def test_multiline_str_literals(self):
         s = """
             md5test("\xaa" * 80,