Merge pull request #691 from reinhrst/python35

python 3.5 support status

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+python: 3.5
 sudo: false
 env:
     - TOXENV=py26
@@ -6,6 +7,7 @@ env:
     - TOXENV=py32
     - TOXENV=py33
     - TOXENV=py34
+    - TOXENV=py35
     - TOXENV=pypy
     - TOXENV=cov
     - TOXENV=sith
@@ -14,8 +16,9 @@ matrix:
         - env: TOXENV=cov
         - env: TOXENV=sith
         - env: TOXENV=pypy
+
 install:
-    - pip install --quiet --use-mirrors tox
+    - pip install tox
 script:
     - tox
 after_script:

jedi/_compatibility.py

@@ -13,6 +13,7 @@ except ImportError:
 
 is_py3 = sys.version_info[0] >= 3
 is_py33 = is_py3 and sys.version_info.minor >= 3
+is_py35 = is_py3 and sys.version_info.minor >= 5
 is_py26 = not is_py3 and sys.version_info[1] < 7
 
 

jedi/api/keywords.py

@@ -1,7 +1,7 @@
 import pydoc
 import keyword
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from jedi import common
 from jedi.evaluate.helpers import FakeName
 from jedi.parser.tree import Leaf
@@ -12,7 +12,12 @@ except ImportError:
     import pydoc_topics
 
 if is_py3:
-    keys = keyword.kwlist
+    if is_py35:
+        # in python 3.5 async and await are not proper keywords, but for
+        # completion pursposes should as as though they are
+        keys = keyword.kwlist + ["async", "await"]
+    else:
+        keys = keyword.kwlist
 else:
     keys = keyword.kwlist + ['None', 'False', 'True']
 

jedi/evaluate/__init__.py

@@ -291,7 +291,7 @@ class Evaluator(object):
             types = set([element])  # TODO this is no real evaluation.
         elif element.type == 'expr_stmt':
             types = self.eval_statement(element)
-        elif element.type == 'power':
+        elif element.type in ('power', 'atom_expr'):
             types = self._eval_atom(element.children[0])
             for trailer in element.children[1:]:
                 if trailer == '**':  # has a power operation.

jedi/evaluate/analysis.py

@@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
     def check_hasattr(node, suite):
         try:
             assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos
-            assert node.type == 'power'
+            assert node.type in ('power', 'atom_expr')
             base = node.children[0]
             assert base.type == 'name' and base.value == 'hasattr'
             trailer = node.children[1]

jedi/evaluate/dynamic.py

@@ -89,7 +89,7 @@ def search_function_call(evaluator, func):
                     parent = parent.parent
 
                 trailer = None
-                if tree.is_node(parent, 'power'):
+                if tree.is_node(parent, 'power', 'atom_expr'):
                     for t in parent.children[1:]:
                         if t == '**':
                             break

jedi/evaluate/finder.py

@@ -448,7 +448,7 @@ def check_flow_information(evaluator, flow, search_name, pos):
 
 def _check_isinstance_type(evaluator, element, search_name):
     try:
-        assert element.type == 'power'
+        assert element.type in ('power', 'atom_expr')
         # this might be removed if we analyze and, etc
         assert len(element.children) == 2
         first, trailer = element.children

jedi/evaluate/helpers.py

@@ -88,7 +88,8 @@ def call_of_name(name, cut_own_trailer=False):
             return name
 
     power = par.parent
-    if tree.is_node(power, 'power') and power.children[0] != name \
+    if tree.is_node(power, 'power', 'atom_expr') \
+            and power.children[0] != name \
             and not (power.children[-2] == '**' and
                      name.start_pos > power.children[-1].start_pos):
         par = power

jedi/evaluate/iterable.py

@@ -528,7 +528,7 @@ def unpack_tuple_to_dict(evaluator, types, exprlist):
             analysis.add(evaluator, 'value-error-too-few-values', has_parts,
                          message="ValueError: need more than %s values to unpack" % n)
         return dct
-    elif exprlist.type == 'power':
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
         # Something like ``arr[x], var = ...``.
         # This is something that is not yet supported, would also be difficult
         # to write into a dict.

jedi/evaluate/param.py

@@ -47,7 +47,10 @@ class Arguments(tree.Base):
             for el in self.argument_node:
                 yield 0, el
         else:
-            if not tree.is_node(self.argument_node, 'arglist'):
+            if not (tree.is_node(self.argument_node, 'arglist') or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (tree.is_node(self.argument_node, 'argument') and
+                     self.argument_node.children[0] in ('*', '**')))):
                 yield 0, self.argument_node
                 return
 
@@ -57,6 +60,10 @@ class Arguments(tree.Base):
                     continue
                 elif child in ('*', '**'):
                     yield len(child.value), next(iterator)
+                elif tree.is_node(child, 'argument') and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
                 else:
                     yield 0, child
 

jedi/evaluate/sys_path.py

@@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt):
     for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
         try:
             assert operator in ['=', '+=']
-            assert tree.is_node(assignee, 'power') and len(assignee.children) > 1
+            assert tree.is_node(assignee, 'power', 'atom_expr') and \
+                len(assignee.children) > 1
             c = assignee.children
             assert c[0].type == 'name' and c[0].value == 'sys'
             trailer = c[1]
@@ -152,7 +153,7 @@ def _check_module(evaluator, module):
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
-            if tree.is_node(power, 'power'):
+            if tree.is_node(power, 'power', 'atom_expr'):
                 c = power.children
                 if isinstance(c[0], tree.Name) and c[0].value == 'sys' \
                         and tree.is_node(c[1], 'trailer'):

jedi/parser/grammar3.5.txt

@@ -15,15 +15,17 @@
 #       file_input is a module or sequence of commands read from an input file;
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
-single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 
-async_funcdef: ASYNC funcdef
+# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens
+# skipping python3.5 compatibility, in favour of 3.7 solution
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 
 parameters: '(' [typedargslist] ')'
@@ -69,7 +71,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -82,7 +84,9 @@ with_stmt: 'with' with_item (',' with_item)*  ':' suite
 with_item: test ['as' expr]
 # NB compile.c makes sure that the default except clause is last
 except_clause: 'except' [test ['as' NAME]]
-suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
 
 test: or_test ['if' or_test 'else' test] | lambdef
 test_nocond: or_test | lambdef_nocond
@@ -104,7 +108,7 @@ arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom_expr ['**' factor]
-atom_expr: [AWAIT] atom trailer*
+atom_expr: ['await'] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |

jedi/parser/token.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from token import *
 
 
@@ -24,6 +24,11 @@ else:
     tok_name[ELLIPSIS] = 'ELLIPSIS'
     N_TOKENS += 1
 
+if not is_py35:
+    ATEQUAL = N_TOKENS
+    tok_name[ATEQUAL] = 'ATEQUAL'
+    N_TOKENS += 1
+
 
 
 # Map from operator to number (since tokenize doesn't do this)
@@ -68,6 +73,7 @@ opmap_raw = """\
 %= PERCENTEQUAL
 &= AMPEREQUAL
 |= VBAREQUAL
+@= ATEQUAL
 ^= CIRCUMFLEXEQUAL
 <<= LEFTSHIFTEQUAL
 >>= RIGHTSHIFTEQUAL

jedi/parser/tokenize.py

@@ -76,7 +76,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
 # recognized as two instances of =).
 operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
-                 r"[+\-*/%&|^=<>]=?",
+                 r"[+\-*@/%&|^=<>]=?",
                  r"~")
 
 bracket = '[][(){}]'

jedi/parser/tree.py

@@ -1342,7 +1342,7 @@ def _defined_names(current):
             names += _defined_names(child)
     elif is_node(current, 'atom'):
         names += _defined_names(current.children[1])
-    elif is_node(current, 'power'):
+    elif is_node(current, 'power', 'atom_expr'):
         if current.children[-2] != '**':  # Just if there's no operation
             trailer = current.children[-1]
             if trailer.children[0] == '.':

test/test_parser/test_pgen2.py

@@ -46,8 +46,8 @@ class GrammarTest(TestCase):
 class TestMatrixMultiplication(GrammarTest):
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_matrix_multiplication_operator(self):
-        parse("a @ b")
+        parse("a @ b", "3.5")
-        parse("a @= b")
+        parse("a @= b", "3.5")
 
 
 class TestYieldFrom(GrammarTest):
@@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest):
     def test_await_expr(self):
         parse("""async def foo():
                              await x
-                      """)
+                      """, "3.5")
 
         parse("""async def foo():
 
@@ -71,46 +71,56 @@ class TestAsyncAwait(GrammarTest):
             def foo(): pass
 
             await x
-        """)
+        """, "3.5")
 
-        parse("""async def foo(): return await a""")
+        parse("""async def foo(): return await a""", "3.5")
 
         parse("""def foo():
             def foo(): pass
             async def foo(): await x
-        """)
+        """, "3.5")
 
-        self.invalid_syntax("await x")
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_await_expr_invalid(self):
+        self.invalid_syntax("await x", version="3.5")
         self.invalid_syntax("""def foo():
-                                   await x""")
+                                   await x""", version="3.5")
 
         self.invalid_syntax("""def foo():
             def foo(): pass
             async def foo(): pass
             await x
-        """)
+        """, version="3.5")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
     def test_async_var(self):
-        parse("""async = 1""")
+        parse("""async = 1""", "3.5")
-        parse("""await = 1""")
+        parse("""await = 1""", "3.5")
-        parse("""def async(): pass""")
+        parse("""def async(): pass""", "3.5")
-
-    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
-    def test_async_with(self):
-        parse("""async def foo():
-                             async for a in b: pass""")
-
-        self.invalid_syntax("""def foo():
-                                   async for a in b: pass""")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_async_for(self):
         parse("""async def foo():
-                             async with a: pass""")
+                             async for a in b: pass""", "3.5")
 
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_for_invalid(self):
         self.invalid_syntax("""def foo():
-                                   async with a: pass""")
+                                   async for a in b: pass""", version="3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_async_with(self):
+        parse("""async def foo():
+                             async with a: pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_with_invalid(self):
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""", version="3.5")
 
 
 class TestRaiseChanges(GrammarTest):

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26, py27, py32, py33, py34
+envlist = py26, py27, py32, py33, py34, py35
 [testenv]
 deps =
     pytest>=2.3.5
@@ -32,6 +32,10 @@ deps =
     typing
     {[testenv]deps}
 [testenv:py34]
+deps =
+    typing
+    {[testenv]deps}
+[testenv:py35]
 deps =
     typing
     {[testenv]deps}