From 1f4c95918f642693a26cdbd3a910ff4c0f3cf954 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:14:56 +0100 Subject: [PATCH 01/17] Add @= ATEQUAL token --- jedi/parser/token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/parser/token.py b/jedi/parser/token.py index e9ab3a62..6890c8ef 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -68,6 +68,7 @@ opmap_raw = """\ %= PERCENTEQUAL &= AMPEREQUAL |= VBAREQUAL +@= ATEQUAL ^= CIRCUMFLEXEQUAL <<= LEFTSHIFTEQUAL >>= RIGHTSHIFTEQUAL From bc0486f7231ca2a7f0a87eae94b18b8623f7bd97 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:21:26 +0100 Subject: [PATCH 02/17] python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 3c5b1fdf..de78f2f5 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -291,7 +291,7 @@ class Evaluator(object): types = set([element]) # TODO this is no real evaluation. elif element.type == 'expr_stmt': types = self.eval_statement(element) - elif element.type == 'power': + elif element.type == 'power' or element.type == 'atom_expr': types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. From 3fb5fe8c77d79e07b9adf70bd6277cf320cf8ce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:22:17 +0100 Subject: [PATCH 03/17] allow empty bodies for better autocompletion --- jedi/parser/grammar3.5.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index 99fcea02..a647880f 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -82,7 +82,9 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows +# classes and functions to be empty, which is beneficial for autocompletion. +suite: simple_stmt | NEWLINE INDENT stmt* DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond From 241abe9cf399355af03eb5090376793f1b61a420 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:42:53 +0100 Subject: [PATCH 04/17] python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/dynamic.py | 3 ++- jedi/evaluate/helpers.py | 3 ++- jedi/evaluate/iterable.py | 2 +- jedi/parser/tree.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index ad65d6db..6ca429fe 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -89,7 +89,8 @@ def search_function_call(evaluator, func): parent = parent.parent trailer = None - if tree.is_node(parent, 'power'): + if tree.is_node(parent, 'power') or \ + tree.is_node(parent, 'atom_expr'): for t in parent.children[1:]: if t == '**': break diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 49edadbf..86eec876 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -88,7 +88,8 @@ def call_of_name(name, cut_own_trailer=False): return name power = par.parent - if tree.is_node(power, 'power') and power.children[0] != name \ + if (tree.is_node(power, 'power') or tree.is_node(power, 'atom_expr')) \ + and power.children[0] != name \ and not (power.children[-2] == '**' and name.start_pos > power.children[-1].start_pos): par = power diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 0c6c20ce..21d4211d 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -528,7 +528,7 @@ def unpack_tuple_to_dict(evaluator, types, exprlist): analysis.add(evaluator, 'value-error-too-few-values', has_parts, message="ValueError: need more than %s values to unpack" % n) return dct - elif exprlist.type == 'power': + elif exprlist.type == 'power' or exprlist.type == 'atom_expr': # Something like ``arr[x], var = ...``. # This is something that is not yet supported, would also be difficult # to write into a dict. diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 7824a634..867d8cde 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1342,7 +1342,7 @@ def _defined_names(current): names += _defined_names(child) elif is_node(current, 'atom'): names += _defined_names(current.children[1]) - elif is_node(current, 'power'): + elif is_node(current, 'power') or is_node(current, 'atom_expr'): if current.children[-2] != '**': # Just if there's no operation trailer = current.children[-1] if trailer.children[0] == '.': From 3b0dcb3fcba8e04843f99161de0ff0081eb25354 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:47:01 +0100 Subject: [PATCH 05/17] move file_input to top of file, as mentioned in 19acdd32b72ec4189d98a9a6fb3b0db1cf8fb471 --- jedi/parser/grammar3.5.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index a647880f..9a27f1ec 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -15,8 +15,8 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE From de98cda2d774b038fa7f1a94c24cc4e2027ea2aa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 12:06:16 +0100 Subject: [PATCH 06/17] python3.5 uses 'argument' node type, not 'arglist' for * and ** arguments --- jedi/evaluate/param.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 3dc515a1..ebc173f9 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -47,7 +47,10 @@ class Arguments(tree.Base): for el in self.argument_node: yield 0, el else: - if not tree.is_node(self.argument_node, 'arglist'): + if not (tree.is_node(self.argument_node, 'arglist') or ( + # in python 3.5 **arg is an argument, not arglist + (tree.is_node(self.argument_node, 'argument') and + self.argument_node.children[0] in ('*', '**')))): yield 0, self.argument_node return From 0ed149070ad2cabd0e79245b567ce80f2ddf6304 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 17:13:25 +0100 Subject: [PATCH 07/17] add python 3.5 '@' operator to tokenizer --- jedi/parser/tokenize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 194a0967..46f9ef76 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -76,7 +76,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') # recognized as two instances of =). operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", - r"[+\-*/%&|^=<>]=?", + r"[+\-*@/%&|^=<>]=?", r"~") bracket = '[][(){}]' From a09611197be718ebbd7e0b72fe1c1246c77718fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:17:31 +0100 Subject: [PATCH 08/17] add ATEQUAL token for python < 3.5 --- jedi/_compatibility.py | 1 + jedi/parser/token.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 60c9dcb5..13f55134 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -13,6 +13,7 @@ except ImportError: is_py3 = sys.version_info[0] >= 3 is_py33 = is_py3 and sys.version_info.minor >= 3 +is_py35 = is_py3 and sys.version_info.minor >= 5 is_py26 = not is_py3 and sys.version_info[1] < 7 diff --git a/jedi/parser/token.py b/jedi/parser/token.py index 6890c8ef..0cb846da 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from token import * @@ -24,6 +24,11 @@ else: tok_name[ELLIPSIS] = 'ELLIPSIS' N_TOKENS += 1 +if not is_py35: + ATEQUAL = N_TOKENS + tok_name[ATEQUAL] = 'ATEQUAL' + N_TOKENS += 1 + # Map from operator to number (since tokenize doesn't do this) From 8819b2133a45731668910011968d686aa6fce117 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:23:24 +0100 Subject: [PATCH 09/17] further fix for *-arguments in arglist --- jedi/evaluate/param.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index ebc173f9..997e0799 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -60,6 +60,10 @@ class Arguments(tree.Base): continue elif child in ('*', '**'): yield len(child.value), next(iterator) + elif tree.is_node(child, 'argument') and \ + child.children[0] in ('*', '**'): + assert len(child.children) == 2 + yield len(child.children[0].value), child.children[1] else: yield 0, child From bf5acb4c7acda60325d252160f430de94cebb949 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:47:48 +0100 Subject: [PATCH 10/17] once more: python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/analysis.py | 2 +- jedi/evaluate/finder.py | 2 +- jedi/evaluate/sys_path.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index 07e7c69a..87c961dd 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): def check_hasattr(node, suite): try: assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type == 'power' + assert node.type == 'power' or node.type == 'atom_expr' base = node.children[0] assert base.type == 'name' and base.value == 'hasattr' trailer = node.children[1] diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 63123acc..db9561c3 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -448,7 +448,7 @@ def check_flow_information(evaluator, flow, search_name, pos): def _check_isinstance_type(evaluator, element, search_name): try: - assert element.type == 'power' + assert element.type == 'power' or element.type == 'atom_expr' # this might be removed if we analyze and, etc assert len(element.children) == 2 first, trailer = element.children diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index e838177a..fabb2c1d 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt): for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]): try: assert operator in ['=', '+='] - assert tree.is_node(assignee, 'power') and len(assignee.children) > 1 + assert tree.is_node(assignee, 'power', 'atom_expr') and \ + len(assignee.children) > 1 c = assignee.children assert c[0].type == 'name' and c[0].value == 'sys' trailer = c[1] @@ -152,7 +153,7 @@ def _check_module(evaluator, module): def get_sys_path_powers(names): for name in names: power = name.parent.parent - if tree.is_node(power, 'power'): + if tree.is_node(power, 'power', 'atom_expr'): c = power.children if isinstance(c[0], tree.Name) and c[0].value == 'sys' \ and tree.is_node(c[1], 'trailer'): From 65187930bd9cf8f54233635a99ee7ca04a05eeff Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 21:07:18 +0100 Subject: [PATCH 11/17] Update tokenizer to adhere to PEP492 magic --- jedi/parser/tokenize.py | 55 ++++++++++++++++++++++++++++++++-- test/test_parser/test_pgen2.py | 37 +++++++++++++---------- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 46f9ef76..301d9ff6 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -16,7 +16,7 @@ import re from io import StringIO from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 cookie_re = re.compile("coding[:=]\s*([-\w.]+)") @@ -151,7 +151,58 @@ def source_tokens(source): """Generate tokens from a the source code (string).""" source = source readline = StringIO(source).readline - return generate_tokens(readline) + tokenizer = generate_tokens(readline) + if is_py35: + tokenizer = fix_async_await_tokenizer(tokenizer) + return tokenizer + + +def fix_async_await_tokenizer(tokenizer): + # We need to recognise async functions as per + # https://www.python.org/dev/peps/pep-0492/#transition-plan + # so we need a three item look-ahead + from jedi.parser.token import ASYNC, AWAIT + assert is_py35 + try: + first = next(tokenizer) + except StopIteration: + return + + try: + second = next(tokenizer) + except StopIteration: + yield first + return + + defs_stack = [] + indent = 0 + for item in tokenizer: + if first[0] == INDENT: + indent += 1 + + if first[0] == DEDENT: + indent -= 1 + + if first[0] == NEWLINE and second[0] != INDENT: + while defs_stack and indent <= defs_stack[-1][0]: + defs_stack.pop() + + if second[0] == NAME and second[1] == "def" and item[0] == NAME: + if first[0] == NAME and first[1] == "async": + defs_stack.append((indent, True)) + else: + defs_stack.append((indent, False)) + in_async_def = defs_stack and defs_stack[-1][1] + if in_async_def and first[0] == NAME and first[1] == "async": + yield (ASYNC, ) + first[1:] + elif in_async_def and first[0] == NAME and first[1] == "await": + yield (AWAIT, ) + first[1:] + else: + yield first + first = second + second = item + yield first + yield second def generate_tokens(readline): diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 6d51ae85..576d338f 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b") - parse("a @= b") + parse("a @ b", "3.5") + parse("a @= b", "3.5") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """) + """, "3.5") parse("""async def foo(): @@ -71,46 +71,51 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """) + """, "3.5") - parse("""async def foo(): return await a""") + parse("""async def foo(): return await a""", "3.5") parse("""def foo(): def foo(): pass async def foo(): await x - """) + """, "3.5") - self.invalid_syntax("await x") + self.invalid_syntax("await x", version="3.5") self.invalid_syntax("""def foo(): - await x""") + await x""", version="3.5") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """) + """, version="3.5") + + self.invalid_syntax("""async def foo(): + def foo(): + await x + """, version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_var(self): - parse("""async = 1""") - parse("""await = 1""") - parse("""def async(): pass""") + parse("""async = 1""", "3.5") + parse("""await = 1""", "3.5") + parse("""def async(): pass""", "3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_with(self): parse("""async def foo(): - async for a in b: pass""") + async for a in b: pass""", "3.5") self.invalid_syntax("""def foo(): - async for a in b: pass""") + async for a in b: pass""", version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""") + async with a: pass""", "3.5") self.invalid_syntax("""def foo(): - async with a: pass""") + async with a: pass""", version="3.5") class TestRaiseChanges(GrammarTest): From 4249563eb28e2a56d5dbff8a9e34eda01c0311fb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 21:08:47 +0100 Subject: [PATCH 12/17] tests can now also run on python 3.5 --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 878d1725..1307b252 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py32, py33, py34 +envlist = py26, py27, py32, py33, py34, py35 [testenv] deps = pytest>=2.3.5 @@ -32,6 +32,10 @@ deps = typing {[testenv]deps} [testenv:py34] +deps = + typing + {[testenv]deps} +[testenv:py35] deps = typing {[testenv]deps} From 04524cd63ccdcb33397f40ec431b03f2217ad92e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Feb 2016 13:06:27 +0100 Subject: [PATCH 13/17] make travis test python3.5 as well --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 876af276..e8295ae9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ env: - TOXENV=py32 - TOXENV=py33 - TOXENV=py34 + - TOXENV=py35 - TOXENV=pypy - TOXENV=cov - TOXENV=sith From 3a36bb3a3680969bbb842a11902f962e7a982fb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Feb 2016 16:53:37 +0100 Subject: [PATCH 14/17] Seems necessary to explicitly specify python3.5: https://github.com/travis-ci/travis-ci/issues/4794 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e8295ae9..9331fafa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +python: 3.5 sudo: false env: - TOXENV=py26 @@ -15,8 +16,9 @@ matrix: - env: TOXENV=cov - env: TOXENV=sith - env: TOXENV=pypy + install: - - pip install --quiet --use-mirrors tox + - pip install tox script: - tox after_script: From 7077d0b76240e5b5f5e388d2ae6f02687a95cb61 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Feb 2016 19:14:31 +0100 Subject: [PATCH 15/17] Using python 3.7-like parser, instead of python 3.5 magic. See https://github.com/davidhalter/jedi/pull/691#issuecomment-182815864 Revert "Update tokenizer to adhere to PEP492 magic" This reverts commit 65187930bd9cf8f54233635a99ee7ca04a05eeff. --- jedi/parser/tokenize.py | 55 ++-------------------------------- test/test_parser/test_pgen2.py | 37 ++++++++++------------- 2 files changed, 18 insertions(+), 74 deletions(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 301d9ff6..46f9ef76 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -16,7 +16,7 @@ import re from io import StringIO from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3, is_py35 +from jedi._compatibility import is_py3 cookie_re = re.compile("coding[:=]\s*([-\w.]+)") @@ -151,58 +151,7 @@ def source_tokens(source): """Generate tokens from a the source code (string).""" source = source readline = StringIO(source).readline - tokenizer = generate_tokens(readline) - if is_py35: - tokenizer = fix_async_await_tokenizer(tokenizer) - return tokenizer - - -def fix_async_await_tokenizer(tokenizer): - # We need to recognise async functions as per - # https://www.python.org/dev/peps/pep-0492/#transition-plan - # so we need a three item look-ahead - from jedi.parser.token import ASYNC, AWAIT - assert is_py35 - try: - first = next(tokenizer) - except StopIteration: - return - - try: - second = next(tokenizer) - except StopIteration: - yield first - return - - defs_stack = [] - indent = 0 - for item in tokenizer: - if first[0] == INDENT: - indent += 1 - - if first[0] == DEDENT: - indent -= 1 - - if first[0] == NEWLINE and second[0] != INDENT: - while defs_stack and indent <= defs_stack[-1][0]: - defs_stack.pop() - - if second[0] == NAME and second[1] == "def" and item[0] == NAME: - if first[0] == NAME and first[1] == "async": - defs_stack.append((indent, True)) - else: - defs_stack.append((indent, False)) - in_async_def = defs_stack and defs_stack[-1][1] - if in_async_def and first[0] == NAME and first[1] == "async": - yield (ASYNC, ) + first[1:] - elif in_async_def and first[0] == NAME and first[1] == "await": - yield (AWAIT, ) + first[1:] - else: - yield first - first = second - second = item - yield first - yield second + return generate_tokens(readline) def generate_tokens(readline): diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 576d338f..6d51ae85 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b", "3.5") - parse("a @= b", "3.5") + parse("a @ b") + parse("a @= b") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """, "3.5") + """) parse("""async def foo(): @@ -71,51 +71,46 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """, "3.5") + """) - parse("""async def foo(): return await a""", "3.5") + parse("""async def foo(): return await a""") parse("""def foo(): def foo(): pass async def foo(): await x - """, "3.5") + """) - self.invalid_syntax("await x", version="3.5") + self.invalid_syntax("await x") self.invalid_syntax("""def foo(): - await x""", version="3.5") + await x""") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """, version="3.5") - - self.invalid_syntax("""async def foo(): - def foo(): - await x - """, version="3.5") + """) @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_var(self): - parse("""async = 1""", "3.5") - parse("""await = 1""", "3.5") - parse("""def async(): pass""", "3.5") + parse("""async = 1""") + parse("""await = 1""") + parse("""def async(): pass""") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_with(self): parse("""async def foo(): - async for a in b: pass""", "3.5") + async for a in b: pass""") self.invalid_syntax("""def foo(): - async for a in b: pass""", version="3.5") + async for a in b: pass""") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""", "3.5") + async with a: pass""") self.invalid_syntax("""def foo(): - async with a: pass""", version="3.5") + async with a: pass""") class TestRaiseChanges(GrammarTest): From d5f08f8bdd2efcd3913770b754d667cbb7fe1cc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Feb 2016 19:29:18 +0100 Subject: [PATCH 16/17] opting for skipping PEP492 backwards compatibility magic, instead directly making await and async keywords See discussion at https://github.com/davidhalter/jedi/pull/691#issuecomment-182815864 --- jedi/api/keywords.py | 9 ++++-- jedi/parser/grammar3.5.txt | 8 +++-- test/test_parser/test_pgen2.py | 54 ++++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index 5493f4b1..5af80d12 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -1,7 +1,7 @@ import pydoc import keyword -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from jedi import common from jedi.evaluate.helpers import FakeName from jedi.parser.tree import Leaf @@ -12,7 +12,12 @@ except ImportError: import pydoc_topics if is_py3: - keys = keyword.kwlist + if is_py35: + # in python 3.5 async and await are not proper keywords, but for + # completion pursposes should as as though they are + keys = keyword.kwlist + ["async", "await"] + else: + keys = keyword.kwlist else: keys = keyword.kwlist + ['None', 'False', 'True'] diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index 9a27f1ec..96a72718 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -23,7 +23,9 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef +# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens +# skipping python3.5 compatibility, in favour of 3.7 solution +async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' @@ -69,7 +71,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -106,7 +108,7 @@ arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] -atom_expr: [AWAIT] atom trailer* +atom_expr: ['await'] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 6d51ae85..e7b4473b 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b") - parse("a @= b") + parse("a @ b", "3.5") + parse("a @= b", "3.5") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """) + """, "3.5") parse("""async def foo(): @@ -71,46 +71,56 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """) + """, "3.5") - parse("""async def foo(): return await a""") + parse("""async def foo(): return await a""", "3.5") parse("""def foo(): def foo(): pass async def foo(): await x - """) + """, "3.5") - self.invalid_syntax("await x") + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_await_expr_invalid(self): + self.invalid_syntax("await x", version="3.5") self.invalid_syntax("""def foo(): - await x""") + await x""", version="3.5") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """) + """, version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") def test_async_var(self): - parse("""async = 1""") - parse("""await = 1""") - parse("""def async(): pass""") - - @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') - def test_async_with(self): - parse("""async def foo(): - async for a in b: pass""") - - self.invalid_syntax("""def foo(): - async for a in b: pass""") + parse("""async = 1""", "3.5") + parse("""await = 1""", "3.5") + parse("""def async(): pass""", "3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""") + async for a in b: pass""", "3.5") + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_for_invalid(self): self.invalid_syntax("""def foo(): - async with a: pass""") + async for a in b: pass""", version="3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_async_with(self): + parse("""async def foo(): + async with a: pass""", "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_with_invalid(self): + self.invalid_syntax("""def foo(): + async with a: pass""", version="3.5") class TestRaiseChanges(GrammarTest): From ca08b8270bbf30bce3c0e256b013a3a371c5df3a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 10:20:25 +0100 Subject: [PATCH 17/17] combine power-or-atom_expr statements into one statement --- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/analysis.py | 2 +- jedi/evaluate/dynamic.py | 3 +-- jedi/evaluate/finder.py | 2 +- jedi/evaluate/helpers.py | 2 +- jedi/parser/tree.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index de78f2f5..0de778be 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -291,7 +291,7 @@ class Evaluator(object): types = set([element]) # TODO this is no real evaluation. elif element.type == 'expr_stmt': types = self.eval_statement(element) - elif element.type == 'power' or element.type == 'atom_expr': + elif element.type in ('power', 'atom_expr'): types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index 87c961dd..7b4b0acc 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): def check_hasattr(node, suite): try: assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type == 'power' or node.type == 'atom_expr' + assert node.type in ('power', 'atom_expr') base = node.children[0] assert base.type == 'name' and base.value == 'hasattr' trailer = node.children[1] diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 6ca429fe..8cd08641 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -89,8 +89,7 @@ def search_function_call(evaluator, func): parent = parent.parent trailer = None - if tree.is_node(parent, 'power') or \ - tree.is_node(parent, 'atom_expr'): + if tree.is_node(parent, 'power', 'atom_expr'): for t in parent.children[1:]: if t == '**': break diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index db9561c3..5003848c 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -448,7 +448,7 @@ def check_flow_information(evaluator, flow, search_name, pos): def _check_isinstance_type(evaluator, element, search_name): try: - assert element.type == 'power' or element.type == 'atom_expr' + assert element.type in ('power', 'atom_expr') # this might be removed if we analyze and, etc assert len(element.children) == 2 first, trailer = element.children diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 86eec876..76d1796e 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -88,7 +88,7 @@ def call_of_name(name, cut_own_trailer=False): return name power = par.parent - if (tree.is_node(power, 'power') or tree.is_node(power, 'atom_expr')) \ + if tree.is_node(power, 'power', 'atom_expr') \ and power.children[0] != name \ and not (power.children[-2] == '**' and name.start_pos > power.children[-1].start_pos): diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 867d8cde..89c37841 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1342,7 +1342,7 @@ def _defined_names(current): names += _defined_names(child) elif is_node(current, 'atom'): names += _defined_names(current.children[1]) - elif is_node(current, 'power') or is_node(current, 'atom_expr'): + elif is_node(current, 'power', 'atom_expr'): if current.children[-2] != '**': # Just if there's no operation trailer = current.children[-1] if trailer.children[0] == '.':