From fadf4f44197a5d48d944d2e6b21c33a8ef274db5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 15:20:26 +0100 Subject: [PATCH 01/26] initial poc pep-0484 type hints --- jedi/evaluate/finder.py | 6 ++++++ jedi/evaluate/pep0484.py | 34 ++++++++++++++++++++++++++++++++++ test/completion/pep0484.py | 14 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 jedi/evaluate/pep0484.py create mode 100644 test/completion/pep0484.py diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 89de08e4..ad2a6c33 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -23,6 +23,7 @@ from jedi.evaluate import representation as er from jedi.evaluate import dynamic from jedi.evaluate import compiled from jedi.evaluate import docstrings +from jedi.evaluate import pep0484 from jedi.evaluate import iterable from jedi.evaluate import imports from jedi.evaluate import analysis @@ -387,6 +388,11 @@ def _eval_param(evaluator, param, scope): and func.instance.is_generated and str(func.name) == '__init__': param = func.var.params[param.position_nr] + # Add pep0484 type hints + pep0484_hints = pep0484.follow_param(evaluator, param) + if pep0484_hints: + return pep0484_hints + # Add docstring knowledge. doc_params = docstrings.follow_param(evaluator, param) if doc_params: diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py new file mode 100644 index 00000000..97433502 --- /dev/null +++ b/jedi/evaluate/pep0484.py @@ -0,0 +1,34 @@ +""" +PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints +through function annotations. There is a strong suggestion in this document +that only the type of type hinting defined in PEP0484 should be allowed +as annotations in future python versions. + +The (initial / probably incomplete) implementation todo list for pep-0484: +v Function parameter annotations with builtin/custom type classes +x Function returntype annotations with builtin/custom type classes +x Function parameter annotations with strings (forward reference) +x Function return type annotations with strings (forward reference) +x Local variable type hints +x Assigned types: `Url = str\ndef get(url:Url) -> str:` +x Type hints in `with` statements +x Stub files support +""" + +from itertools import chain + +from jedi.evaluate.cache import memoize_default + + +@memoize_default(None, evaluator_is_first_arg=True) +def follow_param(evaluator, param): + # annotation is in param.children[0] if present + # either this firstchild is a Name (if no annotation is present) or a Node + if hasattr(param.children[0], "children"): + assert len(param.children[0].children) == 3 and \ + param.children[0].children[1] == ":" + definitions = evaluator.eval_element(param.children[0].children[2]) + return list(chain.from_iterable( + evaluator.execute(d) for d in definitions)) + else: + return [] diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py new file mode 100644 index 00000000..dac4f17a --- /dev/null +++ b/test/completion/pep0484.py @@ -0,0 +1,14 @@ +""" Pep-0484 type hinting """ + +# ----------------- +# sphinx style +# ----------------- +def typehints(a, b: str, c: int, d:int = 4): + #? + a + #? str() + b + #? int() + c + #? int() + d From 5a8c46d509a49ed052ee1c969d70d77366e3bf3b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 21:13:20 +0100 Subject: [PATCH 02/26] seperate parser and testing code --- jedi/evaluate/pep0484.py | 10 ++++------ jedi/parser/tree.py | 9 +++++++-- test/completion/pep0484.py | 8 +++++--- test/test_evaluate/test_annotations.py | 12 ++++++++++-- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 97433502..1256ef2d 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -13,6 +13,7 @@ x Local variable type hints x Assigned types: `Url = str\ndef get(url:Url) -> str:` x Type hints in `with` statements x Stub files support +x support `@no_type_check` and `@no_type_check_decorator` """ from itertools import chain @@ -22,12 +23,9 @@ from jedi.evaluate.cache import memoize_default @memoize_default(None, evaluator_is_first_arg=True) def follow_param(evaluator, param): - # annotation is in param.children[0] if present - # either this firstchild is a Name (if no annotation is present) or a Node - if hasattr(param.children[0], "children"): - assert len(param.children[0].children) == 3 and \ - param.children[0].children[1] == ":" - definitions = evaluator.eval_element(param.children[0].children[2]) + annotation = param.annotation() + if annotation: + definitions = evaluator.eval_element(annotation) return list(chain.from_iterable( evaluator.execute(d) for d in definitions)) else: diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 5a871671..7f0e1960 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1403,8 +1403,13 @@ class Param(BaseNode): return None def annotation(self): - # Generate from tfpdef. - raise NotImplementedError + tfpdef = self._tfpdef() + if is_node(tfpdef, 'tfpdef'): + assert tfpdef.children[1] == ":" + assert len(tfpdef.children) == 3 + return tfpdef.children[2] + else: + return None def _tfpdef(self): """ diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index dac4f17a..64f656fa 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -1,10 +1,12 @@ """ Pep-0484 type hinting """ # ----------------- -# sphinx style +# simple classes # ----------------- -def typehints(a, b: str, c: int, d:int = 4): - #? + + +def typehints(a, b: str, c: int, d: int=4): + #? a #? str() b diff --git a/test/test_evaluate/test_annotations.py b/test/test_evaluate/test_annotations.py index 1fefde3c..1a26a1bc 100644 --- a/test/test_evaluate/test_annotations.py +++ b/test/test_evaluate/test_annotations.py @@ -8,8 +8,8 @@ import pytest def test_simple_annotations(): """ Annotations only exist in Python 3. - At the moment we ignore them. So they should be parsed and not interfere - with anything. + If annotations adhere to PEP-0484, we use them (they override inference), + else they are parsed but ignored """ source = dedent("""\ @@ -27,3 +27,11 @@ def test_simple_annotations(): annot_ret('')""") assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['str'] + + source = dedent("""\ + def annot(a:int): + return a + + annot('')""") + + assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['int'] From c02668a4433915aee752a4208ffc049fb38e8e21 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 21:42:45 +0100 Subject: [PATCH 03/26] Build in version-dependency in integration tests If a line is encountered with the comment or , then the tests are skipped if the current python version is less than the requested one. All tests until the end of the file, or a new comment specifying a compatibe python version are skipped --- test/run.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/run.py b/test/run.py index 6be9b5f2..2b59d85c 100755 --- a/test/run.py +++ b/test/run.py @@ -111,6 +111,7 @@ Tests look like this:: """ import os import re +import sys from ast import literal_eval from io import StringIO from functools import reduce @@ -127,7 +128,7 @@ TEST_USAGES = 3 class IntegrationTestCase(object): def __init__(self, test_type, correct, line_nr, column, start, line, - path=None): + path=None, skip=None): self.test_type = test_type self.correct = correct self.line_nr = line_nr @@ -135,7 +136,7 @@ class IntegrationTestCase(object): self.start = start self.line = line self.path = path - self.skip = None + self.skip = skip @property def module_name(self): @@ -234,10 +235,11 @@ class IntegrationTestCase(object): def collect_file_tests(lines, lines_to_execute): makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column, - start, line) + start, line, path=None, skip=skip) start = None correct = None test_type = None + skip = None for line_nr, line in enumerate(lines, 1): if correct is not None: r = re.match('^(\d+)\s*(.*)$', correct) @@ -257,6 +259,15 @@ def collect_file_tests(lines, lines_to_execute): yield makecase(TEST_DEFINITIONS) correct = None else: + # check for python minimal version number + match = re.match(r" *# *python *>= *(\d+(?:\.\d+)?)$", line) + if match: + minimal_python_version = tuple( + map(int, match.group(1).split("."))) + if sys.version_info >= minimal_python_version: + skip = None + else: + skip = "Minimal python version %s" % match.groups(1) try: r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]*)', line) # test_type is ? for completion and ! for goto_assignments From 68cbabe8195da456efbe28fa2e388979ff6c2591 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 21:43:34 +0100 Subject: [PATCH 04/26] pep0484 tests only on python >= 3.2 --- test/completion/pep0484.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 64f656fa..d9d7608d 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -1,5 +1,7 @@ """ Pep-0484 type hinting """ +# python >= 3.2 + # ----------------- # simple classes # ----------------- From 7e8112d607d14bbe9334c37d8cfd59d13185d9c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 23:05:14 +0100 Subject: [PATCH 05/26] pep0484 return type support --- jedi/evaluate/pep0484.py | 2 +- jedi/evaluate/representation.py | 18 +++++++++ jedi/parser/tree.py | 4 +- test/completion/pep0484.py | 66 +++++++++++++++++++++++++++++---- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 1256ef2d..536e624b 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -6,7 +6,7 @@ as annotations in future python versions. The (initial / probably incomplete) implementation todo list for pep-0484: v Function parameter annotations with builtin/custom type classes -x Function returntype annotations with builtin/custom type classes +v Function returntype annotations with builtin/custom type classes x Function parameter annotations with strings (forward reference) x Function return type annotations with strings (forward reference) x Local variable type hints diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index dffac8c2..173cf9be 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -580,6 +580,21 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): else: return FunctionExecution(self._evaluator, self, params).get_return_types() + @memoize_default() + def py__annotations__(self): + parser_func = self.base + return_annotation = parser_func.annotation() + if return_annotation: + dct = {'return': self._evaluator.eval_element(return_annotation)} + else: + dct = {} + for function_param in parser_func.params: + param_annotation = function_param.annotation() + if param_annotation: + dct[function_param.name.value] = \ + self._evaluator.eval_element(param_annotation) + return dct + def py__class__(self): return compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS') @@ -639,6 +654,9 @@ class FunctionExecution(Executed): else: returns = self.returns types = set(docstrings.find_return_types(self._evaluator, func)) + annotations = func.py__annotations__().get("return", []) + types |= set(chain.from_iterable( + self._evaluator.execute(d) for d in annotations)) for r in returns: check = flow_analysis.break_check(self._evaluator, self, r) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 7f0e1960..c5fb0934 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -866,7 +866,9 @@ class Function(ClassOrFunc): def annotation(self): try: - return self.children[6] # 6th element: def foo(...) -> bar + if self.children[3] == "->": + return self.children[4] + assert self.children[3] == ":" except IndexError: return None diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index d9d7608d..52550405 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -2,17 +2,69 @@ # python >= 3.2 -# ----------------- -# simple classes -# ----------------- + +class A(): + pass -def typehints(a, b: str, c: int, d: int=4): - #? +def function_parameters(a: A, b, c: str, d: int=4): + #? A() a - #? str() + #? b - #? int() + #? str() c #? int() d + + +def return_unspecified(): + pass + +#? +return_unspecified() + + +def return_none() -> None: + """ + Return type None means the same as no return type as far as jedi + is concerned + """ + pass + +#? +return_none() + + +def return_str() -> str: + pass + +#? str() +return_str() + + +def return_custom_class() -> A: + pass + +#? A() +return_custom_class() + + +def return_annotation_and_docstring() -> str: + """ + :rtype: int + """ + pass + +#? str() int() +return_annotation_and_docstring() + + +def return_annotation_and_docstring_different() -> str: + """ + :rtype: str + """ + pass + +#? str() +return_annotation_and_docstring_different() From c61f39cb2b31d1ca431557cd08df88541e48cdb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 23:45:37 +0100 Subject: [PATCH 06/26] add test for annotations to test_parser_tree --- test/test_parser/test_parser_tree.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_parser/test_parser_tree.py b/test/test_parser/test_parser_tree.py index 480230ba..28c7d271 100644 --- a/test/test_parser/test_parser_tree.py +++ b/test/test_parser/test_parser_tree.py @@ -12,10 +12,11 @@ from jedi.parser import tree as pt class TestsFunctionAndLambdaParsing(object): FIXTURES = [ - ('def my_function(x, y, z):\n return x + y * z\n', { + ('def my_function(x, y, z) -> str:\n return x + y * z\n', { 'name': 'my_function', 'call_sig': 'my_function(x, y, z)', 'params': ['x', 'y', 'z'], + 'annotation': "str", }), ('lambda x, y, z: x + y * z\n', { 'name': '', @@ -55,7 +56,11 @@ class TestsFunctionAndLambdaParsing(object): assert not node.yields def test_annotation(self, node, expected): - assert node.annotation() is expected.get('annotation', None) + expected_annotation = expected.get('annotation', None) + if expected_annotation is None: + assert node.annotation() is None + else: + assert node.annotation().value == expected_annotation def test_get_call_signature(self, node, expected): assert node.get_call_signature() == expected['call_sig'] From f8debace0d7add649ea4436a268afbee084f6066 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 23:47:45 +0100 Subject: [PATCH 07/26] forward reference pep-0484 --- jedi/evaluate/pep0484.py | 4 ++-- jedi/parser/tree.py | 22 ++++++++++++++++++++-- test/completion/pep0484.py | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 536e624b..a9dc748c 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -7,8 +7,8 @@ as annotations in future python versions. The (initial / probably incomplete) implementation todo list for pep-0484: v Function parameter annotations with builtin/custom type classes v Function returntype annotations with builtin/custom type classes -x Function parameter annotations with strings (forward reference) -x Function return type annotations with strings (forward reference) +v Function parameter annotations with strings (forward reference) +v Function return type annotations with strings (forward reference) x Local variable type hints x Assigned types: `Url = str\ndef get(url:Url) -> str:` x Type hints in `with` statements diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index c5fb0934..df116820 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -41,6 +41,7 @@ import textwrap from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, literal_eval, use_metaclass, unicode) from jedi import cache +import ast def is_node(node, *symbol_names): @@ -52,6 +53,19 @@ def is_node(node, *symbol_names): return type in symbol_names +def _fix_forward_reference(annotation): + if isinstance(annotation, String): + newannotation = Name( + annotation.position_modifier, + ast.literal_eval(annotation.value), + annotation.start_pos, + annotation.prefix) + newannotation.parent = annotation.parent + else: + newannotation = annotation + return newannotation + + class PositionModifier(object): """A start_pos modifier for the fast parser.""" def __init__(self): @@ -865,9 +879,12 @@ class Function(ClassOrFunc): return bool(self.yields) def annotation(self): + if self.children[0] == "lambda": + # lambda functions have no annotation + return None try: if self.children[3] == "->": - return self.children[4] + return _fix_forward_reference(self.children[4]) assert self.children[3] == ":" except IndexError: return None @@ -1409,7 +1426,8 @@ class Param(BaseNode): if is_node(tfpdef, 'tfpdef'): assert tfpdef.children[1] == ":" assert len(tfpdef.children) == 3 - return tfpdef.children[2] + annotation = tfpdef.children[2] + return _fix_forward_reference(annotation) else: return None diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 52550405..6a7ce6ce 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -68,3 +68,23 @@ def return_annotation_and_docstring_different() -> str: #? str() return_annotation_and_docstring_different() + + +def annotation_forward_reference(b: "B") -> "B": + #? B() + b + +#? B() +annotation_forward_reference(1) + +class B: + pass + + +class SelfReference: + def test(x: "SelfReference") -> "SelfReference": + #? SelfReference() + x + +#? SelfReference() +SelfReference().test() From 7f8b878c8c2dc06aeda5a50124d089ca9404b010 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 13 Dec 2015 23:55:07 +0100 Subject: [PATCH 08/26] if both docstring and annotations are present, use both for function parameters --- jedi/evaluate/finder.py | 10 +++------- test/completion/pep0484.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index ad2a6c33..2e841c99 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -388,15 +388,11 @@ def _eval_param(evaluator, param, scope): and func.instance.is_generated and str(func.name) == '__init__': param = func.var.params[param.position_nr] - # Add pep0484 type hints + # Add pep0484 and docstring knowledge. pep0484_hints = pep0484.follow_param(evaluator, param) - if pep0484_hints: - return pep0484_hints - - # Add docstring knowledge. doc_params = docstrings.follow_param(evaluator, param) - if doc_params: - return doc_params + if pep0484_hints or doc_params: + return list(set(pep0484_hints) | set(doc_params)) if isinstance(param, ExecutedParam): return res_new | param.eval(evaluator) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 6a7ce6ce..b9ab88aa 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -7,7 +7,13 @@ class A(): pass -def function_parameters(a: A, b, c: str, d: int=4): +def function_parameters(a: A, b, c: str, d: int, e: str, f: str, g: int=4): + """ + :param e: if docstring and annotation agree, only one should be returned + :type e: str + :param f: if docstring and annotation disagree, both should be returned + :type f: int + """ #? A() a #? @@ -16,6 +22,12 @@ def function_parameters(a: A, b, c: str, d: int=4): c #? int() d + #? str() + e + #? int() str() + f + # int() + g def return_unspecified(): From 0f08dc6ac6dbbdc81ebf77b4218356ff40150798 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 00:03:07 +0100 Subject: [PATCH 09/26] Addinf myself to AUTHORS --- AUTHORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.txt b/AUTHORS.txt index bc258d6b..9b2736c1 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -34,5 +34,6 @@ Phillip Berndt (@phillipberndt) Ian Lee (@IanLee1521) Farkhad Khatamov (@hatamov) Kevin Kelley (@kelleyk) +Reinoud Elhorst (@reinhrst) Note: (@user) means a github user name. From be399c81c309c4d9e98f814ab55cc23e36a6a164 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 00:49:09 +0100 Subject: [PATCH 10/26] clean out the last_* fields of sys before importing it. The system gets confused if there were uncaught errors in previous tests without this. Particularly, it crashes (at least 2.6) if any tests during test_integrations were skipped. --- test/test_api/test_api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 926f41a2..72ff57e3 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -9,6 +9,13 @@ from jedi._compatibility import is_py3 from pytest import raises +def _clear_last_exception_fields_in_sys(): + import sys + sys.last_type = None + sys.last_value = None + sys.last_traceback = None + + def test_preload_modules(): def check_loaded(*modules): # +1 for None module (currently used) @@ -20,6 +27,7 @@ def test_preload_modules(): temp_cache, cache.parser_cache = cache.parser_cache, {} parser_cache = cache.parser_cache + _clear_last_exception_fields_in_sys() api.preload_module('sys') check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') From 576fdf810678dc937bd27cee5cd134785f9c0fc9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 12:10:00 +0100 Subject: [PATCH 11/26] better separation pep0484 code and py__annotation__() function --- jedi/evaluate/pep0484.py | 18 ++++++++++++++---- jedi/evaluate/representation.py | 12 +++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index a9dc748c..b9008a94 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -21,12 +21,22 @@ from itertools import chain from jedi.evaluate.cache import memoize_default -@memoize_default(None, evaluator_is_first_arg=True) -def follow_param(evaluator, param): - annotation = param.annotation() - if annotation: +def _evaluate_for_annotation(evaluator, annotation): + if annotation is not None: definitions = evaluator.eval_element(annotation) return list(chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] + + +@memoize_default(None, evaluator_is_first_arg=True) +def follow_param(evaluator, param): + annotation = param.annotation() + return _evaluate_for_annotation(evaluator, annotation) + + +@memoize_default(None, evaluator_is_first_arg=True) +def find_return_types(evaluator, func): + annotation = func.py__annotations__().get("return", None) + return _evaluate_for_annotation(evaluator, annotation) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 173cf9be..9d0026c0 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -49,6 +49,7 @@ from jedi.evaluate import compiled from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate import docstrings +from jedi.evaluate import pep0484 from jedi.evaluate import helpers from jedi.evaluate import param from jedi.evaluate import flow_analysis @@ -585,14 +586,13 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): parser_func = self.base return_annotation = parser_func.annotation() if return_annotation: - dct = {'return': self._evaluator.eval_element(return_annotation)} + dct = {'return': return_annotation} else: dct = {} for function_param in parser_func.params: param_annotation = function_param.annotation() - if param_annotation: - dct[function_param.name.value] = \ - self._evaluator.eval_element(param_annotation) + if param_annotation is not None: + dct[function_param.name.value] = param_annotation return dct def py__class__(self): @@ -654,9 +654,7 @@ class FunctionExecution(Executed): else: returns = self.returns types = set(docstrings.find_return_types(self._evaluator, func)) - annotations = func.py__annotations__().get("return", []) - types |= set(chain.from_iterable( - self._evaluator.execute(d) for d in annotations)) + types |= set(pep0484.find_return_types(self._evaluator, func)) for r in returns: check = flow_analysis.break_check(self._evaluator, self, r) From 6ce076f4134630dac6915e009e2c4ccd4377669a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 12:10:48 +0100 Subject: [PATCH 12/26] more elaborate tests --- test/completion/pep0484.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index b9ab88aa..040461e2 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -88,15 +88,25 @@ def annotation_forward_reference(b: "B") -> "B": #? B() annotation_forward_reference(1) +#? ["test_element"] +annotation_forward_reference(1).t class B: + test_element = 1 pass class SelfReference: - def test(x: "SelfReference") -> "SelfReference": + test_element = 1 + def test_method(self, x: "SelfReference") -> "SelfReference": #? SelfReference() x + #? ["test_element", "test_method"] + self.t + #? ["test_element", "test_method"] + x.t + #? ["test_element", "test_method"] + self.test_method(1).t #? SelfReference() -SelfReference().test() +SelfReference().test_method() From 0f6fb23d91890afb39b2b55d492d9b8e63582b89 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 22:02:11 +0100 Subject: [PATCH 13/26] override annotation() in Lambda, instead of checking in Function on type --- jedi/parser/tree.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index df116820..a20030cb 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -879,9 +879,6 @@ class Function(ClassOrFunc): return bool(self.yields) def annotation(self): - if self.children[0] == "lambda": - # lambda functions have no annotation - return None try: if self.children[3] == "->": return _fix_forward_reference(self.children[4]) @@ -964,6 +961,10 @@ class Lambda(Function): def is_generator(self): return False + def annotation(self): + # lambda functions do not support annotations + return None + @property def yields(self): return [] From 626fa60d03ce0ff8ad28d368612d0eaa3d73a48f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Dec 2015 22:37:20 +0100 Subject: [PATCH 14/26] Revert "clean out the last_* fields of sys before importing it." This reverts commit be399c81c309c4d9e98f814ab55cc23e36a6a164. Will break python 2.6 (possibly 2.7) tests; this is expected behaviour. See https://github.com/davidhalter/jedi/pull/661#discussion_r47543815 --- test/test_api/test_api.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 72ff57e3..926f41a2 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -9,13 +9,6 @@ from jedi._compatibility import is_py3 from pytest import raises -def _clear_last_exception_fields_in_sys(): - import sys - sys.last_type = None - sys.last_value = None - sys.last_traceback = None - - def test_preload_modules(): def check_loaded(*modules): # +1 for None module (currently used) @@ -27,7 +20,6 @@ def test_preload_modules(): temp_cache, cache.parser_cache = cache.parser_cache, {} parser_cache = cache.parser_cache - _clear_last_exception_fields_in_sys() api.preload_module('sys') check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') From 3cef8b6d557538ac7a757f6321fbaae8c1a1fca2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Dec 2015 00:31:47 +0100 Subject: [PATCH 15/26] string-annotations should only be interpreted by the pep-0484 code, not the parser --- jedi/evaluate/pep0484.py | 14 ++++++++++++-- jedi/parser/tree.py | 19 +++---------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index b9008a94..2c2cb173 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -17,13 +17,23 @@ x support `@no_type_check` and `@no_type_check_decorator` """ from itertools import chain - +from jedi.parser import Parser, load_grammar from jedi.evaluate.cache import memoize_default +from jedi.evaluate.compiled import CompiledObject def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: - definitions = evaluator.eval_element(annotation) + definitions = set() + for definition in evaluator.eval_element(annotation): + if (isinstance(definition, CompiledObject) and + isinstance(definition.obj, str)): + p = Parser(load_grammar(), definition.obj) + element = p.module.children[0].children[0] + element.parent = annotation.parent + definitions |= evaluator.eval_element(element) + else: + definitions.add(definition) return list(chain.from_iterable( evaluator.execute(d) for d in definitions)) else: diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index a20030cb..2951722c 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -41,7 +41,6 @@ import textwrap from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, literal_eval, use_metaclass, unicode) from jedi import cache -import ast def is_node(node, *symbol_names): @@ -53,19 +52,6 @@ def is_node(node, *symbol_names): return type in symbol_names -def _fix_forward_reference(annotation): - if isinstance(annotation, String): - newannotation = Name( - annotation.position_modifier, - ast.literal_eval(annotation.value), - annotation.start_pos, - annotation.prefix) - newannotation.parent = annotation.parent - else: - newannotation = annotation - return newannotation - - class PositionModifier(object): """A start_pos modifier for the fast parser.""" def __init__(self): @@ -881,8 +867,9 @@ class Function(ClassOrFunc): def annotation(self): try: if self.children[3] == "->": - return _fix_forward_reference(self.children[4]) + return self.children[4] assert self.children[3] == ":" + return None except IndexError: return None @@ -1428,7 +1415,7 @@ class Param(BaseNode): assert tfpdef.children[1] == ":" assert len(tfpdef.children) == 3 annotation = tfpdef.children[2] - return _fix_forward_reference(annotation) + return annotation else: return None From 12588753000baba2dcf68530c70cd6e7d5817b4e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Dec 2015 00:37:23 +0100 Subject: [PATCH 16/26] add test that jedi doesn't break in case of non-pep-0484 comments --- test/completion/pep0484.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 040461e2..ed195b15 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -110,3 +110,13 @@ class SelfReference: #? SelfReference() SelfReference().test_method() + +def function_with_non_pep_0484_annotation(x: "I can put anything here", y: 3 + 3) -> int("42"): + # infers int from function call + #? int() + x + # infers str from function call + #? str() + y +#? +function_with_non_pep_0484_annotation(1, "force string") From 35fda3823edd2b4225b9f82048cca71ac43620a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Dec 2015 11:53:48 +0100 Subject: [PATCH 17/26] test dynamic annotation and dynamic forward reference --- test/completion/pep0484.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index ed195b15..9e1c5e00 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -120,3 +120,16 @@ def function_with_non_pep_0484_annotation(x: "I can put anything here", y: 3 + 3 y #? function_with_non_pep_0484_annotation(1, "force string") + +def function_forward_reference_dynamic( + x: return_str_type(), + y: "return_str_type()") -> None: + # technically should not be resolvable since out of scope, + # but jedi is not smart enough for that + #? str() + x + #? str() + y + +def return_str_type(): + return str From 1e6397b1639bb048711c1ad9db6f0b9f58713dee Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Dec 2015 11:56:54 +0100 Subject: [PATCH 18/26] check 'assigned types'-support (comes out of the jedi-box), and add tests for that --- jedi/evaluate/pep0484.py | 2 +- test/completion/pep0484.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 2c2cb173..e31e842a 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -10,7 +10,7 @@ v Function returntype annotations with builtin/custom type classes v Function parameter annotations with strings (forward reference) v Function return type annotations with strings (forward reference) x Local variable type hints -x Assigned types: `Url = str\ndef get(url:Url) -> str:` +v Assigned types: `Url = str\ndef get(url:Url) -> str:` x Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 9e1c5e00..bba6f50a 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -133,3 +133,12 @@ def function_forward_reference_dynamic( def return_str_type(): return str + + +X = str +def function_with_assined_class_in_reference(x: X, y: "Y"): + #? str() + x + #? int() + y +Y = int From 8bf2fe77e235e322365f178d2a54a79a21f3129d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Dec 2015 15:06:20 +0100 Subject: [PATCH 19/26] add some more non-pep0484-junk to the test --- test/completion/pep0484.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index bba6f50a..d49f8c58 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -111,7 +111,11 @@ class SelfReference: #? SelfReference() SelfReference().test_method() -def function_with_non_pep_0484_annotation(x: "I can put anything here", y: 3 + 3) -> int("42"): +def function_with_non_pep_0484_annotation( + x: "I can put anything here", + xx: "", + yy: "\r\n;+*&^564835(---^&*34", + y: 3 + 3) -> int("42"): # infers int from function call #? int() x @@ -119,7 +123,7 @@ def function_with_non_pep_0484_annotation(x: "I can put anything here", y: 3 + 3 #? str() y #? -function_with_non_pep_0484_annotation(1, "force string") +function_with_non_pep_0484_annotation(1, 2, 3, "force string") def function_forward_reference_dynamic( x: return_str_type(), From 6bee2149485ef3d91a0ebb8006bb59e06fcfd359 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Dec 2015 15:23:40 +0100 Subject: [PATCH 20/26] catch error in certain non-pep0484 annotations --- jedi/evaluate/pep0484.py | 5 ++++- test/completion/pep0484.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index e31e842a..cb3e8a4c 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -29,7 +29,10 @@ def _evaluate_for_annotation(evaluator, annotation): if (isinstance(definition, CompiledObject) and isinstance(definition.obj, str)): p = Parser(load_grammar(), definition.obj) - element = p.module.children[0].children[0] + try: + element = p.module.children[0].children[0] + except (AttributeError, IndexError): + continue element.parent = annotation.parent definitions |= evaluator.eval_element(element) else: diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index d49f8c58..d6ec8291 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -114,14 +114,23 @@ SelfReference().test_method() def function_with_non_pep_0484_annotation( x: "I can put anything here", xx: "", - yy: "\r\n;+*&^564835(---^&*34", - y: 3 + 3) -> int("42"): + yy: "\r\n\0;+*&^564835(---^&*34", + y: 3 + 3, + zz: float) -> int("42"): # infers int from function call #? int() x + # infers int from function call + #? int() + xx + # infers int from function call + #? int() + yy # infers str from function call #? str() y + #? float() + zz #? function_with_non_pep_0484_annotation(1, 2, 3, "force string") From 160b6fca516a6a8925757f131541acf16096291e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Dec 2015 15:29:49 +0100 Subject: [PATCH 21/26] show off some power :) --- test/completion/pep0484.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index d6ec8291..a090b7fa 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -155,3 +155,7 @@ def function_with_assined_class_in_reference(x: X, y: "Y"): #? int() y Y = int + +def just_because_we_can(x: "flo" + "at"): + #? float() + x From b2a691a69ad95ab84b2123ecb1b5c990047c5a39 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 19 Dec 2015 11:10:05 +0100 Subject: [PATCH 22/26] PEP 484 support also means that we should evaluate comments in the future. --- jedi/evaluate/pep0484.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index cb3e8a4c..2dde9793 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -14,6 +14,8 @@ v Assigned types: `Url = str\ndef get(url:Url) -> str:` x Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` +x support for type hint comments `# type: (int, str) -> int`. See comment from + Guido https://github.com/davidhalter/jedi/issues/662 """ from itertools import chain From 9a93d599dace4e431385843e0413fbae7c9ee2bb Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 20 Dec 2015 02:35:23 +0100 Subject: [PATCH 23/26] Fix: __module__ doesn't need to be properly defined. --- jedi/evaluate/compiled/fake.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 66c8bfc3..064b6e91 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -68,7 +68,11 @@ def get_module(obj): # Happens for example in `(_ for _ in []).send.__module__`. return builtins else: - return __import__(imp_plz) + try: + return __import__(imp_plz) + except ImportError: + # __module__ can be something arbitrary that doesn't exist. + return builtins def _faked(module, obj, name): From c4906e0e3f9aca7e6d9d9d14f47f900f038ce316 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 20 Dec 2015 22:21:47 +0100 Subject: [PATCH 24/26] Rework the parser so we can use arbitrary start nodes of the syntax. This also includes a rework for error recovery in the parser. This is now just possible for file_input parsing, which means for full files. Includes also a refactoring of the tokenizer. No more do we have to add an additional newline, because it now works correctly (removes certain confusion. --- jedi/api/__init__.py | 11 +- jedi/evaluate/compiled/fake.py | 4 +- jedi/evaluate/docstrings.py | 4 +- jedi/evaluate/finder.py | 4 +- jedi/evaluate/pep0484.py | 16 +- jedi/evaluate/sys_path.py | 4 +- jedi/parser/__init__.py | 269 ++++++++++-------- jedi/parser/fast.py | 13 +- jedi/parser/pgen2/parse.py | 4 +- jedi/parser/tokenize.py | 10 +- jedi/parser/tree.py | 4 +- jedi/parser/user_context.py | 4 +- test/test_evaluate/test_absolute_import.py | 8 +- test/test_evaluate/test_annotations.py | 17 ++ test/test_evaluate/test_buildout_detection.py | 10 +- test/test_evaluate/test_sys_path.py | 4 +- test/test_new_parser.py | 4 +- test/test_parser/test_get_code.py | 10 +- test/test_parser/test_parser.py | 28 +- test/test_parser/test_parser_tree.py | 4 +- test/test_parser/test_tokenize.py | 8 +- test/test_regression.py | 4 +- 22 files changed, 246 insertions(+), 198 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index ed6b7d88..0097821f 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -322,15 +322,12 @@ class Script(object): @memoize_default() def _get_under_cursor_stmt(self, cursor_txt, start_pos=None): - tokenizer = source_tokens(cursor_txt) - r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer) - try: - # Take the last statement available that is not an endmarker. - # And because it's a simple_stmt, we need to get the first child. - stmt = r.module.children[-2].children[0] - except (AttributeError, IndexError): + node = Parser(self._grammar, cursor_txt, 'eval_input').get_parsed_node() + if node is None: return None + stmt = node.children[0] + user_stmt = self._parser.user_stmt() if user_stmt is None: # Set the start_pos to a pseudo position, that doesn't exist but diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 064b6e91..c5ccdede 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -8,7 +8,7 @@ import os import inspect from jedi._compatibility import is_py3, builtins, unicode -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt from jedi.evaluate.helpers import FakeName @@ -31,7 +31,7 @@ def _load_faked_module(module): modules[module_name] = None return grammar = load_grammar('grammar3.4') - module = Parser(grammar, unicode(source), module_name).module + module = ParserWithRecovery(grammar, unicode(source), module_name).module modules[module_name] = module if module_name == 'builtins' and not is_py3: diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 9c79bddd..3561f619 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -20,7 +20,7 @@ from itertools import chain from textwrap import dedent from jedi.evaluate.cache import memoize_default -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.common import indent_block from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated @@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module): # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. - p = Parser(load_grammar(), code % indent_block(string)) + p = ParserWithRecovery(load_grammar(), code % indent_block(string)) try: pseudo_cls = p.module.subscopes[0] # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 3a96d403..31a76009 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -487,8 +487,8 @@ def global_names_dict_generator(evaluator, scope, position): the current scope is function: >>> from jedi._compatibility import u, no_unicode_pprint - >>> from jedi.parser import Parser, load_grammar - >>> parser = Parser(load_grammar(), u(''' + >>> from jedi.parser import ParserWithRecovery, load_grammar + >>> parser = ParserWithRecovery(load_grammar(), u(''' ... x = ['a', 'b', 'c'] ... def func(): ... y = None diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 2dde9793..1896b8d0 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -19,9 +19,11 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from """ from itertools import chain + from jedi.parser import Parser, load_grammar from jedi.evaluate.cache import memoize_default from jedi.evaluate.compiled import CompiledObject +from jedi import debug def _evaluate_for_annotation(evaluator, annotation): @@ -30,13 +32,13 @@ def _evaluate_for_annotation(evaluator, annotation): for definition in evaluator.eval_element(annotation): if (isinstance(definition, CompiledObject) and isinstance(definition.obj, str)): - p = Parser(load_grammar(), definition.obj) - try: - element = p.module.children[0].children[0] - except (AttributeError, IndexError): - continue - element.parent = annotation.parent - definitions |= evaluator.eval_element(element) + p = Parser(load_grammar(), definition.obj, start='expr') + element = p.get_parsed_node() + if element is None: + debug.warning('Annotation not parsed: %s' % definition.obj) + else: + element.parent = annotation.parent + definitions |= evaluator.eval_element(element) else: definitions.add(definition) return list(chain.from_iterable( diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 94e6052f..e838177a 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -5,7 +5,7 @@ from jedi.evaluate.site import addsitedir from jedi._compatibility import exec_function, unicode from jedi.parser import tree -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi import debug from jedi import common @@ -209,7 +209,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script): debug.dbg('Error trying to read buildout_script: %s', buildout_script) return - p = Parser(evaluator.grammar, source, buildout_script) + p = ParserWithRecovery(evaluator.grammar, source, buildout_script) cache.save_parser(buildout_script, p) return p.module diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 9072a8b3..21a3315e 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -81,92 +81,87 @@ class ParserSyntaxError(object): class Parser(object): - """ - This class is used to parse a Python file, it then divides them into a - class structure of different scopes. + AST_MAPPING = { + 'expr_stmt': pt.ExprStmt, + 'classdef': pt.Class, + 'funcdef': pt.Function, + 'file_input': pt.Module, + 'import_name': pt.ImportName, + 'import_from': pt.ImportFrom, + 'break_stmt': pt.KeywordStatement, + 'continue_stmt': pt.KeywordStatement, + 'return_stmt': pt.ReturnStmt, + 'raise_stmt': pt.KeywordStatement, + 'yield_expr': pt.YieldExpr, + 'del_stmt': pt.KeywordStatement, + 'pass_stmt': pt.KeywordStatement, + 'global_stmt': pt.GlobalStmt, + 'nonlocal_stmt': pt.KeywordStatement, + 'print_stmt': pt.KeywordStatement, + 'assert_stmt': pt.AssertStmt, + 'if_stmt': pt.IfStmt, + 'with_stmt': pt.WithStmt, + 'for_stmt': pt.ForStmt, + 'while_stmt': pt.WhileStmt, + 'try_stmt': pt.TryStmt, + 'comp_for': pt.CompFor, + 'decorator': pt.Decorator, + 'lambdef': pt.Lambda, + 'old_lambdef': pt.Lambda, + 'lambdef_nocond': pt.Lambda, + } - :param grammar: The grammar object of pgen2. Loaded by load_grammar. - :param source: The codebase for the parser. Must be unicode. - :param module_path: The path of the module in the file system, may be None. - :type module_path: str - :param top_module: Use this module as a parent instead of `self.module`. - """ - def __init__(self, grammar, source, module_path=None, tokenizer=None): - self._ast_mapping = { - 'expr_stmt': pt.ExprStmt, - 'classdef': pt.Class, - 'funcdef': pt.Function, - 'file_input': pt.Module, - 'import_name': pt.ImportName, - 'import_from': pt.ImportFrom, - 'break_stmt': pt.KeywordStatement, - 'continue_stmt': pt.KeywordStatement, - 'return_stmt': pt.ReturnStmt, - 'raise_stmt': pt.KeywordStatement, - 'yield_expr': pt.YieldExpr, - 'del_stmt': pt.KeywordStatement, - 'pass_stmt': pt.KeywordStatement, - 'global_stmt': pt.GlobalStmt, - 'nonlocal_stmt': pt.KeywordStatement, - 'print_stmt': pt.KeywordStatement, - 'assert_stmt': pt.AssertStmt, - 'if_stmt': pt.IfStmt, - 'with_stmt': pt.WithStmt, - 'for_stmt': pt.ForStmt, - 'while_stmt': pt.WhileStmt, - 'try_stmt': pt.TryStmt, - 'comp_for': pt.CompFor, - 'decorator': pt.Decorator, - 'lambdef': pt.Lambda, - 'old_lambdef': pt.Lambda, - 'lambdef_nocond': pt.Lambda, - } + class ParserError(Exception): + pass - self.syntax_errors = [] + def __init__(self, grammar, source, start, tokenizer=None): + start_number = grammar.symbol2number[start] - self._global_names = [] - self._omit_dedent_list = [] - self._indent_counter = 0 - self._last_failed_start_pos = (0, 0) - - # TODO do print absolute import detection here. - #try: - # del python_grammar_no_print_statement.keywords["print"] - #except KeyError: - # pass # Doesn't exist in the Python 3 grammar. - - #if self.options["print_function"]: - # python_grammar = pygram.python_grammar_no_print_statement - #else: self._used_names = {} self._scope_names_stack = [{}] self._error_statement_stacks = [] - - added_newline = False - # The Python grammar needs a newline at the end of each statement. - if not source.endswith('\n'): - source += '\n' - added_newline = True + self._last_failed_start_pos = (0, 0) + self._global_names = [] # For the fast parser. self.position_modifier = pt.PositionModifier() - p = PgenParser(grammar, self.convert_node, self.convert_leaf, - self.error_recovery) - tokenizer = tokenizer or tokenize.source_tokens(source) - self.module = p.parse(self._tokenize(tokenizer)) - if self.module.type != 'file_input': - # If there's only one statement, we get back a non-module. That's - # not what we want, we want a module, so we add it here: - self.module = self.convert_node(grammar, - grammar.symbol2number['file_input'], - [self.module]) - if added_newline: - self.remove_last_newline() - self.module.used_names = self._used_names - self.module.path = module_path - self.module.global_names = self._global_names - self.module.error_statement_stacks = self._error_statement_stacks + added_newline = False + # The Python grammar needs a newline at the end of each statement. + if not source.endswith('\n') and start == 'file_input': + source += '\n' + added_newline = True + + p = PgenParser(grammar, self.convert_node, self.convert_leaf, + self.error_recovery, start_number) + if tokenizer is None: + tokenizer = tokenize.source_tokens(source) + try: + self._parsed = p.parse(self._tokenize(tokenizer)) + except Parser.ParserError: + self._parsed = None + else: + if start == 'file_input' != self._parsed.type: + # If there's only one statement, we get back a non-module. That's + # not what we want, we want a module, so we add it here: + self._parsed = self.convert_node(grammar, + grammar.symbol2number['file_input'], + [self._parsed]) + + if added_newline: + self.remove_last_newline() + + def get_parsed_node(self): + return self._parsed + + def _tokenize(self, tokenizer): + for typ, value, start_pos, prefix in tokenizer: + if typ == OP: + typ = token.opmap[value] + yield typ, value, prefix, start_pos + + def error_recovery(self, *args, **kwargs): + raise Parser.ParserError def convert_node(self, grammar, type, children): """ @@ -178,7 +173,7 @@ class Parser(object): """ symbol = grammar.number2symbol[type] try: - new_node = self._ast_mapping[symbol](children) + new_node = Parser.AST_MAPPING[symbol](children) except KeyError: new_node = pt.Node(symbol, children) @@ -231,6 +226,83 @@ class Parser(object): else: return pt.Operator(self.position_modifier, value, start_pos, prefix) + def remove_last_newline(self): + """ + In all of this we need to work with _start_pos, because if we worked + with start_pos, we would need to check the position_modifier as well + (which is accounted for in the start_pos property). + """ + endmarker = self._parsed.children[-1] + # The newline is either in the endmarker as a prefix or the previous + # leaf as a newline token. + if endmarker.prefix.endswith('\n'): + endmarker.prefix = endmarker.prefix[:-1] + last_line = re.sub('.*\n', '', endmarker.prefix) + endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line) + else: + try: + newline = endmarker.get_previous() + except IndexError: + return # This means that the parser is empty. + while True: + if newline.value == '': + # Must be a DEDENT, just continue. + try: + newline = newline.get_previous() + except IndexError: + # If there's a statement that fails to be parsed, there + # will be no previous leaf. So just ignore it. + break + elif newline.value != '\n': + # This may happen if error correction strikes and removes + # a whole statement including '\n'. + break + else: + newline.value = '' + if self._last_failed_start_pos > newline._start_pos: + # It may be the case that there was a syntax error in a + # function. In that case error correction removes the + # right newline. So we use the previously assigned + # _last_failed_start_pos variable to account for that. + endmarker._start_pos = self._last_failed_start_pos + else: + endmarker._start_pos = newline._start_pos + break + + +class ParserWithRecovery(Parser): + """ + This class is used to parse a Python file, it then divides them into a + class structure of different scopes. + + :param grammar: The grammar object of pgen2. Loaded by load_grammar. + :param source: The codebase for the parser. Must be unicode. + :param module_path: The path of the module in the file system, may be None. + :type module_path: str + """ + def __init__(self, grammar, source, module_path=None, tokenizer=None): + self.syntax_errors = [] + + self._omit_dedent_list = [] + self._indent_counter = 0 + + # TODO do print absolute import detection here. + #try: + # del python_grammar_no_print_statement.keywords["print"] + #except KeyError: + # pass # Doesn't exist in the Python 3 grammar. + + #if self.options["print_function"]: + # python_grammar = pygram.python_grammar_no_print_statement + #else: + super(ParserWithRecovery, self).__init__(grammar, source, 'file_input', tokenizer) + + self.module = self._parsed + self.module.used_names = self._used_names + self.module.path = module_path + self.module.global_names = self._global_names + self.module.error_statement_stacks = self._error_statement_stacks + def error_recovery(self, grammar, stack, typ, value, start_pos, prefix, add_token_callback): """ @@ -349,46 +421,3 @@ class Parser(object): def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.module) - - def remove_last_newline(self): - """ - In all of this we need to work with _start_pos, because if we worked - with start_pos, we would need to check the position_modifier as well - (which is accounted for in the start_pos property). - """ - endmarker = self.module.children[-1] - # The newline is either in the endmarker as a prefix or the previous - # leaf as a newline token. - if endmarker.prefix.endswith('\n'): - endmarker.prefix = endmarker.prefix[:-1] - last_line = re.sub('.*\n', '', endmarker.prefix) - endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line) - else: - try: - newline = endmarker.get_previous() - except IndexError: - return # This means that the parser is empty. - while True: - if newline.value == '': - # Must be a DEDENT, just continue. - try: - newline = newline.get_previous() - except IndexError: - # If there's a statement that fails to be parsed, there - # will be no previous leaf. So just ignore it. - break - elif newline.value != '\n': - # This may happen if error correction strikes and removes - # a whole statement including '\n'. - break - else: - newline.value = '' - if self._last_failed_start_pos > newline._start_pos: - # It may be the case that there was a syntax error in a - # function. In that case error correction removes the - # right newline. So we use the previously assigned - # _last_failed_start_pos variable to account for that. - endmarker._start_pos = self._last_failed_start_pos - else: - endmarker._start_pos = newline._start_pos - break diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index 35bb8555..f62f9a53 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -8,7 +8,7 @@ from itertools import chain from jedi._compatibility import use_metaclass from jedi import settings -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.parser import tree from jedi import cache from jedi import debug @@ -52,8 +52,9 @@ class FastModule(tree.Module): return "" % (type(self).__name__, self.name, self.start_pos[0], self.end_pos[0]) - # To avoid issues with with the `parser.Parser`, we need setters that do - # nothing, because if pickle comes along and sets those values. + # To avoid issues with with the `parser.ParserWithRecovery`, we need + # setters that do nothing, because if pickle comes along and sets those + # values. @global_names.setter def global_names(self, value): pass @@ -99,10 +100,10 @@ class CachedFastParser(type): """ This is a metaclass for caching `FastParser`. """ def __call__(self, grammar, source, module_path=None): if not settings.fast_parser: - return Parser(grammar, source, module_path) + return ParserWithRecovery(grammar, source, module_path) pi = cache.parser_cache.get(module_path, None) - if pi is None or isinstance(pi.parser, Parser): + if pi is None or isinstance(pi.parser, ParserWithRecovery): p = super(CachedFastParser, self).__call__(grammar, source, module_path) else: p = pi.parser # pi is a `cache.ParserCacheItem` @@ -432,7 +433,7 @@ class FastParser(use_metaclass(CachedFastParser)): else: tokenizer = FastTokenizer(parser_code) self.number_parsers_used += 1 - p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) + p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) end = line_offset + p.module.end_pos[0] used_lines = self._lines[line_offset:end - 1] diff --git a/jedi/parser/pgen2/parse.py b/jedi/parser/pgen2/parse.py index c8ba70d3..605f5c8b 100644 --- a/jedi/parser/pgen2/parse.py +++ b/jedi/parser/pgen2/parse.py @@ -60,7 +60,7 @@ class PgenParser(object): """ - def __init__(self, grammar, convert_node, convert_leaf, error_recovery): + def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start): """Constructor. The grammar argument is a grammar.Grammar instance; see the @@ -90,8 +90,6 @@ class PgenParser(object): self.convert_node = convert_node self.convert_leaf = convert_leaf - # Prepare for parsing. - start = self.grammar.start # Each stack entry is a tuple: (dfa, state, node). # A node is a tuple: (type, children), # where children is a list of nodes or None diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index b3849046..ebc8a4ee 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -149,7 +149,7 @@ ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except', def source_tokens(source): """Generate tokens from a the source code (string).""" - source = source + '\n' # end with \n, because the parser needs it + source = source readline = StringIO(source).readline return generate_tokens(readline) @@ -165,6 +165,7 @@ def generate_tokens(readline): paren_level = 0 # count parentheses indents = [0] lnum = 0 + max = 0 numchars = '0123456789' contstr = '' contline = None @@ -282,9 +283,12 @@ def generate_tokens(readline): paren_level -= 1 yield OP, token, spos, prefix - end_pos = (lnum, max - 1) + if new_line: + end_pos = lnum + 1, 0 + else: + end_pos = lnum, max - 1 # As the last position we just take the maximally possible position. We # remove -1 for the last new line. for indent in indents[1:]: yield DEDENT, '', end_pos, '' - yield ENDMARKER, '', end_pos, prefix + yield ENDMARKER, '', end_pos, additional_prefix diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index d11f128f..5eb1bc5b 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -14,8 +14,8 @@ The easiest way to play with this module is to use :class:`parsing.Parser`. :attr:`parsing.Parser.module` holds an instance of :class:`Module`: >>> from jedi._compatibility import u ->>> from jedi.parser import Parser, load_grammar ->>> parser = Parser(load_grammar(), u('import os'), 'example.py') +>>> from jedi.parser import ParserWithRecovery, load_grammar +>>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py') >>> submodule = parser.module >>> submodule diff --git a/jedi/parser/user_context.py b/jedi/parser/user_context.py index 65049ad4..d62a383f 100644 --- a/jedi/parser/user_context.py +++ b/jedi/parser/user_context.py @@ -4,7 +4,7 @@ import keyword from jedi import cache from jedi import common -from jedi.parser import tokenize, Parser +from jedi.parser import tokenize, ParserWithRecovery from jedi._compatibility import u from jedi.parser.fast import FastParser from jedi.parser import tree @@ -284,7 +284,7 @@ class UserContextParser(object): # Don't pickle that module, because the main module is changing quickly cache.save_parser(self._path, parser, pickling=False) else: - parser = Parser(self._grammar, self._source, self._path) + parser = ParserWithRecovery(self._grammar, self._source, self._path) self._parser_done_callback(parser) return parser diff --git a/test/test_evaluate/test_absolute_import.py b/test/test_evaluate/test_absolute_import.py index a453a8aa..a45bffa5 100644 --- a/test/test_evaluate/test_absolute_import.py +++ b/test/test_evaluate/test_absolute_import.py @@ -4,7 +4,7 @@ Python 2.X) """ import jedi from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from .. import helpers @@ -12,7 +12,7 @@ def test_explicit_absolute_imports(): """ Detect modules with ``from __future__ import absolute_import``. """ - parser = Parser(load_grammar(), u("from __future__ import absolute_import"), "test.py") + parser = ParserWithRecovery(load_grammar(), u("from __future__ import absolute_import"), "test.py") assert parser.module.has_explicit_absolute_import @@ -20,7 +20,7 @@ def test_no_explicit_absolute_imports(): """ Detect modules without ``from __future__ import absolute_import``. """ - parser = Parser(load_grammar(), u("1"), "test.py") + parser = ParserWithRecovery(load_grammar(), u("1"), "test.py") assert not parser.module.has_explicit_absolute_import @@ -30,7 +30,7 @@ def test_dont_break_imports_without_namespaces(): assume that all imports have non-``None`` namespaces. """ src = u("from __future__ import absolute_import\nimport xyzzy") - parser = Parser(load_grammar(), src, "test.py") + parser = ParserWithRecovery(load_grammar(), src, "test.py") assert parser.module.has_explicit_absolute_import diff --git a/test/test_evaluate/test_annotations.py b/test/test_evaluate/test_annotations.py index 1a26a1bc..a1717cac 100644 --- a/test/test_evaluate/test_annotations.py +++ b/test/test_evaluate/test_annotations.py @@ -35,3 +35,20 @@ def test_simple_annotations(): annot('')""") assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['int'] + + +@pytest.mark.skipif('sys.version_info[0] < 3') +@pytest.mark.parametrize('reference', [ + 'assert 1', + '1', + 'lambda: 3', + 'def x(): pass', + '1, 2', + r'1\n' +]) +def test_illegal_forward_references(reference): + source = """ + def foo(bar: "%s"): + bar""" % reference + + assert not jedi.Script(source).goto_definitions() diff --git a/test/test_evaluate/test_buildout_detection.py b/test/test_evaluate/test_buildout_detection.py index f3164a7d..c5c65568 100644 --- a/test/test_evaluate/test_buildout_detection.py +++ b/test/test_evaluate/test_buildout_detection.py @@ -7,7 +7,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file, sys_path_with_modifications, _check_module) from jedi.evaluate import Evaluator -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from ..helpers import cwd_at @@ -37,7 +37,7 @@ def test_append_on_non_sys_path(): d = Dummy() d.path.append('foo')""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert len(paths) > 0 assert 'foo' not in paths @@ -48,7 +48,7 @@ def test_path_from_invalid_sys_path_assignment(): import sys sys.path = 'invalid'""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert len(paths) > 0 assert 'invalid' not in paths @@ -60,7 +60,7 @@ def test_sys_path_with_modifications(): import os """)) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py')) paths = sys_path_with_modifications(Evaluator(grammar), p.module) assert '/tmp/.buildout/eggs/important_package.egg' in paths @@ -83,7 +83,7 @@ def test_path_from_sys_path_assignment(): if __name__ == '__main__': sys.exit(important_package.main())""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert 1 not in paths assert '/home/test/.buildout/eggs/important_package.egg' in paths diff --git a/test/test_evaluate/test_sys_path.py b/test/test_evaluate/test_sys_path.py index ba1edb25..f7ce0fab 100644 --- a/test/test_evaluate/test_sys_path.py +++ b/test/test_evaluate/test_sys_path.py @@ -5,14 +5,14 @@ import sys import pytest from jedi._compatibility import unicode -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.evaluate import sys_path, Evaluator def test_paths_from_assignment(): def paths(src): grammar = load_grammar() - stmt = Parser(grammar, unicode(src)).module.statements[0] + stmt = ParserWithRecovery(grammar, unicode(src)).module.statements[0] return set(sys_path._paths_from_assignment(Evaluator(grammar), stmt)) assert paths('sys.path[0:0] = ["a"]') == set(['a']) diff --git a/test/test_new_parser.py b/test/test_new_parser.py index 8684fbd4..e66591dc 100644 --- a/test/test_new_parser.py +++ b/test/test_new_parser.py @@ -1,11 +1,11 @@ from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar def test_basic_parsing(): def compare(string): """Generates the AST object and then regenerates the code.""" - assert Parser(load_grammar(), string).module.get_code() == string + assert ParserWithRecovery(load_grammar(), string).module.get_code() == string compare(u('\na #pass\n')) compare(u('wblabla* 1\t\n')) diff --git a/test/test_parser/test_get_code.py b/test/test_parser/test_get_code.py index 8dc83aff..be825208 100644 --- a/test/test_parser/test_get_code.py +++ b/test/test_parser/test_get_code.py @@ -3,7 +3,7 @@ import difflib import pytest from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar code_basic_features = u(''' """A mod docstring""" @@ -44,7 +44,7 @@ def diff_code_assert(a, b, n=4): def test_basic_parsing(): """Validate the parsing features""" - prs = Parser(load_grammar(), code_basic_features) + prs = ParserWithRecovery(load_grammar(), code_basic_features) diff_code_assert( code_basic_features, prs.module.get_code() @@ -53,7 +53,7 @@ def test_basic_parsing(): def test_operators(): src = u('5 * 3') - prs = Parser(load_grammar(), src) + prs = ParserWithRecovery(load_grammar(), src) diff_code_assert(src, prs.module.get_code()) @@ -82,7 +82,7 @@ def method_with_docstring(): """class docstr""" pass ''') - assert Parser(load_grammar(), s).module.get_code() == s + assert ParserWithRecovery(load_grammar(), s).module.get_code() == s def test_end_newlines(): @@ -92,7 +92,7 @@ def test_end_newlines(): line the parser needs. """ def test(source, end_pos): - module = Parser(load_grammar(), u(source)).module + module = ParserWithRecovery(load_grammar(), u(source)).module assert module.get_code() == source assert module.end_pos == end_pos diff --git a/test/test_parser/test_parser.py b/test/test_parser/test_parser.py index 31c92691..588bad1f 100644 --- a/test/test_parser/test_parser.py +++ b/test/test_parser/test_parser.py @@ -3,7 +3,7 @@ import sys import jedi from jedi._compatibility import u, is_py3 -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser.user_context import UserContextParser from jedi.parser import tree as pt from textwrap import dedent @@ -23,7 +23,7 @@ def test_user_statement_on_import(): class TestCallAndName(): def get_call(self, source): # Get the simple_stmt and then the first one. - simple_stmt = Parser(load_grammar(), u(source)).module.children[0] + simple_stmt = ParserWithRecovery(load_grammar(), u(source)).module.children[0] return simple_stmt.children[0] def test_name_and_call_positions(self): @@ -58,7 +58,7 @@ class TestCallAndName(): class TestSubscopes(): def get_sub(self, source): - return Parser(load_grammar(), u(source)).module.subscopes[0] + return ParserWithRecovery(load_grammar(), u(source)).module.subscopes[0] def test_subscope_names(self): name = self.get_sub('class Foo: pass').name @@ -74,7 +74,7 @@ class TestSubscopes(): class TestImports(): def get_import(self, source): - return Parser(load_grammar(), source).module.imports[0] + return ParserWithRecovery(load_grammar(), source).module.imports[0] def test_import_names(self): imp = self.get_import(u('import math\n')) @@ -89,13 +89,13 @@ class TestImports(): def test_module(): - module = Parser(load_grammar(), u('asdf'), 'example.py').module + module = ParserWithRecovery(load_grammar(), u('asdf'), 'example.py').module name = module.name assert str(name) == 'example' assert name.start_pos == (1, 0) assert name.end_pos == (1, 7) - module = Parser(load_grammar(), u('asdf')).module + module = ParserWithRecovery(load_grammar(), u('asdf')).module name = module.name assert str(name) == '' assert name.start_pos == (1, 0) @@ -108,7 +108,7 @@ def test_end_pos(): def func(): y = None ''')) - parser = Parser(load_grammar(), s) + parser = ParserWithRecovery(load_grammar(), s) scope = parser.module.subscopes[0] assert scope.start_pos == (3, 0) assert scope.end_pos == (5, 0) @@ -121,7 +121,7 @@ def test_carriage_return_statements(): # this is a namespace package ''')) source = source.replace('\n', '\r\n') - stmt = Parser(load_grammar(), source).module.statements[0] + stmt = ParserWithRecovery(load_grammar(), source).module.statements[0] assert '#' not in stmt.get_code() @@ -129,7 +129,7 @@ def test_incomplete_list_comprehension(): """ Shouldn't raise an error, same bug as #418. """ # With the old parser this actually returned a statement. With the new # parser only valid statements generate one. - assert Parser(load_grammar(), u('(1 for def')).module.statements == [] + assert ParserWithRecovery(load_grammar(), u('(1 for def')).module.statements == [] def test_hex_values_in_docstring(): @@ -141,7 +141,7 @@ def test_hex_values_in_docstring(): return 1 ''' - doc = Parser(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc + doc = ParserWithRecovery(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc if is_py3: assert doc == '\xff' else: @@ -160,7 +160,7 @@ def test_error_correction_with(): def test_newline_positions(): - endmarker = Parser(load_grammar(), u('a\n')).module.children[-1] + endmarker = ParserWithRecovery(load_grammar(), u('a\n')).module.children[-1] assert endmarker.end_pos == (2, 0) new_line = endmarker.get_previous() assert new_line.start_pos == (1, 1) @@ -174,7 +174,7 @@ def test_end_pos_error_correction(): end_pos, even if something breaks in the parser (error correction). """ s = u('def x():\n .') - m = Parser(load_grammar(), s).module + m = ParserWithRecovery(load_grammar(), s).module func = m.children[0] assert func.type == 'funcdef' # This is not exactly correct, but ok, because it doesn't make a difference @@ -191,7 +191,7 @@ def test_param_splitting(): def check(src, result): # Python 2 tuple params should be ignored for now. grammar = load_grammar('grammar%s.%s' % sys.version_info[:2]) - m = Parser(grammar, u(src)).module + m = ParserWithRecovery(grammar, u(src)).module if is_py3: assert not m.subscopes else: @@ -211,5 +211,5 @@ def test_unicode_string(): def test_backslash_dos_style(): grammar = load_grammar() - m = Parser(grammar, u('\\\r\n')).module + m = ParserWithRecovery(grammar, u('\\\r\n')).module assert m diff --git a/test/test_parser/test_parser_tree.py b/test/test_parser/test_parser_tree.py index 28c7d271..57ea8259 100644 --- a/test/test_parser/test_parser_tree.py +++ b/test/test_parser/test_parser_tree.py @@ -5,7 +5,7 @@ from textwrap import dedent import pytest from jedi._compatibility import u, unicode -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt @@ -27,7 +27,7 @@ class TestsFunctionAndLambdaParsing(object): @pytest.fixture(params=FIXTURES) def node(self, request): - parsed = Parser(load_grammar(), dedent(u(request.param[0]))) + parsed = ParserWithRecovery(load_grammar(), dedent(u(request.param[0]))) request.keywords['expected'] = request.param[1] return parsed.module.subscopes[0] diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index f038cc8d..e53f85a6 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -7,7 +7,7 @@ import pytest from jedi._compatibility import u, is_py3 from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT -from jedi.parser import Parser, load_grammar, tokenize +from jedi.parser import ParserWithRecovery, load_grammar, tokenize from ..helpers import unittest @@ -15,7 +15,7 @@ from ..helpers import unittest class TokenTest(unittest.TestCase): def test_end_pos_one_line(self): - parsed = Parser(load_grammar(), dedent(u(''' + parsed = ParserWithRecovery(load_grammar(), dedent(u(''' def testit(): a = "huhu" '''))) @@ -23,7 +23,7 @@ class TokenTest(unittest.TestCase): assert tok.end_pos == (3, 14) def test_end_pos_multi_line(self): - parsed = Parser(load_grammar(), dedent(u(''' + parsed = ParserWithRecovery(load_grammar(), dedent(u(''' def testit(): a = """huhu asdfasdf""" + "h" @@ -108,7 +108,7 @@ class TokenTest(unittest.TestCase): ] for s in string_tokens: - parsed = Parser(load_grammar(), u('''a = %s\n''' % s)) + parsed = ParserWithRecovery(load_grammar(), u('''a = %s\n''' % s)) simple_stmt = parsed.module.children[0] expr_stmt = simple_stmt.children[0] assert len(expr_stmt.children) == 3 diff --git a/test/test_regression.py b/test/test_regression.py index 94132acd..9ddfb381 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -15,7 +15,7 @@ from jedi._compatibility import u from jedi import Script from jedi import api from jedi.evaluate import imports -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar #jedi.set_debug_function() @@ -102,7 +102,7 @@ class TestRegression(TestCase): def test_end_pos_line(self): # jedi issue #150 s = u("x()\nx( )\nx( )\nx ( )") - parser = Parser(load_grammar(), s) + parser = ParserWithRecovery(load_grammar(), s) for i, s in enumerate(parser.module.statements): assert s.end_pos == (i + 1, i + 3) From 57918608614da5ec84a7fc947ef6cab81466504a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 20 Dec 2015 22:57:24 +0100 Subject: [PATCH 25/26] Actual forward reference annotations are working pretty smooth now. --- jedi/api/__init__.py | 6 ++---- jedi/evaluate/__init__.py | 2 ++ jedi/evaluate/pep0484.py | 2 +- jedi/evaluate/stdlib.py | 4 ++-- test/test_evaluate/test_annotations.py | 13 +++++++++---- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 0097821f..d6df3878 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -322,12 +322,10 @@ class Script(object): @memoize_default() def _get_under_cursor_stmt(self, cursor_txt, start_pos=None): - node = Parser(self._grammar, cursor_txt, 'eval_input').get_parsed_node() - if node is None: + stmt = Parser(self._grammar, cursor_txt, 'eval_input').get_parsed_node() + if stmt is None: return None - stmt = node.children[0] - user_stmt = self._parser.user_stmt() if user_stmt is None: # Set the start_pos to a pseudo position, that doesn't exist but diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 8167a5e6..6f63acc8 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -305,6 +305,8 @@ class Evaluator(object): types = set(chain.from_iterable(self.find_types(typ, next_name) for typ in types)) types = types + elif element.type == 'eval_input': + types = self._eval_element_not_cached(element.children[0]) else: types = precedence.calculate_children(self, element.children) debug.dbg('eval_element result %s', types) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 1896b8d0..d8948f47 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -32,7 +32,7 @@ def _evaluate_for_annotation(evaluator, annotation): for definition in evaluator.eval_element(annotation): if (isinstance(definition, CompiledObject) and isinstance(definition.obj, str)): - p = Parser(load_grammar(), definition.obj, start='expr') + p = Parser(load_grammar(), definition.obj, start='eval_input') element = p.get_parsed_node() if element is None: debug.warning('Annotation not parsed: %s' % definition.obj) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index acc3d901..0d4ebf1c 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -14,7 +14,7 @@ from jedi.common import unite from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate import iterable -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.parser import tree from jedi import debug from jedi.evaluate import precedence @@ -243,7 +243,7 @@ def collections_namedtuple(evaluator, obj, arguments): ) # Parse source - generated_class = Parser(evaluator.grammar, unicode(source)).module.subscopes[0] + generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0] return set([er.Class(evaluator, generated_class)]) diff --git a/test/test_evaluate/test_annotations.py b/test/test_evaluate/test_annotations.py index a1717cac..7897ffb0 100644 --- a/test/test_evaluate/test_annotations.py +++ b/test/test_evaluate/test_annotations.py @@ -41,14 +41,19 @@ def test_simple_annotations(): @pytest.mark.parametrize('reference', [ 'assert 1', '1', - 'lambda: 3', 'def x(): pass', '1, 2', r'1\n' ]) def test_illegal_forward_references(reference): - source = """ - def foo(bar: "%s"): - bar""" % reference + source = 'def foo(bar: "%s"): bar' % reference assert not jedi.Script(source).goto_definitions() + + +def test_lambda_forward_references(): + source = 'def foo(bar: "lambda: 3"): bar' + + # For now just receiving the 3 is ok. I'm doubting that this is what we + # want. We also execute functions. Should we only execute classes? + assert jedi.Script(source).goto_definitions() From c15551ccc1c304fa2682c1d7a4ad7c52a7c53050 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 20 Dec 2015 23:11:52 +0100 Subject: [PATCH 26/26] Errortokens should also make the parser fail in the normal parser. --- jedi/parser/__init__.py | 4 +++- jedi/parser/tokenize.py | 2 +- test/test_evaluate/test_annotations.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 21a3315e..534386aa 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -156,7 +156,9 @@ class Parser(object): def _tokenize(self, tokenizer): for typ, value, start_pos, prefix in tokenizer: - if typ == OP: + if typ == ERRORTOKEN: + raise Parser.ParserError + elif typ == OP: typ = token.opmap[value] yield typ, value, prefix, start_pos diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index ebc8a4ee..ac3cabec 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -286,7 +286,7 @@ def generate_tokens(readline): if new_line: end_pos = lnum + 1, 0 else: - end_pos = lnum, max - 1 + end_pos = lnum, max # As the last position we just take the maximally possible position. We # remove -1 for the last new line. for indent in indents[1:]: diff --git a/test/test_evaluate/test_annotations.py b/test/test_evaluate/test_annotations.py index 7897ffb0..67fe84e1 100644 --- a/test/test_evaluate/test_annotations.py +++ b/test/test_evaluate/test_annotations.py @@ -51,6 +51,7 @@ def test_illegal_forward_references(reference): assert not jedi.Script(source).goto_definitions() +@pytest.mark.skipif('sys.version_info[0] < 3') def test_lambda_forward_references(): source = 'def foo(bar: "lambda: 3"): bar'