From daeee4ba0c342c70681b73c1d2b544cb50c45b10 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Dec 2015 01:00:20 +0100 Subject: [PATCH 1/9] simple typehints in comments --- jedi/evaluate/finder.py | 3 +++ jedi/evaluate/pep0484.py | 28 ++++++++++++++++++++++++++++ jedi/parser/tree.py | 9 +++++++++ test/completion/pep0484.py | 24 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 5003848c..1d95e3f7 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -355,6 +355,9 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var + pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt) + if pep0484types: + return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) if check_instance is not None: diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 08588d71..519a217d 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -28,6 +28,7 @@ from jedi.common import unite from jedi.evaluate import compiled from jedi import debug from jedi import _compatibility +import re def _evaluate_for_annotation(evaluator, annotation): @@ -136,3 +137,30 @@ def get_types_for_typing_module(evaluator, typ, node): result = evaluator.execute_evaluated(factory, compiled_classname, args) return result + + +def find_type_from_comment_hint(evaluator, stmt): + try: + stmtpos = stmt.parent.children.index(stmt) + except ValueError: + return [] + try: + next_sibling = stmt.parent.children[stmtpos + 1] + except IndexError: + return [] + if not isinstance(next_sibling, tree.Whitespace): + return [] + comment = next_sibling.get_pre_comment() + if comment is None: + return [] + match = re.match(r"\s*type:\s*([^#]*)", comment) + if not match: + return [] + start_pos = (next_sibling.start_pos[0], + next_sibling.start_pos[1] - len(comment)) + annotation = tree.String( + tree.zero_position_modifier, + repr(str(match.group(1).strip())), + start_pos) + annotation.parent = stmt.parent + return _evaluate_for_annotation(evaluator, annotation) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 89c37841..3f8549d4 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -280,6 +280,15 @@ class Leaf(Base): def nodes_to_execute(self, last_added=False): return [] + def get_pre_comment(self): + """ + returns comment before this leaf, excluding #, or None if no comment + """ + match = re.match(r"\s*#(.*)$", self.prefix) + if match: + return match.group(1) + return None + @utf8_repr def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.value) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index fc08460f..1990cda0 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,3 +157,27 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x + +# python >= 2.6 + +x = 3 # type: str +#? str() +x + +y = 3 # type: str but I write more +#? int() +y + +z = 3 # type: str # I comment more +#? str() +z + +class BB: pass + +def test(a, b): + a = a # type: BB + c = a # type: str + #? BB() + a + #? str() + c From a658f7940c7c133d81c232434c6eb921d65a6449 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 16:02:59 +0100 Subject: [PATCH 2/9] typehints for variables in comments --- jedi/evaluate/pep0484.py | 18 ++------- jedi/parser/tree.py | 59 ++++++++++++++++++++++++----- test/completion/pep0484.py | 24 ------------ test/completion/pep0484_comments.py | 39 +++++++++++++++++++ 4 files changed, 91 insertions(+), 49 deletions(-) create mode 100644 test/completion/pep0484_comments.py diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 519a217d..260e15e4 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -140,27 +140,15 @@ def get_types_for_typing_module(evaluator, typ, node): def find_type_from_comment_hint(evaluator, stmt): - try: - stmtpos = stmt.parent.children.index(stmt) - except ValueError: - return [] - try: - next_sibling = stmt.parent.children[stmtpos + 1] - except IndexError: - return [] - if not isinstance(next_sibling, tree.Whitespace): - return [] - comment = next_sibling.get_pre_comment() + comment = stmt.get_following_comment_same_line() if comment is None: return [] - match = re.match(r"\s*type:\s*([^#]*)", comment) + match = re.match(r"^#\s*type:\s*([^#]*)", comment) if not match: return [] - start_pos = (next_sibling.start_pos[0], - next_sibling.start_pos[1] - len(comment)) annotation = tree.String( tree.zero_position_modifier, repr(str(match.group(1).strip())), - start_pos) + stmt.start_pos) annotation.parent = stmt.parent return _evaluate_for_annotation(evaluator, annotation) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 3f8549d4..1906d364 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -239,13 +239,35 @@ class Leaf(Base): else: node = c[i - 1] break - while True: try: node = node.children[-1] except AttributeError: # A Leaf doesn't have children. return node + def get_next(self): + """ + Returns the next leaf in the parser tree. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + try: + node = c[i + 1] + except IndexError: + node = node.parent + if node.parent is None: + raise IndexError('Cannot access the next element of the last one.') + else: + break + while True: + try: + node = node.children[0] + except AttributeError: # A Leaf doesn't have children. + return node + + def get_code(self, normalized=False): if normalized: return self.value @@ -264,6 +286,7 @@ class Leaf(Base): except IndexError: return None + def prev_sibling(self): """ The node/leaf immediately preceding the invocant in their parent's @@ -277,18 +300,10 @@ class Leaf(Base): return None return self.parent.children[i - 1] + def nodes_to_execute(self, last_added=False): return [] - def get_pre_comment(self): - """ - returns comment before this leaf, excluding #, or None if no comment - """ - match = re.match(r"\s*#(.*)$", self.prefix) - if match: - return match.group(1) - return None - @utf8_repr def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.value) @@ -497,6 +512,30 @@ class BaseNode(Base): except AttributeError: return self.children[0] + def last_leaf(self): + try: + return self.children[-1].first_leaf() + except AttributeError: + return self.children[-1] + + def get_following_comment_same_line(self): + """ + returns (as string) any comment that appears on the same line, + after the node, including the # + """ + try: + whitespace = self.last_leaf().get_next().prefix + except AttributeError: + return None + if "#" not in whitespace: + return None + comment = whitespace[whitespace.index("#"):] + if "\r" in comment: + comment = comment[:comment.index("\r")] + if "\n" in comment: + comment = comment[:comment.index("\n")] + return comment + @utf8_repr def __repr__(self): code = self.get_code().replace('\n', ' ').strip() diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 1990cda0..fc08460f 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,27 +157,3 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x - -# python >= 2.6 - -x = 3 # type: str -#? str() -x - -y = 3 # type: str but I write more -#? int() -y - -z = 3 # type: str # I comment more -#? str() -z - -class BB: pass - -def test(a, b): - a = a # type: BB - c = a # type: str - #? BB() - a - #? str() - c diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py new file mode 100644 index 00000000..54ad38c6 --- /dev/null +++ b/test/completion/pep0484_comments.py @@ -0,0 +1,39 @@ +a = 3 # type: str +#? str() +a + +b = 3 # type: str but I write more +#? int() +b + +c = 3 # type: str # I comment more +#? str() +c + +d = "It should not read comments from the next line" +# type: int +#? str() +d + +# type: int +e = "It should not read comments from the previous line" +#? str() +e + +class BB: pass + +def test(a, b): + a = a # type: BB + c = a # type: str + d = a + # type: str + e = a # type: str # Should ignore long whitespace + + #? BB() + a + #? str() + c + #? BB() + d + #? str() + e From 8b28678d1988f5192953a141730661ec52358625 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:04:19 +0100 Subject: [PATCH 3/9] support tuple-assignment --- jedi/evaluate/finder.py | 2 +- jedi/evaluate/pep0484.py | 30 ++++++++++++++++++++++++++--- jedi/parser/tree.py | 6 +++++- test/completion/pep0484_comments.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 1d95e3f7..5cf45434 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -355,7 +355,7 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var - pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt) + pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt, name) if pep0484types: return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 260e15e4..69162178 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -31,10 +31,20 @@ from jedi import _compatibility import re -def _evaluate_for_annotation(evaluator, annotation): +def _evaluate_for_annotation(evaluator, annotation, index=None): + """ + Evaluates a string-node, looking for an annotation + If index is not None, the annotation is expected to be a tuple + and we're interested in that index + """ if annotation is not None: definitions = evaluator.eval_element( _fix_forward_reference(evaluator, annotation)) + if index is not None: + definitions = list(itertools.chain.from_iterable( + definition.py__getitem__(index) for definition in definitions + if definition.type == 'tuple' and + len(list(definition.py__iter__())) >= index)) return list(itertools.chain.from_iterable( evaluator.execute(d) for d in definitions)) else: @@ -139,7 +149,21 @@ def get_types_for_typing_module(evaluator, typ, node): return result -def find_type_from_comment_hint(evaluator, stmt): +def find_type_from_comment_hint(evaluator, stmt, name): + index = None + if stmt.children[0].type == "testlist_star_expr": + # something like "a, b = 1, 2" + leftside = stmt.children[0] + index = 0 + for child in leftside.children: + if child == name: + break + if child.type == "operator": + continue + index += 1 + else: + return [] + comment = stmt.get_following_comment_same_line() if comment is None: return [] @@ -151,4 +175,4 @@ def find_type_from_comment_hint(evaluator, stmt): repr(str(match.group(1).strip())), stmt.start_pos) annotation.parent = stmt.parent - return _evaluate_for_annotation(evaluator, annotation) + return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 1906d364..0625a98a 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -514,7 +514,7 @@ class BaseNode(Base): def last_leaf(self): try: - return self.children[-1].first_leaf() + return self.children[-1].last_leaf() except AttributeError: return self.children[-1] @@ -527,6 +527,10 @@ class BaseNode(Base): whitespace = self.last_leaf().get_next().prefix except AttributeError: return None + except ValueError: + # in some particular cases, the tree doesn't seem to be linked + # correctly + return None if "#" not in whitespace: return None comment = whitespace[whitespace.index("#"):] diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 54ad38c6..e49717e6 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -37,3 +37,32 @@ def test(a, b): d #? str() e + +a,b = 1, 2 # type: str, float +#? str() +a +#? float() +b + +class Employee: + pass + +from typing import List +x = [] # type: List[Employee] +#? Employee() +x[1] +x, y, z = [], [], [] # type: List[int], List[int], List[str] +#? int() +y[2] +x, y, z = [], [], [] # type: (List[float], List[float], List[BB]) +for zi in z: + #? BB() + zi + +x = [ + 1, + 2, +] # type: List[str] + +#? str() +x[1] From 3a1b2e7104da044433b2822526d923d17e67cca7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:37:03 +0100 Subject: [PATCH 4/9] add support for 'for-assignment' hints --- jedi/evaluate/finder.py | 7 ++++++- jedi/evaluate/pep0484.py | 23 ++++++++++++++++------- jedi/parser/tree.py | 5 ++++- test/completion/pep0484_comments.py | 11 +++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 5cf45434..1d98d4dd 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -304,6 +304,10 @@ class NameFinder(object): @memoize_default(set(), evaluator_is_first_arg=True) def _name_to_types(evaluator, name, scope): typ = name.get_definition() + if typ.isinstance(tree.ForStmt): + types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) + if types: + return types if typ.isinstance(tree.ForStmt, tree.CompFor): container_types = evaluator.eval_element(typ.children[3]) for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) @@ -355,7 +359,8 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var - pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt, name) + pep0484types = \ + pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name) if pep0484types: return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 69162178..91f5251e 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -149,13 +149,22 @@ def get_types_for_typing_module(evaluator, typ, node): return result -def find_type_from_comment_hint(evaluator, stmt, name): +def find_type_from_comment_hint_for(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[1], name) + + +def find_type_from_comment_hint_assign(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[0], name) + + +def _find_type_from_comment_hint(evaluator, node, varlist, name): index = None - if stmt.children[0].type == "testlist_star_expr": + if varlist.type in ("testlist_star_expr", "exprlist"): # something like "a, b = 1, 2" - leftside = stmt.children[0] index = 0 - for child in leftside.children: + for child in varlist.children: if child == name: break if child.type == "operator": @@ -164,7 +173,7 @@ def find_type_from_comment_hint(evaluator, stmt, name): else: return [] - comment = stmt.get_following_comment_same_line() + comment = node.get_following_comment_same_line() if comment is None: return [] match = re.match(r"^#\s*type:\s*([^#]*)", comment) @@ -173,6 +182,6 @@ def find_type_from_comment_hint(evaluator, stmt, name): annotation = tree.String( tree.zero_position_modifier, repr(str(match.group(1).strip())), - stmt.start_pos) - annotation.parent = stmt.parent + node.start_pos) + annotation.parent = node.parent return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 0625a98a..cf33faca 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -524,7 +524,10 @@ class BaseNode(Base): after the node, including the # """ try: - whitespace = self.last_leaf().get_next().prefix + if self.isinstance(ForStmt): + whitespace = self.children[5].first_leaf().prefix + else: + whitespace = self.last_leaf().get_next().prefix except AttributeError: return None except ValueError: diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index e49717e6..606bb09a 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -66,3 +66,14 @@ x = [ #? str() x[1] + + +for bar in foo(): # type: str + #? str() + bar + +for bar, baz in foo(): # type: int, float + #? int() + bar + #? float() + baz From 641fb8077398100179d8dc8a2711f50eac9566fa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:52:21 +0100 Subject: [PATCH 5/9] add support for 'with-assignment' hints --- jedi/evaluate/finder.py | 4 ++++ jedi/evaluate/pep0484.py | 7 +++++++ jedi/parser/tree.py | 2 ++ test/completion/pep0484_comments.py | 7 +++++++ 4 files changed, 20 insertions(+) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 1d98d4dd..4114518a 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -308,6 +308,10 @@ def _name_to_types(evaluator, name, scope): types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) if types: return types + if typ.isinstance(tree.WithStmt): + types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name) + if types: + return types if typ.isinstance(tree.ForStmt, tree.CompFor): container_types = evaluator.eval_element(typ.children[3]) for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 91f5251e..aeb57735 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -154,6 +154,13 @@ def find_type_from_comment_hint_for(evaluator, node, name): _find_type_from_comment_hint(evaluator, node, node.children[1], name) +def find_type_from_comment_hint_with(evaluator, node, name): + assert len(node.children[1].children) == 3, \ + "Can only be here when children[1] is 'foo() as f'" + return _find_type_from_comment_hint( + evaluator, node, node.children[1].children[2], name) + + def find_type_from_comment_hint_assign(evaluator, node, name): return \ _find_type_from_comment_hint(evaluator, node, node.children[0], name) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index cf33faca..133b75ae 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -526,6 +526,8 @@ class BaseNode(Base): try: if self.isinstance(ForStmt): whitespace = self.children[5].first_leaf().prefix + elif self.isinstance(WithStmt): + whitespace = self.children[3].first_leaf().prefix else: whitespace = self.last_leaf().get_next().prefix except AttributeError: diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 606bb09a..8e907065 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -77,3 +77,10 @@ for bar, baz in foo(): # type: int, float bar #? float() baz + +with foo(): # type: int + ... + +with foo() as f: # type: str + #? str() + f From a9ebe71c64f17d6522ec6ad8683ba50728dbeecc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:54:20 +0100 Subject: [PATCH 6/9] add some tests to show that type-hints on the next line don't work --- test/completion/pep0484_comments.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 8e907065..6b8b59bd 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -78,9 +78,23 @@ for bar, baz in foo(): # type: int, float #? float() baz +for bar, baz in foo(): + # type: str, str + """ type hinting on next line should not work """ + #? + bar + #? + baz + with foo(): # type: int ... with foo() as f: # type: str #? str() f + +with foo() as f: + # type: str + """ type hinting on next line should not work """ + #? + f From 71ab855802db983e53f8a46c086d51cfff01fb41 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:56:14 +0100 Subject: [PATCH 7/9] update list of things that are completed --- jedi/evaluate/pep0484.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index aeb57735..c5d7a169 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -9,13 +9,13 @@ v Function parameter annotations with builtin/custom type classes v Function returntype annotations with builtin/custom type classes v Function parameter annotations with strings (forward reference) v Function return type annotations with strings (forward reference) -x Local variable type hints +v Local variable type hints v Assigned types: `Url = str\ndef get(url:Url) -> str:` -x Type hints in `with` statements +v Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` -x support for type hint comments `# type: (int, str) -> int`. See comment from - Guido https://github.com/davidhalter/jedi/issues/662 +x support for type hint comments for functions, `# type: (int, str) -> int`. + See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ import itertools From 4fe710d4d3507ae859f12e0182888b3c7c7a6238 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 18:02:11 +0100 Subject: [PATCH 8/9] more tests --- test/completion/pep0484_comments.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 6b8b59bd..7707fcc3 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -98,3 +98,8 @@ with foo() as f: """ type hinting on next line should not work """ #? f + +aaa = some_extremely_long_function_name_that_doesnt_leave_room_for_hints() \ + # type: float # We should be able to put hints on the next line with a \ +#? float() +aaa From 795a72a8660fd4e503df1be597497e3f8030e88b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 18:13:47 +0100 Subject: [PATCH 9/9] add `typing.cast()` to the todo list --- jedi/evaluate/pep0484.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index c5d7a169..dc951aaf 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -14,6 +14,7 @@ v Assigned types: `Url = str\ndef get(url:Url) -> str:` v Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` +x support for typing.cast() operator x support for type hint comments for functions, `# type: (int, str) -> int`. See comment from Guido https://github.com/davidhalter/jedi/issues/662 """