From f5a31ad78e66b8d04dae94c7368b3aab8679302e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Dec 2015 00:03:08 +0100 Subject: [PATCH 01/60] first try at the typing library --- jedi/evaluate/iterable.py | 14 ++++++++++- jedi/evaluate/pep0484.py | 44 ++++++++++++++++++++++++++++++--- jedi/evaluate/representation.py | 18 ++++++++++++++ test/completion/pep0484.py | 23 +++++++++++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index b11b95a1..5b0871fd 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -29,6 +29,7 @@ from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.cache import CachedMetaClass, memoize_default from jedi.evaluate import analysis +from jedi.evaluate import pep0484 class IterableWrapper(tree.Base): @@ -560,12 +561,23 @@ def py__iter__types(evaluator, types, node=None): def py__getitem__(evaluator, types, index, node): + from jedi.evaluate.representation import Class result = set() # Index handling. if isinstance(index, (compiled.CompiledObject, Slice)): index = index.obj + # special case: PEP0484 typing module, see + # https://github.com/davidhalter/jedi/issues/663 + for typ in list(types): + if isinstance(typ, Class): + replacementclass = \ + pep0484.get_typing_replacement_class(evaluator, typ) + if replacementclass: + types.remove(typ) + result |= replacementclass.py__getitem__(index) + if type(index) not in (float, int, str, unicode, slice): # If the index is not clearly defined, we have to get all the # possiblities. @@ -588,7 +600,7 @@ def py__getitem__(evaluator, types, index, node): except IndexError: result |= py__iter__types(evaluator, set([typ])) except KeyError: - # Must be a dict. Lists don't raise IndexErrors. + # Must be a dict. Lists don't raise KeyErrors. result |= typ.dict_values() return result diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 4eec2fed..761886a4 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -20,9 +20,10 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from from itertools import chain -from jedi.parser import Parser, load_grammar, ParseError +from jedi.parser import Parser, load_grammar, ParseError, tree, ParserWithRecovery from jedi.evaluate.cache import memoize_default -from jedi.evaluate.compiled import CompiledObject +from jedi.evaluate import compiled +from textwrap import dedent from jedi import debug @@ -30,7 +31,7 @@ def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: definitions = set() for definition in evaluator.eval_element(annotation): - if (isinstance(definition, CompiledObject) and + if (isinstance(definition, compiled.CompiledObject) and isinstance(definition.obj, str)): try: p = Parser(load_grammar(), definition.obj, start='eval_input') @@ -60,3 +61,40 @@ def follow_param(evaluator, param): def find_return_types(evaluator, func): annotation = func.py__annotations__().get("return", None) return _evaluate_for_annotation(evaluator, annotation) + + +# TODO: Memoize +def get_typing_replacement_module(): + """ + The idea is to return our jedi replacement for the PEP-0484 typing module + as discussed at https://github.com/davidhalter/jedi/issues/663 + """ + + code = dedent(""" + from collections import abc + + class MakeSequence: + def __getitem__(self, indextype): + class Sequence(abc.Sequence): + def __getitem__(self) -> indextype: + pass + return Sequence + """) + p = ParserWithRecovery(load_grammar(), code) + return p.module + + +def get_typing_replacement_class(evaluator, typ): + if not typ.base.get_parent_until(tree.Module).name.value == "typing": + return None + # we assume that any class using [] in a module called + # "typing" with a name for which we have a replacement + # should be replaced by that class. This is not 100% + # airtight but I don't have a better idea to check that it's + # actually the PEP-0484 typing module and not some other + typing = get_typing_replacement_module() + types = evaluator.find_types(typing, "Make" + typ.name.value) + if not types: + return None + else: + return list(types)[0] diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index c460efd2..35359dbd 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -469,6 +469,24 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): def py__class__(self): return compiled.create(self._evaluator, type) + def py__getitem__(self, index): + instances = self.py__call__(()) + assert len(instances) == 1 + instance = list(instances)[0] + try: + method = instance.get_subscope_by_name('__getitem__') + except KeyError: + debug.warning('No __getitem__, cannot access the array.') + return set() + else: + # it feels like this should be handled somewhere else, + # not sure where though + if isinstance(index, type(self)): + index_obj = index + else: + index_obj = compiled.create(self._evaluator, index) + return instance._evaluator.execute_evaluated(method, index_obj) + @property def params(self): return self.get_subscope_by_name('__init__').params diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index fc08460f..ba9044bc 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,3 +157,26 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x + +import typing +def we_can_has_sequence( + p: typing.Sequence[int], + q: typing.Sequence[B], + r: "typing.Sequence[int]", + s: typing.Sequence["int"]): + #? ["count"] + p.c + #? int() + p[1] + #? ["count"] + q.c + #? B() + q[1] + #? ["count"] + r.c + #? int() + r[1] + #? ["count"] + s.c + #? int() + s[1] From 52cc721f450eaa438dee6a28006a19a6b3420b14 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 16:39:17 +0100 Subject: [PATCH 02/60] made typing classes inheritable; added MutableSequence and List --- jedi/evaluate/iterable.py | 12 ++++++---- jedi/evaluate/jedi_typing.py | 25 ++++++++++++++++++++ jedi/evaluate/pep0484.py | 42 +++++++++++++++++++-------------- jedi/evaluate/representation.py | 18 -------------- test/completion/pep0484.py | 14 ++++++++++- 5 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 jedi/evaluate/jedi_typing.py diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 5b0871fd..d6f0da0f 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -572,11 +572,15 @@ def py__getitem__(evaluator, types, index, node): # https://github.com/davidhalter/jedi/issues/663 for typ in list(types): if isinstance(typ, Class): - replacementclass = \ - pep0484.get_typing_replacement_class(evaluator, typ) - if replacementclass: + typing_module_types = \ + pep0484.get_types_for_typing_module(evaluator, typ, index) + if typing_module_types is not None: types.remove(typ) - result |= replacementclass.py__getitem__(index) + result |= typing_module_types + + if not types: + # all consumed by special cases + return result if type(index) not in (float, int, str, unicode, slice): # If the index is not clearly defined, we have to get all the diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py new file mode 100644 index 00000000..9538624a --- /dev/null +++ b/jedi/evaluate/jedi_typing.py @@ -0,0 +1,25 @@ +""" +This module is not intended to be used in jedi, rather it will be fed to the +jedi-parser to replace classes in the typing module +""" + +from collections import abc + + +def factory(typing_name, indextype): + class Sequence(abc.Sequence): + def __getitem__(self) -> indextype: + pass + + class MutableSequence(Sequence, abc.MutableSequence): + pass + + class List(MutableSequence, list): + pass + + dct = { + "Sequence": Sequence, + "MutableSequence": MutableSequence, + "List": List, + } + return dct[typing_name] diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 761886a4..d3fb46e0 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -20,10 +20,11 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from from itertools import chain -from jedi.parser import Parser, load_grammar, ParseError, tree, ParserWithRecovery +import os +from jedi.parser import \ + Parser, load_grammar, ParseError, tree, ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi.evaluate import compiled -from textwrap import dedent from jedi import debug @@ -34,7 +35,8 @@ def _evaluate_for_annotation(evaluator, annotation): if (isinstance(definition, compiled.CompiledObject) and isinstance(definition.obj, str)): try: - p = Parser(load_grammar(), definition.obj, start='eval_input') + p = Parser( + load_grammar(), definition.obj, start='eval_input') element = p.get_parsed_node() except ParseError: debug.warning('Annotation not parsed: %s' % definition.obj) @@ -70,21 +72,15 @@ def get_typing_replacement_module(): as discussed at https://github.com/davidhalter/jedi/issues/663 """ - code = dedent(""" - from collections import abc - - class MakeSequence: - def __getitem__(self, indextype): - class Sequence(abc.Sequence): - def __getitem__(self) -> indextype: - pass - return Sequence - """) + typing_path = os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) + with open(typing_path) as f: + code = f.read() p = ParserWithRecovery(load_grammar(), code) return p.module -def get_typing_replacement_class(evaluator, typ): +def get_types_for_typing_module(evaluator, typ, index): + from jedi.evaluate.representation import Class if not typ.base.get_parent_until(tree.Module).name.value == "typing": return None # we assume that any class using [] in a module called @@ -93,8 +89,18 @@ def get_typing_replacement_class(evaluator, typ): # airtight but I don't have a better idea to check that it's # actually the PEP-0484 typing module and not some other typing = get_typing_replacement_module() - types = evaluator.find_types(typing, "Make" + typ.name.value) - if not types: - return None + factories = evaluator.find_types(typing, "factory") + assert len(factories) == 1 + factory = list(factories)[0] + assert factory + compiled_classname = compiled.create(evaluator, typ.name.value) + if isinstance(index, Class): + index_obj = index else: - return list(types)[0] + index_obj = compiled.create(evaluator, index) + result = \ + evaluator.execute_evaluated(factory, compiled_classname, index_obj) + if result: + return result + else: + return None diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 35359dbd..c460efd2 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -469,24 +469,6 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): def py__class__(self): return compiled.create(self._evaluator, type) - def py__getitem__(self, index): - instances = self.py__call__(()) - assert len(instances) == 1 - instance = list(instances)[0] - try: - method = instance.get_subscope_by_name('__getitem__') - except KeyError: - debug.warning('No __getitem__, cannot access the array.') - return set() - else: - # it feels like this should be handled somewhere else, - # not sure where though - if isinstance(index, type(self)): - index_obj = index - else: - index_obj = compiled.create(self._evaluator, index) - return instance._evaluator.execute_evaluated(method, index_obj) - @property def params(self): return self.get_subscope_by_name('__init__').params diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index ba9044bc..75742846 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -163,7 +163,9 @@ def we_can_has_sequence( p: typing.Sequence[int], q: typing.Sequence[B], r: "typing.Sequence[int]", - s: typing.Sequence["int"]): + s: typing.Sequence["int"], + t: typing.MutableSequence[dict], + u: typing.List[float]): #? ["count"] p.c #? int() @@ -180,3 +182,13 @@ def we_can_has_sequence( s.c #? int() s[1] + #? [] + s.a + #? ["append"] + t.a + #? dict() + t[1] + #? ["append"] + u.a + #? float() + u[1] From cc6bd7d1611b73486143a8e6eb75c1d725d64db0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 18:47:08 +0100 Subject: [PATCH 03/60] rework so that it also works withouty pep0484 type hints in jedi_typing.py --- jedi/evaluate/__init__.py | 3 +- jedi/evaluate/iterable.py | 61 +++++++++++++++++-------------- jedi/evaluate/jedi_typing.py | 5 +-- jedi/evaluate/pep0484.py | 69 +++++++++++++++++++++--------------- 4 files changed, 79 insertions(+), 59 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 29d73818..3c5b1fdf 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -391,8 +391,7 @@ class Evaluator(object): new_types = set() if trailer_op == '[': - for trailer_typ in iterable.create_index_types(self, node): - new_types |= iterable.py__getitem__(self, types, trailer_typ, trailer_op) + new_types |= iterable.py__getitem__(self, types, trailer) else: for typ in types: debug.dbg('eval_trailer: %s in scope %s', trailer, typ) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index d6f0da0f..957f9784 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -560,20 +560,17 @@ def py__iter__types(evaluator, types, node=None): return unite(py__iter__(evaluator, types, node)) -def py__getitem__(evaluator, types, index, node): +def py__getitem__(evaluator, types, trailer): from jedi.evaluate.representation import Class result = set() - # Index handling. - if isinstance(index, (compiled.CompiledObject, Slice)): - index = index.obj # special case: PEP0484 typing module, see # https://github.com/davidhalter/jedi/issues/663 for typ in list(types): if isinstance(typ, Class): typing_module_types = \ - pep0484.get_types_for_typing_module(evaluator, typ, index) + pep0484.get_types_for_typing_module(evaluator, typ, trailer) if typing_module_types is not None: types.remove(typ) result |= typing_module_types @@ -582,30 +579,40 @@ def py__getitem__(evaluator, types, index, node): # all consumed by special cases return result - if type(index) not in (float, int, str, unicode, slice): - # If the index is not clearly defined, we have to get all the - # possiblities. - for typ in list(types): - if isinstance(typ, Array) and typ.type == 'dict': - types.remove(typ) - result |= typ.dict_values() - return result | py__iter__types(evaluator, types) + trailer_op, node, trailer_cl = trailer.children[:3] + assert trailer_op == "[" + if trailer_cl != "]": + debug.warning("No support for complex indices: %s" % trailer) + return result - for typ in types: - # The actual getitem call. - try: - getitem = typ.py__getitem__ - except AttributeError: - analysis.add(evaluator, 'type-error-not-subscriptable', node, - message="TypeError: '%s' object is not subscriptable" % typ) - else: + for index in create_index_types(evaluator, node): + if isinstance(index, (compiled.CompiledObject, Slice)): + index = index.obj + + if type(index) not in (float, int, str, unicode, slice): + # If the index is not clearly defined, we have to get all the + # possiblities. + for typ in list(types): + if isinstance(typ, Array) and typ.type == 'dict': + types.remove(typ) + result |= typ.dict_values() + return result | py__iter__types(evaluator, types) + + for typ in types: + # The actual getitem call. try: - result |= getitem(index) - except IndexError: - result |= py__iter__types(evaluator, set([typ])) - except KeyError: - # Must be a dict. Lists don't raise KeyErrors. - result |= typ.dict_values() + getitem = typ.py__getitem__ + except AttributeError: + analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op, + message="TypeError: '%s' object is not subscriptable" % typ) + else: + try: + result |= getitem(index) + except IndexError: + result |= py__iter__types(evaluator, set([typ])) + except KeyError: + # Must be a dict. Lists don't raise KeyErrors. + result |= typ.dict_values() return result diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 9538624a..14c74570 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -8,10 +8,11 @@ from collections import abc def factory(typing_name, indextype): class Sequence(abc.Sequence): - def __getitem__(self) -> indextype: - pass + def __getitem__(self, index: int): + return indextype() class MutableSequence(Sequence, abc.MutableSequence): + def __setitem__(self, index: int, value: indextype): pass class List(MutableSequence, list): diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index d3fb46e0..771a7a14 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -21,8 +21,7 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from from itertools import chain import os -from jedi.parser import \ - Parser, load_grammar, ParseError, tree, ParserWithRecovery +from jedi.parser import Parser, load_grammar, ParseError, ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi.evaluate import compiled from jedi import debug @@ -31,28 +30,34 @@ from jedi import debug def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: definitions = set() + module = annotation.get_parent_until() for definition in evaluator.eval_element(annotation): - if (isinstance(definition, compiled.CompiledObject) and - isinstance(definition.obj, str)): - try: - p = Parser( - load_grammar(), definition.obj, start='eval_input') - element = p.get_parsed_node() - except ParseError: - debug.warning('Annotation not parsed: %s' % definition.obj) - else: - module = annotation.get_parent_until() - p.position_modifier.line = module.end_pos[0] - element.parent = module - definitions |= evaluator.eval_element(element) - else: - definitions.add(definition) + definitions |= \ + _fix_forward_reference(evaluator, definition, module) return list(chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] +def _fix_forward_reference(evaluator, item, module): + if (isinstance(item, compiled.CompiledObject) and + isinstance(item.obj, str)): + try: + p = Parser( + load_grammar(), item.obj, start='eval_input') + element = p.get_parsed_node() + except ParseError: + debug.warning('Annotation not parsed: %s' % item.obj) + return set() + else: + p.position_modifier.line = module.end_pos[0] + element.parent = module + return evaluator.eval_element(element) + else: + return {item} + + @memoize_default(None, evaluator_is_first_arg=True) def follow_param(evaluator, param): annotation = param.annotation() @@ -66,7 +71,7 @@ def find_return_types(evaluator, func): # TODO: Memoize -def get_typing_replacement_module(): +def _get_typing_replacement_module(): """ The idea is to return our jedi replacement for the PEP-0484 typing module as discussed at https://github.com/davidhalter/jedi/issues/663 @@ -79,27 +84,35 @@ def get_typing_replacement_module(): return p.module -def get_types_for_typing_module(evaluator, typ, index): - from jedi.evaluate.representation import Class - if not typ.base.get_parent_until(tree.Module).name.value == "typing": +def get_types_for_typing_module(evaluator, typ, trailer): + if not typ.base.get_parent_until().name.value == "typing": return None # we assume that any class using [] in a module called # "typing" with a name for which we have a replacement # should be replaced by that class. This is not 100% # airtight but I don't have a better idea to check that it's # actually the PEP-0484 typing module and not some other - typing = get_typing_replacement_module() + indextypes = evaluator.eval_element(trailer.children[1]) + if not isinstance(indextypes, set): + indextypes = {indextypes} + + module = trailer.get_parent_until() + dereferencedindextypes = set() + for indextyp in indextypes: + dereferencedindextypes |= \ + _fix_forward_reference(evaluator, indextyp, module) + + typing = _get_typing_replacement_module() factories = evaluator.find_types(typing, "factory") assert len(factories) == 1 factory = list(factories)[0] assert factory compiled_classname = compiled.create(evaluator, typ.name.value) - if isinstance(index, Class): - index_obj = index - else: - index_obj = compiled.create(evaluator, index) - result = \ - evaluator.execute_evaluated(factory, compiled_classname, index_obj) + + result = set() + for indextyp in dereferencedindextypes: + result |= \ + evaluator.execute_evaluated(factory, compiled_classname, indextyp) if result: return result else: From 85023a22aa1e5ca0c3e661eec607adf725b0a5bc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 19:07:13 +0100 Subject: [PATCH 04/60] Not implemented classes should not default to everything --- jedi/evaluate/pep0484.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 771a7a14..3730e7bc 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -21,7 +21,8 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from from itertools import chain import os -from jedi.parser import Parser, load_grammar, ParseError, ParserWithRecovery +from jedi.parser import \ + Parser, load_grammar, ParseError, ParserWithRecovery, tree from jedi.evaluate.cache import memoize_default from jedi.evaluate import compiled from jedi import debug @@ -107,13 +108,16 @@ def get_types_for_typing_module(evaluator, typ, trailer): assert len(factories) == 1 factory = list(factories)[0] assert factory + function_body_nodes = factory.children[4].children + valid_classnames = {child.name.value + for child in function_body_nodes + if isinstance(child, tree.Class)} + if typ.name.value not in valid_classnames: + return None compiled_classname = compiled.create(evaluator, typ.name.value) result = set() for indextyp in dereferencedindextypes: result |= \ evaluator.execute_evaluated(factory, compiled_classname, indextyp) - if result: - return result - else: - return None + return result From e688a498abee778128837baae43a0fcd1bb8e482 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 19:32:31 +0100 Subject: [PATCH 05/60] Add sets and iterable/iterator --- jedi/evaluate/jedi_typing.py | 31 ++++++++++++++++++++++++++++++- test/completion/pep0484.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 14c74570..8d66258f 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -7,7 +7,19 @@ from collections import abc def factory(typing_name, indextype): - class Sequence(abc.Sequence): + class Iterable(abc.Iterable): + def __iter__(self): + yield indextype() + + class Iterator(Iterable, abc.Iterator): + def next(self): + """ needed for python 2 """ + return self.__next__() + + def __next__(self): + return indextype() + + class Sequence(Iterable, abc.Sequence): def __getitem__(self, index: int): return indextype() @@ -15,12 +27,29 @@ def factory(typing_name, indextype): def __setitem__(self, index: int, value: indextype): pass + def __delitem__(self, index: int, value: indextype): + pass + class List(MutableSequence, list): + pass + + class AbstractSet(Iterable, abc.Set): + pass + + class MutableSet(AbstractSet, abc.MutableSet): + def add(item: indextype): + pass + + def discard(item: indextype): pass dct = { "Sequence": Sequence, "MutableSequence": MutableSequence, "List": List, + "Iterable": Iterable, + "Iterator": Iterator, + "AbstractSet": AbstractSet, + "MutableSet": MutableSet, } return dct[typing_name] diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 75742846..f5c2f93d 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -192,3 +192,35 @@ def we_can_has_sequence( u.a #? float() u[1] + +def iterators( + ps: typing.Iterable[int], + qs: typing.Iterator[str], + rs: typing.Sequence["B"], + ts: typing.AbstractSet["float"]): + for p in ps: + #? int() + p + #? + next(ps) + for q in qs: + #? str() + q + #? str() + next(qs) + for r in rs: + #? B() + r + #? + next(rs) + for t in ts: + #? float() + t + +def sets( + p: typing.AbstractSet[int], + q: typing.MutableSet[float]): + #? [] + p.a + #? ["add"] + q.a From 90c4ca8c043897aa4c19f639fb81b00e0177d03b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 19:34:48 +0100 Subject: [PATCH 06/60] should obviously keep typing.py parsable in python 2 --- jedi/evaluate/jedi_typing.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 8d66258f..a9fc813c 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -20,15 +20,11 @@ def factory(typing_name, indextype): return indextype() class Sequence(Iterable, abc.Sequence): - def __getitem__(self, index: int): + def __getitem__(self, index): return indextype() class MutableSequence(Sequence, abc.MutableSequence): - def __setitem__(self, index: int, value: indextype): - pass - - def __delitem__(self, index: int, value: indextype): - pass + pass class List(MutableSequence, list): pass @@ -37,11 +33,7 @@ def factory(typing_name, indextype): pass class MutableSet(AbstractSet, abc.MutableSet): - def add(item: indextype): - pass - - def discard(item: indextype): - pass + pass dct = { "Sequence": Sequence, From 67cbc5ebd12a8beb32c80c5dc14dbbaddca408a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 20:00:08 +0100 Subject: [PATCH 07/60] made code slightly more pytho2 friendly --- jedi/evaluate/jedi_typing.py | 6 ++- jedi/evaluate/pep0484.py | 10 ++--- test/completion/pep0484.py | 67 ---------------------------- test/completion/pep0484_typing.py | 73 +++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 73 deletions(-) create mode 100644 test/completion/pep0484_typing.py diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index a9fc813c..2c547482 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -3,7 +3,11 @@ This module is not intended to be used in jedi, rather it will be fed to the jedi-parser to replace classes in the typing module """ -from collections import abc +try: + from collections import abc +except ImportError: + # python 2 + import collections as abc def factory(typing_name, indextype): diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 3730e7bc..4b507a56 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -56,7 +56,7 @@ def _fix_forward_reference(evaluator, item, module): element.parent = module return evaluator.eval_element(element) else: - return {item} + return set([item]) @memoize_default(None, evaluator_is_first_arg=True) @@ -95,7 +95,7 @@ def get_types_for_typing_module(evaluator, typ, trailer): # actually the PEP-0484 typing module and not some other indextypes = evaluator.eval_element(trailer.children[1]) if not isinstance(indextypes, set): - indextypes = {indextypes} + indextypes = set([indextypes]) module = trailer.get_parent_until() dereferencedindextypes = set() @@ -109,9 +109,9 @@ def get_types_for_typing_module(evaluator, typ, trailer): factory = list(factories)[0] assert factory function_body_nodes = factory.children[4].children - valid_classnames = {child.name.value - for child in function_body_nodes - if isinstance(child, tree.Class)} + valid_classnames = set(child.name.value + for child in function_body_nodes + if isinstance(child, tree.Class)) if typ.name.value not in valid_classnames: return None compiled_classname = compiled.create(evaluator, typ.name.value) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index f5c2f93d..fc08460f 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,70 +157,3 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x - -import typing -def we_can_has_sequence( - p: typing.Sequence[int], - q: typing.Sequence[B], - r: "typing.Sequence[int]", - s: typing.Sequence["int"], - t: typing.MutableSequence[dict], - u: typing.List[float]): - #? ["count"] - p.c - #? int() - p[1] - #? ["count"] - q.c - #? B() - q[1] - #? ["count"] - r.c - #? int() - r[1] - #? ["count"] - s.c - #? int() - s[1] - #? [] - s.a - #? ["append"] - t.a - #? dict() - t[1] - #? ["append"] - u.a - #? float() - u[1] - -def iterators( - ps: typing.Iterable[int], - qs: typing.Iterator[str], - rs: typing.Sequence["B"], - ts: typing.AbstractSet["float"]): - for p in ps: - #? int() - p - #? - next(ps) - for q in qs: - #? str() - q - #? str() - next(qs) - for r in rs: - #? B() - r - #? - next(rs) - for t in ts: - #? float() - t - -def sets( - p: typing.AbstractSet[int], - q: typing.MutableSet[float]): - #? [] - p.a - #? ["add"] - q.a diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py new file mode 100644 index 00000000..ef5ff908 --- /dev/null +++ b/test/completion/pep0484_typing.py @@ -0,0 +1,73 @@ +# python >= 3.2 +import typing +class B: + pass + +def we_can_has_sequence( + p: typing.Sequence[int], + q: typing.Sequence[B], + r: "typing.Sequence[int]", + s: typing.Sequence["int"], + t: typing.MutableSequence[dict], + u: typing.List[float]): + #? ["count"] + p.c + #? int() + p[1] + #? ["count"] + q.c + #? B() + q[1] + #? ["count"] + r.c + #? int() + r[1] + #? ["count"] + s.c + #? int() + s[1] + #? [] + s.a + #? ["append"] + t.a + #? dict() + t[1] + #? ["append"] + u.a + #? float() + u[1] + +def iterators( + ps: typing.Iterable[int], + qs: typing.Iterator[str], + rs: typing.Sequence["ForwardReference"], + ts: typing.AbstractSet["float"]): + for p in ps: + #? int() + p + #? + next(ps) + for q in qs: + #? str() + q + #? str() + next(qs) + for r in rs: + #? ForwardReference() + r + #? + next(rs) + for t in ts: + #? float() + t + +def sets( + p: typing.AbstractSet[int], + q: typing.MutableSet[float]): + #? [] + p.a + #? ["add"] + q.a + +class ForwardReference: + pass From 5948c63cf95f01cf4360d58d94b6c70b9cce75b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 20:13:33 +0100 Subject: [PATCH 08/60] Make the classes descriptions look better --- not sure whether this is a good idea --- jedi/evaluate/pep0484.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 4b507a56..5d0ec018 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -120,4 +120,6 @@ def get_types_for_typing_module(evaluator, typ, trailer): for indextyp in dereferencedindextypes: result |= \ evaluator.execute_evaluated(factory, compiled_classname, indextyp) + for singleresult in result: + singleresult.name.value += "[%s]" % indextyp.name return result From 10f5e15325dd065d2d1ffee35e96b3d36817b48c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Dec 2015 23:05:01 +0100 Subject: [PATCH 09/60] I feel this is a nicer solution. Forward Reference busting should be part of the annotation resolving. It doesn not have anything to do with the typing module (and should indeed also happen if someone writes his own types outside of the typing module) --- jedi/evaluate/pep0484.py | 50 +++++++++++++++++++++----------------- test/completion/pep0484.py | 4 --- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 5d0ec018..db80dd79 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -30,33 +30,45 @@ from jedi import debug def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: - definitions = set() - module = annotation.get_parent_until() - for definition in evaluator.eval_element(annotation): - definitions |= \ - _fix_forward_reference(evaluator, definition, module) + dereferenced_annotation = _fix_forward_reference(evaluator, annotation) + definitions = evaluator.eval_element(dereferenced_annotation) return list(chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] -def _fix_forward_reference(evaluator, item, module): - if (isinstance(item, compiled.CompiledObject) and - isinstance(item.obj, str)): +def _fix_forward_reference(evaluator, item): + """ + Gets something from the parse tree, and replaces any string literal + in there with the result of evaluating that string at the bottom of the + module + """ + if isinstance(item, tree.String): + compiledobjects = evaluator.eval_element(item) + assert len(compiledobjects) == 1 + compiledobject = list(compiledobjects)[0] try: - p = Parser( - load_grammar(), item.obj, start='eval_input') + p = Parser(load_grammar(), compiledobject.obj, start='eval_input') element = p.get_parsed_node() except ParseError: - debug.warning('Annotation not parsed: %s' % item.obj) - return set() + debug.warning('Annotation not parsed: %s' % compiledobject.obj) + return item else: + module = item.get_parent_until() p.position_modifier.line = module.end_pos[0] element.parent = module - return evaluator.eval_element(element) - else: - return set([item]) + dereferenced = _fix_forward_reference(evaluator, element) + return dereferenced + if isinstance(item, tree.Node): + newnode = tree.Node(item.type, []) + for child in item.children: + newchild = _fix_forward_reference(evaluator, child) + newchild.parent = newnode + newnode.children.append(newchild) + newnode.parent = item.parent + return newnode + return item @memoize_default(None, evaluator_is_first_arg=True) @@ -97,12 +109,6 @@ def get_types_for_typing_module(evaluator, typ, trailer): if not isinstance(indextypes, set): indextypes = set([indextypes]) - module = trailer.get_parent_until() - dereferencedindextypes = set() - for indextyp in indextypes: - dereferencedindextypes |= \ - _fix_forward_reference(evaluator, indextyp, module) - typing = _get_typing_replacement_module() factories = evaluator.find_types(typing, "factory") assert len(factories) == 1 @@ -117,7 +123,7 @@ def get_types_for_typing_module(evaluator, typ, trailer): compiled_classname = compiled.create(evaluator, typ.name.value) result = set() - for indextyp in dereferencedindextypes: + for indextyp in indextypes: result |= \ evaluator.execute_evaluated(factory, compiled_classname, indextyp) for singleresult in result: diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index fc08460f..298a7a4f 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -153,7 +153,3 @@ def function_with_assined_class_in_reference(x: X, y: "Y"): #? int() y Y = int - -def just_because_we_can(x: "flo" + "at"): - #? float() - x From 7b9731250975a2b5e2dbf2b11473309091028d0f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 31 Dec 2015 01:29:03 +0100 Subject: [PATCH 10/60] tuples and mappings in typing --- jedi/evaluate/iterable.py | 11 ++-- jedi/evaluate/jedi_typing.py | 49 ++++++++++++++++-- jedi/evaluate/pep0484.py | 43 ++++++++++++---- test/completion/pep0484_typing.py | 85 +++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 21 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 957f9784..f92d1a97 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -564,13 +564,16 @@ def py__getitem__(evaluator, types, trailer): from jedi.evaluate.representation import Class result = set() + trailer_op, node, trailer_cl = trailer.children + assert trailer_op == "[" + assert trailer_cl == "]" # special case: PEP0484 typing module, see # https://github.com/davidhalter/jedi/issues/663 for typ in list(types): if isinstance(typ, Class): typing_module_types = \ - pep0484.get_types_for_typing_module(evaluator, typ, trailer) + pep0484.get_types_for_typing_module(evaluator, typ, node) if typing_module_types is not None: types.remove(typ) result |= typing_module_types @@ -579,12 +582,6 @@ def py__getitem__(evaluator, types, trailer): # all consumed by special cases return result - trailer_op, node, trailer_cl = trailer.children[:3] - assert trailer_op == "[" - if trailer_cl != "]": - debug.warning("No support for complex indices: %s" % trailer) - return result - for index in create_index_types(evaluator, node): if isinstance(index, (compiled.CompiledObject, Slice)): index = index.obj diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 2c547482..62f52fd4 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -10,10 +10,10 @@ except ImportError: import collections as abc -def factory(typing_name, indextype): +def factory(typing_name, indextypes): class Iterable(abc.Iterable): def __iter__(self): - yield indextype() + yield indextypes[0]() class Iterator(Iterable, abc.Iterator): def next(self): @@ -21,11 +21,11 @@ def factory(typing_name, indextype): return self.__next__() def __next__(self): - return indextype() + return indextypes[0]() class Sequence(Iterable, abc.Sequence): def __getitem__(self, index): - return indextype() + return indextypes[0]() class MutableSequence(Sequence, abc.MutableSequence): pass @@ -33,12 +33,46 @@ def factory(typing_name, indextype): class List(MutableSequence, list): pass + class Tuple(Sequence, tuple): + def __getitem__(self, index): + return indextypes[index]() + class AbstractSet(Iterable, abc.Set): pass class MutableSet(AbstractSet, abc.MutableSet): pass + class KeysView(Iterable, abc.KeysView): + pass + + class ValuesView(abc.ValuesView): + def __iter__(self): + yield indextypes[1]() + + class ItemsView(abc.ItemsView): + def __iter__(self): + yield Tuple() + + class Mapping(Iterable, abc.Mapping): + def __getitem__(self, item): + return indextypes[1]() + + def keys(self): + return KeysView() + + def values(self): + return ValuesView() + + def items(self): + return ItemsView() + + class MutableMapping(Mapping, abc.MutableMapping): + pass + + class Dict(MutableMapping, dict): + pass + dct = { "Sequence": Sequence, "MutableSequence": MutableSequence, @@ -47,5 +81,12 @@ def factory(typing_name, indextype): "Iterator": Iterator, "AbstractSet": AbstractSet, "MutableSet": MutableSet, + "Mapping": Mapping, + "MutableMapping": MutableMapping, + "Tuple": Tuple, + "KeysView": KeysView, + "ItemsView": ItemsView, + "ValuesView": ValuesView, + "Dict": Dict, } return dct[typing_name] diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index db80dd79..422ca85f 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -18,7 +18,7 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ -from itertools import chain +import itertools import os from jedi.parser import \ @@ -32,7 +32,7 @@ def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: dereferenced_annotation = _fix_forward_reference(evaluator, annotation) definitions = evaluator.eval_element(dereferenced_annotation) - return list(chain.from_iterable( + return list(itertools.chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] @@ -97,7 +97,8 @@ def _get_typing_replacement_module(): return p.module -def get_types_for_typing_module(evaluator, typ, trailer): +def get_types_for_typing_module(evaluator, typ, node): + from jedi.evaluate.iterable import FakeSequence if not typ.base.get_parent_until().name.value == "typing": return None # we assume that any class using [] in a module called @@ -105,9 +106,10 @@ def get_types_for_typing_module(evaluator, typ, trailer): # should be replaced by that class. This is not 100% # airtight but I don't have a better idea to check that it's # actually the PEP-0484 typing module and not some other - indextypes = evaluator.eval_element(trailer.children[1]) - if not isinstance(indextypes, set): - indextypes = set([indextypes]) + if tree.is_node(node, "subscriptlist"): + nodes = node.children[::2] # skip the commas + else: + nodes = [node] typing = _get_typing_replacement_module() factories = evaluator.find_types(typing, "factory") @@ -123,9 +125,30 @@ def get_types_for_typing_module(evaluator, typ, trailer): compiled_classname = compiled.create(evaluator, typ.name.value) result = set() - for indextyp in indextypes: - result |= \ - evaluator.execute_evaluated(factory, compiled_classname, indextyp) + # don't know what the last parameter is for, this seems to work :) + args = FakeSequence(evaluator, nodes, "x-type") + + result |= evaluator.execute_evaluated(factory, compiled_classname, args) + human_nodes = [] + for node in nodes: + evalled_node = evaluator.eval_element(node) + if len(evalled_node) != 1: + human_nodes.append("???") + continue + evalled_node = list(evalled_node)[0] + try: + human_nodes.append(str(evalled_node.name)) + except AttributeError: + pass + else: + continue + try: + human_nodes.append(evalled_node.obj.__name__) + except AttributeError: + pass + else: + continue + human_nodes.append("???") for singleresult in result: - singleresult.name.value += "[%s]" % indextyp.name + singleresult.name.value += "[%s]" % ", ".join(human_nodes) return result diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index ef5ff908..8b8ad852 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -69,5 +69,90 @@ def sets( #? ["add"] q.a +def tuple( + p: typing.Tuple[int], + q: typing.Tuple[int, str, float], + r: typing.Tuple[B, ...]): + #? int() + p[0] + #? int() + q[0] + #? str() + q[1] + #? float() + q[2] + #? B() + r[0] + ##? B() --- TODO fix support for arbitrary length + r[1] + #? B() + r[2] + #? B() + r[10000] + i, s, f = q + #? int() + i + ##? str() --- TODO fix support for tuple assignment + s + ##? float() --- TODO fix support for tuple assignment + f + +class Key: + pass + +class Value: + pass + +def mapping( + p: typing.Mapping[Key, Value], + q: typing.MutableMapping[Key, Value], + d: typing.Dict[Key, Value], + r: typing.KeysView[Key], + s: typing.ValuesView[Value], + t: typing.ItemsView[Key, Value]): + #? [] + p.setd + #? ["setdefault"] + q.setd + #? ["setdefault"] + d.setd + #? Value() + p[1] + for key in p: + #? Key() + key + for key in p.keys(): + #? Key() + key + for value in p.values(): + #? Value() + value + for item in p.items(): + #? Key() + item[0] + #? Value() + item[1] + (key, value) = item + #? Key() + key + ##? Value() --- TODO fix support for tuple assignment + value + for key, value in p.items(): + #? Key() + key + ##? Value() --- TODO fix support for tuple assignment + value + for key in r: + #? Key() + key + for value in s: + #? Value() + value + for key, value in t: + #? Key() + key + ##? Value() --- TODO fix support for tuple assignment + value + class ForwardReference: pass From 3852431549d3e7c4f5c99d0f907c71b40110f8df Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 31 Dec 2015 01:59:34 +0100 Subject: [PATCH 11/60] typing.Union and typing.Optional --- jedi/evaluate/pep0484.py | 7 +++++++ test/completion/pep0484_typing.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 422ca85f..8b4381ff 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -110,6 +110,13 @@ def get_types_for_typing_module(evaluator, typ, node): nodes = node.children[::2] # skip the commas else: nodes = [node] + del node + + # hacked in Union and Optional, since it's hard to do nicely in parsed code + if typ.name.value == "Union": + return set().union(*[evaluator.eval_element(node) for node in nodes]) + if typ.name.value == "Optional": + return evaluator.eval_element(nodes[0]) typing = _get_typing_replacement_module() factories = evaluator.find_types(typing, "factory") diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 8b8ad852..6b085fc2 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -154,5 +154,32 @@ def mapping( ##? Value() --- TODO fix support for tuple assignment value +def union( + p: typing.Union[int], + q: typing.Union[int, int], + r: typing.Union[int, str, "int"], + s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]], + t: typing.Union[int, None]): + #? int() + p + #? int() + q + #? int() str() + r + #? int() str() float() dict() + s + #? int() + t + +def optional( + p: typing.Optional[int]): + """ + Optional does not do anything special. However it should be recognised + as being of that type. Jedi doesn't do anything with the extra into that + it can be None as well + """ + #? int() + p + class ForwardReference: pass From 409ee5568a1937c055dc1b944d4b0311abb7ff12 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 31 Dec 2015 11:04:26 +0100 Subject: [PATCH 12/60] test with different ways of importing the typing module --- test/completion/pep0484_typing.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 6b085fc2..0467c8c5 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -183,3 +183,18 @@ def optional( class ForwardReference: pass + +import typing as t +def union2(x: t.Union[int, str]): + #? int() str() + x + +from typing import Union +def union3(x: Union[int, str]): + #? int() str() + x + +from typing import Union as U +def union4(x: U[int, str]): + #? int() str() + x From 1b787e2a110a099e9b100ab89b3911aa682e8f23 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 31 Dec 2015 11:19:59 +0100 Subject: [PATCH 13/60] add test to check instanciated subclasses --- test/completion/pep0484_typing.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 0467c8c5..2dfc50fc 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -198,3 +198,27 @@ from typing import Union as U def union4(x: U[int, str]): #? int() str() x + +class TestDict(typing.Dict[str, int]): + def setdud(self): + pass + +def testdict(x: TestDict): + #? ["setdud", "setdefault"] + x.setd + for key in x.keys(): + #? str() + key + for value in x.values(): + #? int() + value + +x = TestDict() +#? ["setdud", "setdefault"] +x.setd +for key in x.keys(): + #? str() + key +for value in x.values(): + #? int() + value From 9d7e1ce81b9e24fb1a05c3dbc4fa43ed70fdc69c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 1 Jan 2016 13:40:14 +0100 Subject: [PATCH 14/60] add the typing module for testing --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 526092fc..1229d735 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,8 @@ deps = docopt # coloroma for colored debug output colorama +# for testing the typing module + typing setenv = # https://github.com/tomchristie/django-rest-framework/issues/1957 # tox corrupts __pycache__, solution from here: From 59161c0b5d3224be3e622367fc95fb0d31b211b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 10:51:06 +0100 Subject: [PATCH 15/60] fix FakeSequence type --- jedi/evaluate/iterable.py | 3 +++ jedi/evaluate/pep0484.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index f92d1a97..0c6c20ce 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -431,6 +431,9 @@ class ImplicitTuple(_FakeArray): class FakeSequence(_FakeArray): def __init__(self, evaluator, sequence_values, type): + """ + type should be one of "tuple", "list" + """ super(FakeSequence, self).__init__(evaluator, sequence_values, type) self._sequence_values = sequence_values diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 8b4381ff..56efcd67 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -132,8 +132,7 @@ def get_types_for_typing_module(evaluator, typ, node): compiled_classname = compiled.create(evaluator, typ.name.value) result = set() - # don't know what the last parameter is for, this seems to work :) - args = FakeSequence(evaluator, nodes, "x-type") + args = FakeSequence(evaluator, nodes, "tuple") result |= evaluator.execute_evaluated(factory, compiled_classname, args) human_nodes = [] From a5fc149f9d4d9c832e242004d507ad9b225dd94b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 10:57:38 +0100 Subject: [PATCH 16/60] use jedi.common.unite in flatten array of sets --- jedi/evaluate/pep0484.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 56efcd67..c5499f16 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -24,6 +24,7 @@ import os from jedi.parser import \ Parser, load_grammar, ParseError, ParserWithRecovery, tree from jedi.evaluate.cache import memoize_default +from jedi.common import unite from jedi.evaluate import compiled from jedi import debug @@ -114,7 +115,7 @@ def get_types_for_typing_module(evaluator, typ, node): # hacked in Union and Optional, since it's hard to do nicely in parsed code if typ.name.value == "Union": - return set().union(*[evaluator.eval_element(node) for node in nodes]) + return unite(evaluator.eval_element(node) for node in nodes) if typ.name.value == "Optional": return evaluator.eval_element(nodes[0]) From ae701b2f9a23f9123327dd9bd87272269f5a8f72 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 12:43:23 +0100 Subject: [PATCH 17/60] Support for typing.Tuple[type, ...] --- jedi/evaluate/jedi_typing.py | 7 ++++++- test/completion/pep0484_typing.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 62f52fd4..e34ee989 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -35,7 +35,12 @@ def factory(typing_name, indextypes): class Tuple(Sequence, tuple): def __getitem__(self, index): - return indextypes[index]() + if indextypes[1] == ...: + # https://www.python.org/dev/peps/pep-0484/#the-typing-module + # Tuple[int, ...] means a tuple of ints of indetermined length + return indextypes[0]() + else: + return indextypes[index]() class AbstractSet(Iterable, abc.Set): pass diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 2dfc50fc..74a99e51 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -83,7 +83,7 @@ def tuple( q[2] #? B() r[0] - ##? B() --- TODO fix support for arbitrary length + #? B() r[1] #? B() r[2] From b4999063987a4c03ef04eb05fda5c615c9bb3f5a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 16:12:43 +0100 Subject: [PATCH 18/60] Reverted 10f5e1 --- needed some more work to get it working again --- jedi/evaluate/pep0484.py | 64 +++++++++++++++++--------------------- test/completion/pep0484.py | 4 +++ 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index c5499f16..1c7ca2fe 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -31,45 +31,36 @@ from jedi import debug def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: - dereferenced_annotation = _fix_forward_reference(evaluator, annotation) - definitions = evaluator.eval_element(dereferenced_annotation) + definitions = evaluator.eval_element( + _fix_forward_reference(evaluator, annotation)) return list(itertools.chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] -def _fix_forward_reference(evaluator, item): - """ - Gets something from the parse tree, and replaces any string literal - in there with the result of evaluating that string at the bottom of the - module - """ - if isinstance(item, tree.String): - compiledobjects = evaluator.eval_element(item) - assert len(compiledobjects) == 1 - compiledobject = list(compiledobjects)[0] +def _fix_forward_reference(evaluator, node): + evaled_nodes = evaluator.eval_element(node) + if len(evaled_nodes) != 1: + debug.warning("Eval'ed typing index %s should lead to 1 object, " + " not %s" % (node, evaled_nodes)) + return node + evaled_node = list(evaled_nodes)[0] + if isinstance(evaled_node, compiled.CompiledObject) and \ + isinstance(evaled_node.obj, str): try: - p = Parser(load_grammar(), compiledobject.obj, start='eval_input') - element = p.get_parsed_node() + p = Parser(load_grammar(), evaled_node.obj, start='eval_input') + newnode = p.get_parsed_node() except ParseError: - debug.warning('Annotation not parsed: %s' % compiledobject.obj) - return item + debug.warning('Annotation not parsed: %s' % evaled_node.obj) + return node else: - module = item.get_parent_until() + module = node.get_parent_until() p.position_modifier.line = module.end_pos[0] - element.parent = module - dereferenced = _fix_forward_reference(evaluator, element) - return dereferenced - if isinstance(item, tree.Node): - newnode = tree.Node(item.type, []) - for child in item.children: - newchild = _fix_forward_reference(evaluator, child) - newchild.parent = newnode - newnode.children.append(newchild) - newnode.parent = item.parent - return newnode - return item + newnode.parent = module + return newnode + else: + return node @memoize_default(None, evaluator_is_first_arg=True) @@ -113,6 +104,8 @@ def get_types_for_typing_module(evaluator, typ, node): nodes = [node] del node + nodes = [_fix_forward_reference(evaluator, node) for node in nodes] + # hacked in Union and Optional, since it's hard to do nicely in parsed code if typ.name.value == "Union": return unite(evaluator.eval_element(node) for node in nodes) @@ -132,25 +125,24 @@ def get_types_for_typing_module(evaluator, typ, node): return None compiled_classname = compiled.create(evaluator, typ.name.value) - result = set() args = FakeSequence(evaluator, nodes, "tuple") - result |= evaluator.execute_evaluated(factory, compiled_classname, args) + result = evaluator.execute_evaluated(factory, compiled_classname, args) human_nodes = [] for node in nodes: - evalled_node = evaluator.eval_element(node) - if len(evalled_node) != 1: + evaled_node = evaluator.eval_element(node) + if len(evaled_node) != 1: human_nodes.append("???") continue - evalled_node = list(evalled_node)[0] + evaled_node = list(evaled_node)[0] try: - human_nodes.append(str(evalled_node.name)) + human_nodes.append(str(evaled_node.name)) except AttributeError: pass else: continue try: - human_nodes.append(evalled_node.obj.__name__) + human_nodes.append(evaled_node.obj.__name__) except AttributeError: pass else: diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 298a7a4f..fc08460f 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -153,3 +153,7 @@ def function_with_assined_class_in_reference(x: X, y: "Y"): #? int() y Y = int + +def just_because_we_can(x: "flo" + "at"): + #? float() + x From 885f7cb06873ca30ed483e80816ec724399de49e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 16:45:12 +0100 Subject: [PATCH 19/60] fix for iterators -- should start working when py__iter__ gets fixed: https://github.com/davidhalter/jedi/pull/663\#issuecomment-172317854 --- jedi/evaluate/jedi_typing.py | 22 ++++++++++++++++++---- test/completion/pep0484_typing.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index e34ee989..2fd0e0d8 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -13,7 +13,8 @@ except ImportError: def factory(typing_name, indextypes): class Iterable(abc.Iterable): def __iter__(self): - yield indextypes[0]() + while True: + yield indextypes[0]() class Iterator(Iterable, abc.Iterator): def next(self): @@ -23,10 +24,14 @@ def factory(typing_name, indextypes): def __next__(self): return indextypes[0]() - class Sequence(Iterable, abc.Sequence): + class Sequence(abc.Sequence): def __getitem__(self, index): return indextypes[0]() + def __len__(self): + import sys + return sys.maxint + class MutableSequence(Sequence, abc.MutableSequence): pass @@ -42,6 +47,13 @@ def factory(typing_name, indextypes): else: return indextypes[index]() + def __len__(self): + if indextypes[1] == ...: + import sys + return sys.maxint + else: + return len(indextypes) + class AbstractSet(Iterable, abc.Set): pass @@ -53,11 +65,13 @@ def factory(typing_name, indextypes): class ValuesView(abc.ValuesView): def __iter__(self): - yield indextypes[1]() + while True: + yield indextypes[1]() class ItemsView(abc.ItemsView): def __iter__(self): - yield Tuple() + while True: + yield Tuple() class Mapping(Iterable, abc.Mapping): def __getitem__(self, item): diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 74a99e51..b3fe6055 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -47,6 +47,16 @@ def iterators( p #? next(ps) + a, b = ps + #? int() + a + ##? int() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + # test below is just to make sure that in case it gets fixed by accident + # these tests will be fixed as well the way they should be + #? + b + for q in qs: #? str() q @@ -93,8 +103,12 @@ def tuple( #? int() i ##? str() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? s ##? float() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? f class Key: @@ -136,11 +150,15 @@ def mapping( #? Key() key ##? Value() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? value for key, value in p.items(): #? Key() key ##? Value() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? value for key in r: #? Key() @@ -152,6 +170,8 @@ def mapping( #? Key() key ##? Value() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? value def union( From b316fb94c45911381f00b36721c48b5e3ed26efb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 17:05:29 +0100 Subject: [PATCH 20/60] enable tests for the value type in tuple assignment from typing.Mapping[].items() --- test/completion/pep0484_typing.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index b3fe6055..f31542ea 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -149,16 +149,12 @@ def mapping( (key, value) = item #? Key() key - ##? Value() --- TODO fix support for tuple assignment - # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 - #? + #? Value() value for key, value in p.items(): #? Key() key - ##? Value() --- TODO fix support for tuple assignment - # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 - #? + #? Value() value for key in r: #? Key() @@ -169,9 +165,7 @@ def mapping( for key, value in t: #? Key() key - ##? Value() --- TODO fix support for tuple assignment - # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 - #? + #? Value() value def union( From 941da773f61f398da2e2ba5ca8a1d8999480915a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 17:05:31 +0100 Subject: [PATCH 21/60] temporary fix for typing.Mapping[...].items(), can be removed after #683 is fixed --- jedi/evaluate/jedi_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 2fd0e0d8..e7d77ff9 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -71,7 +71,7 @@ def factory(typing_name, indextypes): class ItemsView(abc.ItemsView): def __iter__(self): while True: - yield Tuple() + yield (indextypes[0](), indextypes[1]()) class Mapping(Iterable, abc.Mapping): def __getitem__(self, item): From 442d948e322ea0099777f091752bb60002d02a30 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 17 Jan 2016 18:04:59 +0100 Subject: [PATCH 22/60] I don't need the __len__ for __iter__ to work (eventually), so leaving it out for now --- jedi/evaluate/jedi_typing.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index e7d77ff9..52b0ab89 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -28,10 +28,6 @@ def factory(typing_name, indextypes): def __getitem__(self, index): return indextypes[0]() - def __len__(self): - import sys - return sys.maxint - class MutableSequence(Sequence, abc.MutableSequence): pass @@ -47,13 +43,6 @@ def factory(typing_name, indextypes): else: return indextypes[index]() - def __len__(self): - if indextypes[1] == ...: - import sys - return sys.maxint - else: - return len(indextypes) - class AbstractSet(Iterable, abc.Set): pass From c9bf521efd0b8e63a4e07f993d30d35a04a4733b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 23 Jan 2016 22:10:52 +0100 Subject: [PATCH 23/60] remove renaming of class based on parameters --- jedi/evaluate/pep0484.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 1c7ca2fe..ac08908c 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -128,26 +128,4 @@ def get_types_for_typing_module(evaluator, typ, node): args = FakeSequence(evaluator, nodes, "tuple") result = evaluator.execute_evaluated(factory, compiled_classname, args) - human_nodes = [] - for node in nodes: - evaled_node = evaluator.eval_element(node) - if len(evaled_node) != 1: - human_nodes.append("???") - continue - evaled_node = list(evaled_node)[0] - try: - human_nodes.append(str(evaled_node.name)) - except AttributeError: - pass - else: - continue - try: - human_nodes.append(evaled_node.obj.__name__) - except AttributeError: - pass - else: - continue - human_nodes.append("???") - for singleresult in result: - singleresult.name.value += "[%s]" % ", ".join(human_nodes) return result From e267f6365786582dd325ec9de98854c459921b52 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 23 Jan 2016 22:53:48 +0100 Subject: [PATCH 24/60] python 2.7 compatibility, typing module tested with docstring, so that it can also be tested in python 2.7 --- jedi/evaluate/pep0484.py | 6 +- test/completion/pep0484_typing.py | 125 ++++++++++++++++++------------ 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index ac08908c..6c7f6e8e 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -27,6 +27,7 @@ from jedi.evaluate.cache import memoize_default from jedi.common import unite from jedi.evaluate import compiled from jedi import debug +from jedi import _compatibility def _evaluate_for_annotation(evaluator, annotation): @@ -49,7 +50,8 @@ def _fix_forward_reference(evaluator, node): if isinstance(evaled_node, compiled.CompiledObject) and \ isinstance(evaled_node.obj, str): try: - p = Parser(load_grammar(), evaled_node.obj, start='eval_input') + p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj), + start='eval_input') newnode = p.get_parsed_node() except ParseError: debug.warning('Annotation not parsed: %s' % evaled_node.obj) @@ -84,7 +86,7 @@ def _get_typing_replacement_module(): typing_path = os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) with open(typing_path) as f: - code = f.read() + code = _compatibility.unicode(f.read()) p = ParserWithRecovery(load_grammar(), code) return p.module diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index f31542ea..7bee1e64 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -1,15 +1,21 @@ -# python >= 3.2 +""" +Test the typing library, with docstrings. This is needed since annotations +are not supported in python 2.7 else then annotating by comment (and this is +still TODO at 2016-01-23) +""" import typing class B: pass -def we_can_has_sequence( - p: typing.Sequence[int], - q: typing.Sequence[B], - r: "typing.Sequence[int]", - s: typing.Sequence["int"], - t: typing.MutableSequence[dict], - u: typing.List[float]): +def we_can_has_sequence(p, q, r, s, t, u): + """ + :type p: typing.Sequence[int] + :type q: typing.Sequence[B] + :type r: typing.Sequence[int] + :type s: typing.Sequence["int"] + :type t: typing.MutableSequence[dict] + :type u: typing.List[float] + """ #? ["count"] p.c #? int() @@ -37,11 +43,13 @@ def we_can_has_sequence( #? float() u[1] -def iterators( - ps: typing.Iterable[int], - qs: typing.Iterator[str], - rs: typing.Sequence["ForwardReference"], - ts: typing.AbstractSet["float"]): +def iterators(ps, qs, rs, ts): + """ + :type ps: typing.Iterable[int] + :type qs: typing.Iterator[str] + :type rs: typing.Sequence["ForwardReference"] + :type ts: typing.AbstractSet["float"] + """ for p in ps: #? int() p @@ -71,18 +79,22 @@ def iterators( #? float() t -def sets( - p: typing.AbstractSet[int], - q: typing.MutableSet[float]): +def sets(p, q): + """ + :type p: typing.AbstractSet[int] + :type q: typing.MutableSet[float] + """ #? [] p.a #? ["add"] q.a -def tuple( - p: typing.Tuple[int], - q: typing.Tuple[int, str, float], - r: typing.Tuple[B, ...]): +def tuple(p, q, r): + """ + :type p: typing.Tuple[int] + :type q: typing.Tuple[int, str, float] + :type r: typing.Tuple[B, ...] + """ #? int() p[0] #? int() @@ -117,13 +129,15 @@ class Key: class Value: pass -def mapping( - p: typing.Mapping[Key, Value], - q: typing.MutableMapping[Key, Value], - d: typing.Dict[Key, Value], - r: typing.KeysView[Key], - s: typing.ValuesView[Value], - t: typing.ItemsView[Key, Value]): +def mapping(p, q, d, r, s, t): + """ + :type p: typing.Mapping[Key, Value] + :type q: typing.MutableMapping[Key, Value] + :type d: typing.Dict[Key, Value] + :type r: typing.KeysView[Key] + :type s: typing.ValuesView[Value] + :type t: typing.ItemsView[Key, Value] + """ #? [] p.setd #? ["setdefault"] @@ -168,12 +182,14 @@ def mapping( #? Value() value -def union( - p: typing.Union[int], - q: typing.Union[int, int], - r: typing.Union[int, str, "int"], - s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]], - t: typing.Union[int, None]): +def union(p, q, r, s, t): + """ + :type p: typing.Union[int] + :type q: typing.Union[int, int] + :type r: typing.Union[int, str, "int"] + :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]] + :type t: typing.Union[int, None] + """ #? int() p #? int() @@ -185,9 +201,9 @@ def union( #? int() t -def optional( - p: typing.Optional[int]): +def optional(p): """ + :type p: typing.Optional[int] Optional does not do anything special. However it should be recognised as being of that type. Jedi doesn't do anything with the extra into that it can be None as well @@ -198,26 +214,14 @@ def optional( class ForwardReference: pass -import typing as t -def union2(x: t.Union[int, str]): - #? int() str() - x - -from typing import Union -def union3(x: Union[int, str]): - #? int() str() - x - -from typing import Union as U -def union4(x: U[int, str]): - #? int() str() - x - class TestDict(typing.Dict[str, int]): def setdud(self): pass -def testdict(x: TestDict): +def testdict(x): + """ + :type x: TestDict + """ #? ["setdud", "setdefault"] x.setd for key in x.keys(): @@ -236,3 +240,22 @@ for key in x.keys(): for value in x.values(): #? int() value +# python >= 3.2 +""" +docstrings have some auto-import, annotations can use all of Python's +import logic +""" +import typing as t +def union2(x: t.Union[int, str]): + #? int() str() + x + +from typing import Union +def union3(x: Union[int, str]): + #? int() str() + x + +from typing import Union as U +def union4(x: U[int, str]): + #? int() str() + x From 244c9976e5ebbac163af1f31e5ef517d1851c6d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 23 Jan 2016 23:06:28 +0100 Subject: [PATCH 25/60] cache the parsed jedi_typing module --- jedi/evaluate/pep0484.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 6c7f6e8e..08588d71 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -77,18 +77,23 @@ def find_return_types(evaluator, func): return _evaluate_for_annotation(evaluator, annotation) -# TODO: Memoize +_typing_module = None + + def _get_typing_replacement_module(): """ The idea is to return our jedi replacement for the PEP-0484 typing module as discussed at https://github.com/davidhalter/jedi/issues/663 """ - - typing_path = os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) - with open(typing_path) as f: - code = _compatibility.unicode(f.read()) - p = ParserWithRecovery(load_grammar(), code) - return p.module + global _typing_module + if _typing_module is None: + typing_path = \ + os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) + with open(typing_path) as f: + code = _compatibility.unicode(f.read()) + p = ParserWithRecovery(load_grammar(), code) + _typing_module = p.module + return _typing_module def get_types_for_typing_module(evaluator, typ, node): From 079e3bbd28ce360e440f2d6ff36e6955bdb65517 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 23 Jan 2016 23:09:45 +0100 Subject: [PATCH 26/60] use Ellipsis instead of ..., for python 2.7 compatibility --- jedi/evaluate/jedi_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py index 52b0ab89..f48a5673 100644 --- a/jedi/evaluate/jedi_typing.py +++ b/jedi/evaluate/jedi_typing.py @@ -36,7 +36,7 @@ def factory(typing_name, indextypes): class Tuple(Sequence, tuple): def __getitem__(self, index): - if indextypes[1] == ...: + if indextypes[1] == Ellipsis: # https://www.python.org/dev/peps/pep-0484/#the-typing-module # Tuple[int, ...] means a tuple of ints of indetermined length return indextypes[0]() From f9a64fd63736e180c553e3d242093ef9c01deeba Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Jan 2016 14:59:26 -0200 Subject: [PATCH 27/60] Fix some issues in Python 2.7 --- test/test_parser/test_pgen2.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 416aef8d..6d51ae85 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -9,6 +9,7 @@ test_grammar.py files from both Python 2 and Python 3. from textwrap import dedent +from jedi._compatibility import unicode from jedi.parser import Parser, load_grammar, ParseError import pytest @@ -18,7 +19,7 @@ from test.helpers import TestCase def parse(code, version='3.4'): code = dedent(code) + "\n\n" grammar = load_grammar(version=version) - return Parser(grammar, code, 'file_input').get_parsed_node() + return Parser(grammar, unicode(code), 'file_input').get_parsed_node() class TestDriver(TestCase): @@ -232,6 +233,9 @@ class TestParserIdempotency(TestCase): class TestLiterals(GrammarTest): + # It's not possible to get the same result when using \xaa in Python 2/3, + # because it's treated differently. + @pytest.mark.skipif('sys.version_info[0] < 3') def test_multiline_bytes_literals(self): s = """ md5test(b"\xaa" * 80, @@ -250,6 +254,7 @@ class TestLiterals(GrammarTest): ''' parse(s) + @pytest.mark.skipif('sys.version_info[0] < 3') def test_multiline_str_literals(self): s = """ md5test("\xaa" * 80, From 633e5aa76ffb23935d87e139736638b131d537d4 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Jan 2016 15:05:58 -0200 Subject: [PATCH 28/60] The typing library only works in Python >= 2.7. --- tox.ini | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1229d735..8e0656d5 100644 --- a/tox.ini +++ b/tox.ini @@ -8,8 +8,6 @@ deps = docopt # coloroma for colored debug output colorama -# for testing the typing module - typing setenv = # https://github.com/tomchristie/django-rest-framework/issues/1957 # tox corrupts __pycache__, solution from here: @@ -20,6 +18,19 @@ commands = deps = unittest2 {[testenv]deps} +[testenv:py27] +deps = +# for testing the typing module + typing +[testenv:py32] +deps = + typing +[testenv:py33] +deps = + typing +[testenv:py34] +deps = + typing [testenv:cov] deps = coverage From 257009d23884f03dead9f01ef51d97f1440782ee Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Jan 2016 15:59:27 -0200 Subject: [PATCH 29/60] Skip pep0484 tests when using Python 2.6. --- test/completion/pep0484_typing.py | 2 ++ test/run.py | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 7bee1e64..75c1c0b0 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -3,6 +3,8 @@ Test the typing library, with docstrings. This is needed since annotations are not supported in python 2.7 else then annotating by comment (and this is still TODO at 2016-01-23) """ +# There's no Python 2.6 typing module. +# python >= 2.7 import typing class B: pass diff --git a/test/run.py b/test/run.py index 943a824e..33a346df 100755 --- a/test/run.py +++ b/test/run.py @@ -249,14 +249,16 @@ def skip_python_version(line): map(int, match.group(2).split("."))) operation = getattr(operator, comp_map[match.group(1)]) if not operation(sys.version_info, minimal_python_version): - return "Minimal python version %s" % match.group(1) + return "Minimal python version %s %s" % (match.group(1), match.group(2)) return None -def collect_file_tests(lines, lines_to_execute): - makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column, - start, line, path=None, skip=skip) +def collect_file_tests(path, lines, lines_to_execute): + def makecase(t): + return IntegrationTestCase(t, correct, line_nr, column, + start, line, path=path, skip=skip) + start = None correct = None test_type = None @@ -325,9 +327,8 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False): else: source = unicode(open(path).read(), 'UTF-8') - for case in collect_file_tests(StringIO(source), + for case in collect_file_tests(path, StringIO(source), lines_to_execute): - case.path = path case.source = source if skip: case.skip = skip From c09a916ab5dc46c48dbe60f54fe8ba7acbf5cf61 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Jan 2016 17:52:42 -0200 Subject: [PATCH 30/60] Didn't load grammar for Python 2.6 correctly. --- jedi/parser/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 93be5f16..e607e142 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -47,7 +47,7 @@ def load_grammar(version='3.4'): if version in ('3.2', '3.3'): version = '3.4' elif version == '2.6': - version == '2.7' + version = '2.7' file = 'grammar' + version + '.txt' From 7fe5280bda7de38c296d90ef191d7e9d7c984131 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Jan 2016 19:01:33 -0200 Subject: [PATCH 31/60] Forgot to include all tox dependencies. --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 8e0656d5..878d1725 100644 --- a/tox.ini +++ b/tox.ini @@ -22,15 +22,19 @@ deps = deps = # for testing the typing module typing + {[testenv]deps} [testenv:py32] deps = typing + {[testenv]deps} [testenv:py33] deps = typing + {[testenv]deps} [testenv:py34] deps = typing + {[testenv]deps} [testenv:cov] deps = coverage From 17a1a0ebfda94923292c2ff2b98593d24a23938a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Jan 2016 14:39:18 -0200 Subject: [PATCH 32/60] Colorama 0.3.6 is buggy, so just don't import it if it's not there. --- jedi/debug.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jedi/debug.py b/jedi/debug.py index 430a0329..4906b2ef 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -14,7 +14,12 @@ try: # pytest resets the stream at the end - causes troubles. Since after # every output the stream is reset automatically we don't need this. initialise.atexit_done = True - init() + try: + init() + except Exception: + # Colorama fails with initializing under vim and is buggy in + # version 0.3.6. + pass except ImportError: class Fore(object): RED = '' From 1f4c95918f642693a26cdbd3a910ff4c0f3cf954 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:14:56 +0100 Subject: [PATCH 33/60] Add @= ATEQUAL token --- jedi/parser/token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/parser/token.py b/jedi/parser/token.py index e9ab3a62..6890c8ef 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -68,6 +68,7 @@ opmap_raw = """\ %= PERCENTEQUAL &= AMPEREQUAL |= VBAREQUAL +@= ATEQUAL ^= CIRCUMFLEXEQUAL <<= LEFTSHIFTEQUAL >>= RIGHTSHIFTEQUAL From bc0486f7231ca2a7f0a87eae94b18b8623f7bd97 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:21:26 +0100 Subject: [PATCH 34/60] python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 3c5b1fdf..de78f2f5 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -291,7 +291,7 @@ class Evaluator(object): types = set([element]) # TODO this is no real evaluation. elif element.type == 'expr_stmt': types = self.eval_statement(element) - elif element.type == 'power': + elif element.type == 'power' or element.type == 'atom_expr': types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. From 3fb5fe8c77d79e07b9adf70bd6277cf320cf8ce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:22:17 +0100 Subject: [PATCH 35/60] allow empty bodies for better autocompletion --- jedi/parser/grammar3.5.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index 99fcea02..a647880f 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -82,7 +82,9 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows +# classes and functions to be empty, which is beneficial for autocompletion. +suite: simple_stmt | NEWLINE INDENT stmt* DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond From 241abe9cf399355af03eb5090376793f1b61a420 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:42:53 +0100 Subject: [PATCH 36/60] python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/dynamic.py | 3 ++- jedi/evaluate/helpers.py | 3 ++- jedi/evaluate/iterable.py | 2 +- jedi/parser/tree.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index ad65d6db..6ca429fe 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -89,7 +89,8 @@ def search_function_call(evaluator, func): parent = parent.parent trailer = None - if tree.is_node(parent, 'power'): + if tree.is_node(parent, 'power') or \ + tree.is_node(parent, 'atom_expr'): for t in parent.children[1:]: if t == '**': break diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 49edadbf..86eec876 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -88,7 +88,8 @@ def call_of_name(name, cut_own_trailer=False): return name power = par.parent - if tree.is_node(power, 'power') and power.children[0] != name \ + if (tree.is_node(power, 'power') or tree.is_node(power, 'atom_expr')) \ + and power.children[0] != name \ and not (power.children[-2] == '**' and name.start_pos > power.children[-1].start_pos): par = power diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 0c6c20ce..21d4211d 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -528,7 +528,7 @@ def unpack_tuple_to_dict(evaluator, types, exprlist): analysis.add(evaluator, 'value-error-too-few-values', has_parts, message="ValueError: need more than %s values to unpack" % n) return dct - elif exprlist.type == 'power': + elif exprlist.type == 'power' or exprlist.type == 'atom_expr': # Something like ``arr[x], var = ...``. # This is something that is not yet supported, would also be difficult # to write into a dict. diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 7824a634..867d8cde 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1342,7 +1342,7 @@ def _defined_names(current): names += _defined_names(child) elif is_node(current, 'atom'): names += _defined_names(current.children[1]) - elif is_node(current, 'power'): + elif is_node(current, 'power') or is_node(current, 'atom_expr'): if current.children[-2] != '**': # Just if there's no operation trailer = current.children[-1] if trailer.children[0] == '.': From 3b0dcb3fcba8e04843f99161de0ff0081eb25354 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 11:47:01 +0100 Subject: [PATCH 37/60] move file_input to top of file, as mentioned in 19acdd32b72ec4189d98a9a6fb3b0db1cf8fb471 --- jedi/parser/grammar3.5.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index a647880f..9a27f1ec 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -15,8 +15,8 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE From de98cda2d774b038fa7f1a94c24cc4e2027ea2aa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 12:06:16 +0100 Subject: [PATCH 38/60] python3.5 uses 'argument' node type, not 'arglist' for * and ** arguments --- jedi/evaluate/param.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 3dc515a1..ebc173f9 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -47,7 +47,10 @@ class Arguments(tree.Base): for el in self.argument_node: yield 0, el else: - if not tree.is_node(self.argument_node, 'arglist'): + if not (tree.is_node(self.argument_node, 'arglist') or ( + # in python 3.5 **arg is an argument, not arglist + (tree.is_node(self.argument_node, 'argument') and + self.argument_node.children[0] in ('*', '**')))): yield 0, self.argument_node return From 0ed149070ad2cabd0e79245b567ce80f2ddf6304 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 17:13:25 +0100 Subject: [PATCH 39/60] add python 3.5 '@' operator to tokenizer --- jedi/parser/tokenize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 194a0967..46f9ef76 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -76,7 +76,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') # recognized as two instances of =). operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", - r"[+\-*/%&|^=<>]=?", + r"[+\-*@/%&|^=<>]=?", r"~") bracket = '[][(){}]' From a09611197be718ebbd7e0b72fe1c1246c77718fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:17:31 +0100 Subject: [PATCH 40/60] add ATEQUAL token for python < 3.5 --- jedi/_compatibility.py | 1 + jedi/parser/token.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 60c9dcb5..13f55134 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -13,6 +13,7 @@ except ImportError: is_py3 = sys.version_info[0] >= 3 is_py33 = is_py3 and sys.version_info.minor >= 3 +is_py35 = is_py3 and sys.version_info.minor >= 5 is_py26 = not is_py3 and sys.version_info[1] < 7 diff --git a/jedi/parser/token.py b/jedi/parser/token.py index 6890c8ef..0cb846da 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from token import * @@ -24,6 +24,11 @@ else: tok_name[ELLIPSIS] = 'ELLIPSIS' N_TOKENS += 1 +if not is_py35: + ATEQUAL = N_TOKENS + tok_name[ATEQUAL] = 'ATEQUAL' + N_TOKENS += 1 + # Map from operator to number (since tokenize doesn't do this) From 8819b2133a45731668910011968d686aa6fce117 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:23:24 +0100 Subject: [PATCH 41/60] further fix for *-arguments in arglist --- jedi/evaluate/param.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index ebc173f9..997e0799 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -60,6 +60,10 @@ class Arguments(tree.Base): continue elif child in ('*', '**'): yield len(child.value), next(iterator) + elif tree.is_node(child, 'argument') and \ + child.children[0] in ('*', '**'): + assert len(child.children) == 2 + yield len(child.children[0].value), child.children[1] else: yield 0, child From bf5acb4c7acda60325d252160f430de94cebb949 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 18:47:48 +0100 Subject: [PATCH 42/60] once more: python 3.5 uses atom_expr node in many places where previous python would use power node --- jedi/evaluate/analysis.py | 2 +- jedi/evaluate/finder.py | 2 +- jedi/evaluate/sys_path.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index 07e7c69a..87c961dd 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): def check_hasattr(node, suite): try: assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type == 'power' + assert node.type == 'power' or node.type == 'atom_expr' base = node.children[0] assert base.type == 'name' and base.value == 'hasattr' trailer = node.children[1] diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 63123acc..db9561c3 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -448,7 +448,7 @@ def check_flow_information(evaluator, flow, search_name, pos): def _check_isinstance_type(evaluator, element, search_name): try: - assert element.type == 'power' + assert element.type == 'power' or element.type == 'atom_expr' # this might be removed if we analyze and, etc assert len(element.children) == 2 first, trailer = element.children diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index e838177a..fabb2c1d 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt): for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]): try: assert operator in ['=', '+='] - assert tree.is_node(assignee, 'power') and len(assignee.children) > 1 + assert tree.is_node(assignee, 'power', 'atom_expr') and \ + len(assignee.children) > 1 c = assignee.children assert c[0].type == 'name' and c[0].value == 'sys' trailer = c[1] @@ -152,7 +153,7 @@ def _check_module(evaluator, module): def get_sys_path_powers(names): for name in names: power = name.parent.parent - if tree.is_node(power, 'power'): + if tree.is_node(power, 'power', 'atom_expr'): c = power.children if isinstance(c[0], tree.Name) and c[0].value == 'sys' \ and tree.is_node(c[1], 'trailer'): From 65187930bd9cf8f54233635a99ee7ca04a05eeff Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 21:07:18 +0100 Subject: [PATCH 43/60] Update tokenizer to adhere to PEP492 magic --- jedi/parser/tokenize.py | 55 ++++++++++++++++++++++++++++++++-- test/test_parser/test_pgen2.py | 37 +++++++++++++---------- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 46f9ef76..301d9ff6 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -16,7 +16,7 @@ import re from io import StringIO from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 cookie_re = re.compile("coding[:=]\s*([-\w.]+)") @@ -151,7 +151,58 @@ def source_tokens(source): """Generate tokens from a the source code (string).""" source = source readline = StringIO(source).readline - return generate_tokens(readline) + tokenizer = generate_tokens(readline) + if is_py35: + tokenizer = fix_async_await_tokenizer(tokenizer) + return tokenizer + + +def fix_async_await_tokenizer(tokenizer): + # We need to recognise async functions as per + # https://www.python.org/dev/peps/pep-0492/#transition-plan + # so we need a three item look-ahead + from jedi.parser.token import ASYNC, AWAIT + assert is_py35 + try: + first = next(tokenizer) + except StopIteration: + return + + try: + second = next(tokenizer) + except StopIteration: + yield first + return + + defs_stack = [] + indent = 0 + for item in tokenizer: + if first[0] == INDENT: + indent += 1 + + if first[0] == DEDENT: + indent -= 1 + + if first[0] == NEWLINE and second[0] != INDENT: + while defs_stack and indent <= defs_stack[-1][0]: + defs_stack.pop() + + if second[0] == NAME and second[1] == "def" and item[0] == NAME: + if first[0] == NAME and first[1] == "async": + defs_stack.append((indent, True)) + else: + defs_stack.append((indent, False)) + in_async_def = defs_stack and defs_stack[-1][1] + if in_async_def and first[0] == NAME and first[1] == "async": + yield (ASYNC, ) + first[1:] + elif in_async_def and first[0] == NAME and first[1] == "await": + yield (AWAIT, ) + first[1:] + else: + yield first + first = second + second = item + yield first + yield second def generate_tokens(readline): diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 6d51ae85..576d338f 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b") - parse("a @= b") + parse("a @ b", "3.5") + parse("a @= b", "3.5") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """) + """, "3.5") parse("""async def foo(): @@ -71,46 +71,51 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """) + """, "3.5") - parse("""async def foo(): return await a""") + parse("""async def foo(): return await a""", "3.5") parse("""def foo(): def foo(): pass async def foo(): await x - """) + """, "3.5") - self.invalid_syntax("await x") + self.invalid_syntax("await x", version="3.5") self.invalid_syntax("""def foo(): - await x""") + await x""", version="3.5") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """) + """, version="3.5") + + self.invalid_syntax("""async def foo(): + def foo(): + await x + """, version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_var(self): - parse("""async = 1""") - parse("""await = 1""") - parse("""def async(): pass""") + parse("""async = 1""", "3.5") + parse("""await = 1""", "3.5") + parse("""def async(): pass""", "3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_with(self): parse("""async def foo(): - async for a in b: pass""") + async for a in b: pass""", "3.5") self.invalid_syntax("""def foo(): - async for a in b: pass""") + async for a in b: pass""", version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""") + async with a: pass""", "3.5") self.invalid_syntax("""def foo(): - async with a: pass""") + async with a: pass""", version="3.5") class TestRaiseChanges(GrammarTest): From 4249563eb28e2a56d5dbff8a9e34eda01c0311fb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Feb 2016 21:08:47 +0100 Subject: [PATCH 44/60] tests can now also run on python 3.5 --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 878d1725..1307b252 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py32, py33, py34 +envlist = py26, py27, py32, py33, py34, py35 [testenv] deps = pytest>=2.3.5 @@ -32,6 +32,10 @@ deps = typing {[testenv]deps} [testenv:py34] +deps = + typing + {[testenv]deps} +[testenv:py35] deps = typing {[testenv]deps} From 04524cd63ccdcb33397f40ec431b03f2217ad92e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Feb 2016 13:06:27 +0100 Subject: [PATCH 45/60] make travis test python3.5 as well --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 876af276..e8295ae9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ env: - TOXENV=py32 - TOXENV=py33 - TOXENV=py34 + - TOXENV=py35 - TOXENV=pypy - TOXENV=cov - TOXENV=sith From 3a36bb3a3680969bbb842a11902f962e7a982fb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Feb 2016 16:53:37 +0100 Subject: [PATCH 46/60] Seems necessary to explicitly specify python3.5: https://github.com/travis-ci/travis-ci/issues/4794 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e8295ae9..9331fafa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +python: 3.5 sudo: false env: - TOXENV=py26 @@ -15,8 +16,9 @@ matrix: - env: TOXENV=cov - env: TOXENV=sith - env: TOXENV=pypy + install: - - pip install --quiet --use-mirrors tox + - pip install tox script: - tox after_script: From 7077d0b76240e5b5f5e388d2ae6f02687a95cb61 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Feb 2016 19:14:31 +0100 Subject: [PATCH 47/60] Using python 3.7-like parser, instead of python 3.5 magic. See https://github.com/davidhalter/jedi/pull/691#issuecomment-182815864 Revert "Update tokenizer to adhere to PEP492 magic" This reverts commit 65187930bd9cf8f54233635a99ee7ca04a05eeff. --- jedi/parser/tokenize.py | 55 ++-------------------------------- test/test_parser/test_pgen2.py | 37 ++++++++++------------- 2 files changed, 18 insertions(+), 74 deletions(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 301d9ff6..46f9ef76 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -16,7 +16,7 @@ import re from io import StringIO from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3, is_py35 +from jedi._compatibility import is_py3 cookie_re = re.compile("coding[:=]\s*([-\w.]+)") @@ -151,58 +151,7 @@ def source_tokens(source): """Generate tokens from a the source code (string).""" source = source readline = StringIO(source).readline - tokenizer = generate_tokens(readline) - if is_py35: - tokenizer = fix_async_await_tokenizer(tokenizer) - return tokenizer - - -def fix_async_await_tokenizer(tokenizer): - # We need to recognise async functions as per - # https://www.python.org/dev/peps/pep-0492/#transition-plan - # so we need a three item look-ahead - from jedi.parser.token import ASYNC, AWAIT - assert is_py35 - try: - first = next(tokenizer) - except StopIteration: - return - - try: - second = next(tokenizer) - except StopIteration: - yield first - return - - defs_stack = [] - indent = 0 - for item in tokenizer: - if first[0] == INDENT: - indent += 1 - - if first[0] == DEDENT: - indent -= 1 - - if first[0] == NEWLINE and second[0] != INDENT: - while defs_stack and indent <= defs_stack[-1][0]: - defs_stack.pop() - - if second[0] == NAME and second[1] == "def" and item[0] == NAME: - if first[0] == NAME and first[1] == "async": - defs_stack.append((indent, True)) - else: - defs_stack.append((indent, False)) - in_async_def = defs_stack and defs_stack[-1][1] - if in_async_def and first[0] == NAME and first[1] == "async": - yield (ASYNC, ) + first[1:] - elif in_async_def and first[0] == NAME and first[1] == "await": - yield (AWAIT, ) + first[1:] - else: - yield first - first = second - second = item - yield first - yield second + return generate_tokens(readline) def generate_tokens(readline): diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 576d338f..6d51ae85 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b", "3.5") - parse("a @= b", "3.5") + parse("a @ b") + parse("a @= b") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """, "3.5") + """) parse("""async def foo(): @@ -71,51 +71,46 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """, "3.5") + """) - parse("""async def foo(): return await a""", "3.5") + parse("""async def foo(): return await a""") parse("""def foo(): def foo(): pass async def foo(): await x - """, "3.5") + """) - self.invalid_syntax("await x", version="3.5") + self.invalid_syntax("await x") self.invalid_syntax("""def foo(): - await x""", version="3.5") + await x""") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """, version="3.5") - - self.invalid_syntax("""async def foo(): - def foo(): - await x - """, version="3.5") + """) @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_var(self): - parse("""async = 1""", "3.5") - parse("""await = 1""", "3.5") - parse("""def async(): pass""", "3.5") + parse("""async = 1""") + parse("""await = 1""") + parse("""def async(): pass""") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_with(self): parse("""async def foo(): - async for a in b: pass""", "3.5") + async for a in b: pass""") self.invalid_syntax("""def foo(): - async for a in b: pass""", version="3.5") + async for a in b: pass""") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""", "3.5") + async with a: pass""") self.invalid_syntax("""def foo(): - async with a: pass""", version="3.5") + async with a: pass""") class TestRaiseChanges(GrammarTest): From d5f08f8bdd2efcd3913770b754d667cbb7fe1cc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Feb 2016 19:29:18 +0100 Subject: [PATCH 48/60] opting for skipping PEP492 backwards compatibility magic, instead directly making await and async keywords See discussion at https://github.com/davidhalter/jedi/pull/691#issuecomment-182815864 --- jedi/api/keywords.py | 9 ++++-- jedi/parser/grammar3.5.txt | 8 +++-- test/test_parser/test_pgen2.py | 54 ++++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index 5493f4b1..5af80d12 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -1,7 +1,7 @@ import pydoc import keyword -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from jedi import common from jedi.evaluate.helpers import FakeName from jedi.parser.tree import Leaf @@ -12,7 +12,12 @@ except ImportError: import pydoc_topics if is_py3: - keys = keyword.kwlist + if is_py35: + # in python 3.5 async and await are not proper keywords, but for + # completion pursposes should as as though they are + keys = keyword.kwlist + ["async", "await"] + else: + keys = keyword.kwlist else: keys = keyword.kwlist + ['None', 'False', 'True'] diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt index 9a27f1ec..96a72718 100644 --- a/jedi/parser/grammar3.5.txt +++ b/jedi/parser/grammar3.5.txt @@ -23,7 +23,9 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef +# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens +# skipping python3.5 compatibility, in favour of 3.7 solution +async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' @@ -69,7 +71,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -106,7 +108,7 @@ arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom_expr ['**' factor] -atom_expr: [AWAIT] atom trailer* +atom_expr: ['await'] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py index 6d51ae85..e7b4473b 100644 --- a/test/test_parser/test_pgen2.py +++ b/test/test_parser/test_pgen2.py @@ -46,8 +46,8 @@ class GrammarTest(TestCase): class TestMatrixMultiplication(GrammarTest): @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_matrix_multiplication_operator(self): - parse("a @ b") - parse("a @= b") + parse("a @ b", "3.5") + parse("a @= b", "3.5") class TestYieldFrom(GrammarTest): @@ -62,7 +62,7 @@ class TestAsyncAwait(GrammarTest): def test_await_expr(self): parse("""async def foo(): await x - """) + """, "3.5") parse("""async def foo(): @@ -71,46 +71,56 @@ class TestAsyncAwait(GrammarTest): def foo(): pass await x - """) + """, "3.5") - parse("""async def foo(): return await a""") + parse("""async def foo(): return await a""", "3.5") parse("""def foo(): def foo(): pass async def foo(): await x - """) + """, "3.5") - self.invalid_syntax("await x") + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_await_expr_invalid(self): + self.invalid_syntax("await x", version="3.5") self.invalid_syntax("""def foo(): - await x""") + await x""", version="3.5") self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x - """) + """, version="3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") def test_async_var(self): - parse("""async = 1""") - parse("""await = 1""") - parse("""def async(): pass""") - - @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') - def test_async_with(self): - parse("""async def foo(): - async for a in b: pass""") - - self.invalid_syntax("""def foo(): - async for a in b: pass""") + parse("""async = 1""", "3.5") + parse("""await = 1""", "3.5") + parse("""def async(): pass""", "3.5") @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') def test_async_for(self): parse("""async def foo(): - async with a: pass""") + async for a in b: pass""", "3.5") + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_for_invalid(self): self.invalid_syntax("""def foo(): - async with a: pass""") + async for a in b: pass""", version="3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_async_with(self): + parse("""async def foo(): + async with a: pass""", "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_with_invalid(self): + self.invalid_syntax("""def foo(): + async with a: pass""", version="3.5") class TestRaiseChanges(GrammarTest): From ca08b8270bbf30bce3c0e256b013a3a371c5df3a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 10:20:25 +0100 Subject: [PATCH 49/60] combine power-or-atom_expr statements into one statement --- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/analysis.py | 2 +- jedi/evaluate/dynamic.py | 3 +-- jedi/evaluate/finder.py | 2 +- jedi/evaluate/helpers.py | 2 +- jedi/parser/tree.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index de78f2f5..0de778be 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -291,7 +291,7 @@ class Evaluator(object): types = set([element]) # TODO this is no real evaluation. elif element.type == 'expr_stmt': types = self.eval_statement(element) - elif element.type == 'power' or element.type == 'atom_expr': + elif element.type in ('power', 'atom_expr'): types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index 87c961dd..7b4b0acc 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): def check_hasattr(node, suite): try: assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type == 'power' or node.type == 'atom_expr' + assert node.type in ('power', 'atom_expr') base = node.children[0] assert base.type == 'name' and base.value == 'hasattr' trailer = node.children[1] diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 6ca429fe..8cd08641 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -89,8 +89,7 @@ def search_function_call(evaluator, func): parent = parent.parent trailer = None - if tree.is_node(parent, 'power') or \ - tree.is_node(parent, 'atom_expr'): + if tree.is_node(parent, 'power', 'atom_expr'): for t in parent.children[1:]: if t == '**': break diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index db9561c3..5003848c 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -448,7 +448,7 @@ def check_flow_information(evaluator, flow, search_name, pos): def _check_isinstance_type(evaluator, element, search_name): try: - assert element.type == 'power' or element.type == 'atom_expr' + assert element.type in ('power', 'atom_expr') # this might be removed if we analyze and, etc assert len(element.children) == 2 first, trailer = element.children diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 86eec876..76d1796e 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -88,7 +88,7 @@ def call_of_name(name, cut_own_trailer=False): return name power = par.parent - if (tree.is_node(power, 'power') or tree.is_node(power, 'atom_expr')) \ + if tree.is_node(power, 'power', 'atom_expr') \ and power.children[0] != name \ and not (power.children[-2] == '**' and name.start_pos > power.children[-1].start_pos): diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 867d8cde..89c37841 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1342,7 +1342,7 @@ def _defined_names(current): names += _defined_names(child) elif is_node(current, 'atom'): names += _defined_names(current.children[1]) - elif is_node(current, 'power') or is_node(current, 'atom_expr'): + elif is_node(current, 'power', 'atom_expr'): if current.children[-2] != '**': # Just if there's no operation trailer = current.children[-1] if trailer.children[0] == '.': From daeee4ba0c342c70681b73c1d2b544cb50c45b10 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Dec 2015 01:00:20 +0100 Subject: [PATCH 50/60] simple typehints in comments --- jedi/evaluate/finder.py | 3 +++ jedi/evaluate/pep0484.py | 28 ++++++++++++++++++++++++++++ jedi/parser/tree.py | 9 +++++++++ test/completion/pep0484.py | 24 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 5003848c..1d95e3f7 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -355,6 +355,9 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var + pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt) + if pep0484types: + return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) if check_instance is not None: diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 08588d71..519a217d 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -28,6 +28,7 @@ from jedi.common import unite from jedi.evaluate import compiled from jedi import debug from jedi import _compatibility +import re def _evaluate_for_annotation(evaluator, annotation): @@ -136,3 +137,30 @@ def get_types_for_typing_module(evaluator, typ, node): result = evaluator.execute_evaluated(factory, compiled_classname, args) return result + + +def find_type_from_comment_hint(evaluator, stmt): + try: + stmtpos = stmt.parent.children.index(stmt) + except ValueError: + return [] + try: + next_sibling = stmt.parent.children[stmtpos + 1] + except IndexError: + return [] + if not isinstance(next_sibling, tree.Whitespace): + return [] + comment = next_sibling.get_pre_comment() + if comment is None: + return [] + match = re.match(r"\s*type:\s*([^#]*)", comment) + if not match: + return [] + start_pos = (next_sibling.start_pos[0], + next_sibling.start_pos[1] - len(comment)) + annotation = tree.String( + tree.zero_position_modifier, + repr(str(match.group(1).strip())), + start_pos) + annotation.parent = stmt.parent + return _evaluate_for_annotation(evaluator, annotation) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 89c37841..3f8549d4 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -280,6 +280,15 @@ class Leaf(Base): def nodes_to_execute(self, last_added=False): return [] + def get_pre_comment(self): + """ + returns comment before this leaf, excluding #, or None if no comment + """ + match = re.match(r"\s*#(.*)$", self.prefix) + if match: + return match.group(1) + return None + @utf8_repr def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.value) diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index fc08460f..1990cda0 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,3 +157,27 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x + +# python >= 2.6 + +x = 3 # type: str +#? str() +x + +y = 3 # type: str but I write more +#? int() +y + +z = 3 # type: str # I comment more +#? str() +z + +class BB: pass + +def test(a, b): + a = a # type: BB + c = a # type: str + #? BB() + a + #? str() + c From a658f7940c7c133d81c232434c6eb921d65a6449 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 16:02:59 +0100 Subject: [PATCH 51/60] typehints for variables in comments --- jedi/evaluate/pep0484.py | 18 ++------- jedi/parser/tree.py | 59 ++++++++++++++++++++++++----- test/completion/pep0484.py | 24 ------------ test/completion/pep0484_comments.py | 39 +++++++++++++++++++ 4 files changed, 91 insertions(+), 49 deletions(-) create mode 100644 test/completion/pep0484_comments.py diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 519a217d..260e15e4 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -140,27 +140,15 @@ def get_types_for_typing_module(evaluator, typ, node): def find_type_from_comment_hint(evaluator, stmt): - try: - stmtpos = stmt.parent.children.index(stmt) - except ValueError: - return [] - try: - next_sibling = stmt.parent.children[stmtpos + 1] - except IndexError: - return [] - if not isinstance(next_sibling, tree.Whitespace): - return [] - comment = next_sibling.get_pre_comment() + comment = stmt.get_following_comment_same_line() if comment is None: return [] - match = re.match(r"\s*type:\s*([^#]*)", comment) + match = re.match(r"^#\s*type:\s*([^#]*)", comment) if not match: return [] - start_pos = (next_sibling.start_pos[0], - next_sibling.start_pos[1] - len(comment)) annotation = tree.String( tree.zero_position_modifier, repr(str(match.group(1).strip())), - start_pos) + stmt.start_pos) annotation.parent = stmt.parent return _evaluate_for_annotation(evaluator, annotation) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 3f8549d4..1906d364 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -239,13 +239,35 @@ class Leaf(Base): else: node = c[i - 1] break - while True: try: node = node.children[-1] except AttributeError: # A Leaf doesn't have children. return node + def get_next(self): + """ + Returns the next leaf in the parser tree. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + try: + node = c[i + 1] + except IndexError: + node = node.parent + if node.parent is None: + raise IndexError('Cannot access the next element of the last one.') + else: + break + while True: + try: + node = node.children[0] + except AttributeError: # A Leaf doesn't have children. + return node + + def get_code(self, normalized=False): if normalized: return self.value @@ -264,6 +286,7 @@ class Leaf(Base): except IndexError: return None + def prev_sibling(self): """ The node/leaf immediately preceding the invocant in their parent's @@ -277,18 +300,10 @@ class Leaf(Base): return None return self.parent.children[i - 1] + def nodes_to_execute(self, last_added=False): return [] - def get_pre_comment(self): - """ - returns comment before this leaf, excluding #, or None if no comment - """ - match = re.match(r"\s*#(.*)$", self.prefix) - if match: - return match.group(1) - return None - @utf8_repr def __repr__(self): return "<%s: %s>" % (type(self).__name__, self.value) @@ -497,6 +512,30 @@ class BaseNode(Base): except AttributeError: return self.children[0] + def last_leaf(self): + try: + return self.children[-1].first_leaf() + except AttributeError: + return self.children[-1] + + def get_following_comment_same_line(self): + """ + returns (as string) any comment that appears on the same line, + after the node, including the # + """ + try: + whitespace = self.last_leaf().get_next().prefix + except AttributeError: + return None + if "#" not in whitespace: + return None + comment = whitespace[whitespace.index("#"):] + if "\r" in comment: + comment = comment[:comment.index("\r")] + if "\n" in comment: + comment = comment[:comment.index("\n")] + return comment + @utf8_repr def __repr__(self): code = self.get_code().replace('\n', ' ').strip() diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py index 1990cda0..fc08460f 100644 --- a/test/completion/pep0484.py +++ b/test/completion/pep0484.py @@ -157,27 +157,3 @@ Y = int def just_because_we_can(x: "flo" + "at"): #? float() x - -# python >= 2.6 - -x = 3 # type: str -#? str() -x - -y = 3 # type: str but I write more -#? int() -y - -z = 3 # type: str # I comment more -#? str() -z - -class BB: pass - -def test(a, b): - a = a # type: BB - c = a # type: str - #? BB() - a - #? str() - c diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py new file mode 100644 index 00000000..54ad38c6 --- /dev/null +++ b/test/completion/pep0484_comments.py @@ -0,0 +1,39 @@ +a = 3 # type: str +#? str() +a + +b = 3 # type: str but I write more +#? int() +b + +c = 3 # type: str # I comment more +#? str() +c + +d = "It should not read comments from the next line" +# type: int +#? str() +d + +# type: int +e = "It should not read comments from the previous line" +#? str() +e + +class BB: pass + +def test(a, b): + a = a # type: BB + c = a # type: str + d = a + # type: str + e = a # type: str # Should ignore long whitespace + + #? BB() + a + #? str() + c + #? BB() + d + #? str() + e From 8b28678d1988f5192953a141730661ec52358625 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:04:19 +0100 Subject: [PATCH 52/60] support tuple-assignment --- jedi/evaluate/finder.py | 2 +- jedi/evaluate/pep0484.py | 30 ++++++++++++++++++++++++++--- jedi/parser/tree.py | 6 +++++- test/completion/pep0484_comments.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 1d95e3f7..5cf45434 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -355,7 +355,7 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var - pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt) + pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt, name) if pep0484types: return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 260e15e4..69162178 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -31,10 +31,20 @@ from jedi import _compatibility import re -def _evaluate_for_annotation(evaluator, annotation): +def _evaluate_for_annotation(evaluator, annotation, index=None): + """ + Evaluates a string-node, looking for an annotation + If index is not None, the annotation is expected to be a tuple + and we're interested in that index + """ if annotation is not None: definitions = evaluator.eval_element( _fix_forward_reference(evaluator, annotation)) + if index is not None: + definitions = list(itertools.chain.from_iterable( + definition.py__getitem__(index) for definition in definitions + if definition.type == 'tuple' and + len(list(definition.py__iter__())) >= index)) return list(itertools.chain.from_iterable( evaluator.execute(d) for d in definitions)) else: @@ -139,7 +149,21 @@ def get_types_for_typing_module(evaluator, typ, node): return result -def find_type_from_comment_hint(evaluator, stmt): +def find_type_from_comment_hint(evaluator, stmt, name): + index = None + if stmt.children[0].type == "testlist_star_expr": + # something like "a, b = 1, 2" + leftside = stmt.children[0] + index = 0 + for child in leftside.children: + if child == name: + break + if child.type == "operator": + continue + index += 1 + else: + return [] + comment = stmt.get_following_comment_same_line() if comment is None: return [] @@ -151,4 +175,4 @@ def find_type_from_comment_hint(evaluator, stmt): repr(str(match.group(1).strip())), stmt.start_pos) annotation.parent = stmt.parent - return _evaluate_for_annotation(evaluator, annotation) + return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 1906d364..0625a98a 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -514,7 +514,7 @@ class BaseNode(Base): def last_leaf(self): try: - return self.children[-1].first_leaf() + return self.children[-1].last_leaf() except AttributeError: return self.children[-1] @@ -527,6 +527,10 @@ class BaseNode(Base): whitespace = self.last_leaf().get_next().prefix except AttributeError: return None + except ValueError: + # in some particular cases, the tree doesn't seem to be linked + # correctly + return None if "#" not in whitespace: return None comment = whitespace[whitespace.index("#"):] diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 54ad38c6..e49717e6 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -37,3 +37,32 @@ def test(a, b): d #? str() e + +a,b = 1, 2 # type: str, float +#? str() +a +#? float() +b + +class Employee: + pass + +from typing import List +x = [] # type: List[Employee] +#? Employee() +x[1] +x, y, z = [], [], [] # type: List[int], List[int], List[str] +#? int() +y[2] +x, y, z = [], [], [] # type: (List[float], List[float], List[BB]) +for zi in z: + #? BB() + zi + +x = [ + 1, + 2, +] # type: List[str] + +#? str() +x[1] From 3a1b2e7104da044433b2822526d923d17e67cca7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:37:03 +0100 Subject: [PATCH 53/60] add support for 'for-assignment' hints --- jedi/evaluate/finder.py | 7 ++++++- jedi/evaluate/pep0484.py | 23 ++++++++++++++++------- jedi/parser/tree.py | 5 ++++- test/completion/pep0484_comments.py | 11 +++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 5cf45434..1d98d4dd 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -304,6 +304,10 @@ class NameFinder(object): @memoize_default(set(), evaluator_is_first_arg=True) def _name_to_types(evaluator, name, scope): typ = name.get_definition() + if typ.isinstance(tree.ForStmt): + types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) + if types: + return types if typ.isinstance(tree.ForStmt, tree.CompFor): container_types = evaluator.eval_element(typ.children[3]) for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) @@ -355,7 +359,8 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var - pep0484types = pep0484.find_type_from_comment_hint(evaluator, stmt, name) + pep0484types = \ + pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name) if pep0484types: return pep0484types types |= evaluator.eval_statement(stmt, seek_name=name) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 69162178..91f5251e 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -149,13 +149,22 @@ def get_types_for_typing_module(evaluator, typ, node): return result -def find_type_from_comment_hint(evaluator, stmt, name): +def find_type_from_comment_hint_for(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[1], name) + + +def find_type_from_comment_hint_assign(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[0], name) + + +def _find_type_from_comment_hint(evaluator, node, varlist, name): index = None - if stmt.children[0].type == "testlist_star_expr": + if varlist.type in ("testlist_star_expr", "exprlist"): # something like "a, b = 1, 2" - leftside = stmt.children[0] index = 0 - for child in leftside.children: + for child in varlist.children: if child == name: break if child.type == "operator": @@ -164,7 +173,7 @@ def find_type_from_comment_hint(evaluator, stmt, name): else: return [] - comment = stmt.get_following_comment_same_line() + comment = node.get_following_comment_same_line() if comment is None: return [] match = re.match(r"^#\s*type:\s*([^#]*)", comment) @@ -173,6 +182,6 @@ def find_type_from_comment_hint(evaluator, stmt, name): annotation = tree.String( tree.zero_position_modifier, repr(str(match.group(1).strip())), - stmt.start_pos) - annotation.parent = stmt.parent + node.start_pos) + annotation.parent = node.parent return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 0625a98a..cf33faca 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -524,7 +524,10 @@ class BaseNode(Base): after the node, including the # """ try: - whitespace = self.last_leaf().get_next().prefix + if self.isinstance(ForStmt): + whitespace = self.children[5].first_leaf().prefix + else: + whitespace = self.last_leaf().get_next().prefix except AttributeError: return None except ValueError: diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index e49717e6..606bb09a 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -66,3 +66,14 @@ x = [ #? str() x[1] + + +for bar in foo(): # type: str + #? str() + bar + +for bar, baz in foo(): # type: int, float + #? int() + bar + #? float() + baz From 641fb8077398100179d8dc8a2711f50eac9566fa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:52:21 +0100 Subject: [PATCH 54/60] add support for 'with-assignment' hints --- jedi/evaluate/finder.py | 4 ++++ jedi/evaluate/pep0484.py | 7 +++++++ jedi/parser/tree.py | 2 ++ test/completion/pep0484_comments.py | 7 +++++++ 4 files changed, 20 insertions(+) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 1d98d4dd..4114518a 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -308,6 +308,10 @@ def _name_to_types(evaluator, name, scope): types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) if types: return types + if typ.isinstance(tree.WithStmt): + types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name) + if types: + return types if typ.isinstance(tree.ForStmt, tree.CompFor): container_types = evaluator.eval_element(typ.children[3]) for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 91f5251e..aeb57735 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -154,6 +154,13 @@ def find_type_from_comment_hint_for(evaluator, node, name): _find_type_from_comment_hint(evaluator, node, node.children[1], name) +def find_type_from_comment_hint_with(evaluator, node, name): + assert len(node.children[1].children) == 3, \ + "Can only be here when children[1] is 'foo() as f'" + return _find_type_from_comment_hint( + evaluator, node, node.children[1].children[2], name) + + def find_type_from_comment_hint_assign(evaluator, node, name): return \ _find_type_from_comment_hint(evaluator, node, node.children[0], name) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index cf33faca..133b75ae 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -526,6 +526,8 @@ class BaseNode(Base): try: if self.isinstance(ForStmt): whitespace = self.children[5].first_leaf().prefix + elif self.isinstance(WithStmt): + whitespace = self.children[3].first_leaf().prefix else: whitespace = self.last_leaf().get_next().prefix except AttributeError: diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 606bb09a..8e907065 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -77,3 +77,10 @@ for bar, baz in foo(): # type: int, float bar #? float() baz + +with foo(): # type: int + ... + +with foo() as f: # type: str + #? str() + f From a9ebe71c64f17d6522ec6ad8683ba50728dbeecc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:54:20 +0100 Subject: [PATCH 55/60] add some tests to show that type-hints on the next line don't work --- test/completion/pep0484_comments.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 8e907065..6b8b59bd 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -78,9 +78,23 @@ for bar, baz in foo(): # type: int, float #? float() baz +for bar, baz in foo(): + # type: str, str + """ type hinting on next line should not work """ + #? + bar + #? + baz + with foo(): # type: int ... with foo() as f: # type: str #? str() f + +with foo() as f: + # type: str + """ type hinting on next line should not work """ + #? + f From 71ab855802db983e53f8a46c086d51cfff01fb41 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 17:56:14 +0100 Subject: [PATCH 56/60] update list of things that are completed --- jedi/evaluate/pep0484.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index aeb57735..c5d7a169 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -9,13 +9,13 @@ v Function parameter annotations with builtin/custom type classes v Function returntype annotations with builtin/custom type classes v Function parameter annotations with strings (forward reference) v Function return type annotations with strings (forward reference) -x Local variable type hints +v Local variable type hints v Assigned types: `Url = str\ndef get(url:Url) -> str:` -x Type hints in `with` statements +v Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` -x support for type hint comments `# type: (int, str) -> int`. See comment from - Guido https://github.com/davidhalter/jedi/issues/662 +x support for type hint comments for functions, `# type: (int, str) -> int`. + See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ import itertools From 4fe710d4d3507ae859f12e0182888b3c7c7a6238 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 18:02:11 +0100 Subject: [PATCH 57/60] more tests --- test/completion/pep0484_comments.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py index 6b8b59bd..7707fcc3 100644 --- a/test/completion/pep0484_comments.py +++ b/test/completion/pep0484_comments.py @@ -98,3 +98,8 @@ with foo() as f: """ type hinting on next line should not work """ #? f + +aaa = some_extremely_long_function_name_that_doesnt_leave_room_for_hints() \ + # type: float # We should be able to put hints on the next line with a \ +#? float() +aaa From 795a72a8660fd4e503df1be597497e3f8030e88b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Feb 2016 18:13:47 +0100 Subject: [PATCH 58/60] add `typing.cast()` to the todo list --- jedi/evaluate/pep0484.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index c5d7a169..dc951aaf 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -14,6 +14,7 @@ v Assigned types: `Url = str\ndef get(url:Url) -> str:` v Type hints in `with` statements x Stub files support x support `@no_type_check` and `@no_type_check_decorator` +x support for typing.cast() operator x support for type hint comments for functions, `# type: (int, str) -> int`. See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ From 5a555076bbfe294efb40293d162ddd409b7f84e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Feb 2016 09:59:03 +0100 Subject: [PATCH 59/60] update documentation add python 3.5 and PEP-0484 --- README.rst | 2 +- docs/docs/features.rst | 48 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index de2b4864..7e25acae 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,7 @@ understands, see: `Features `_. A list of caveats can be found on the same page. -You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3 or 3.4, but it should also +You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3, 3.4 or 3.5 but it should also understand/parse code older than those versions. Tips on how to use Jedi efficiently can be found `here diff --git a/docs/docs/features.rst b/docs/docs/features.rst index 48c41ead..12fc5972 100644 --- a/docs/docs/features.rst +++ b/docs/docs/features.rst @@ -24,8 +24,8 @@ General Features - ignores syntax errors and wrong indentation - can deal with complex module / function / class structures - virtualenv support -- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings - (:ref:`type hinting `) +- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings, + and PEP0484-style type hints (:ref:`type hinting `) Supported Python Features @@ -125,7 +125,49 @@ Type Hinting If |jedi| cannot detect the type of a function argument correctly (due to the dynamic nature of Python), you can help it by hinting the type using -one of the following docstring syntax styles: +one of the following docstring/annotation syntax styles: + +**PEP-0484 style** + +https://www.python.org/dev/peps/pep-0484/ + +function annotations (python 3 only; python 2 function annotations with +comments in planned but not yet implemented) + +:: + + def myfunction(node: ProgramNode, foo: str) -> None: + """Do something with a ``node``. + + """ + node.| # complete here + + +assignment, for-loop and with-statement type hints (all python versions). +Note that the type hints must be on the same line as the statement + +:: + + x = foo() # type: int + x, y = 2, 3 # type: typing.Optional[int], typing.Union[int, str] # typing module is mostly supported + for key, value in foo.items(): # type: str, Employee # note that Employee must be in scope + pass + with foo() as f: # type: int + print(f + 3) + +Most of the features in PEP-0484 are supported including the typing module +(for python < 3.5 you have to do ``pip install typing`` to use these), +and forward references. + +Things that are missing (and this is not an exhaustive list; some of these +are planned, others might be hard to implement and provide little worth): + +- annotating functions with comments: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code +- understanding ``typing.cast()`` +- stub files: https://www.python.org/dev/peps/pep-0484/#stub-files +- ``typing.Callable`` +- ``typing.TypeVar`` +- User defined generic types: https://www.python.org/dev/peps/pep-0484/#user-defined-generic-types **Sphinx style** From 09310dae7d599b0f3f7900b27ef4f4845ffa82fd Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 29 Feb 2016 22:19:02 +0100 Subject: [PATCH 60/60] add python 3.5 classifier to setup.py Since 5a555076bbfe294efb40293d162ddd409b7f84e6 3.5 is listed as supported python version in the docs --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 85dee723..ae7c0157 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup(name='jedi', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Text Editors :: Integrated Development Environments (IDE)', 'Topic :: Utilities',