diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 29d73818..3c5b1fdf 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -391,8 +391,7 @@ class Evaluator(object): new_types = set() if trailer_op == '[': - for trailer_typ in iterable.create_index_types(self, node): - new_types |= iterable.py__getitem__(self, types, trailer_typ, trailer_op) + new_types |= iterable.py__getitem__(self, types, trailer) else: for typ in types: debug.dbg('eval_trailer: %s in scope %s', trailer, typ) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index b11b95a1..0c6c20ce 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -29,6 +29,7 @@ from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.cache import CachedMetaClass, memoize_default from jedi.evaluate import analysis +from jedi.evaluate import pep0484 class IterableWrapper(tree.Base): @@ -430,6 +431,9 @@ class ImplicitTuple(_FakeArray): class FakeSequence(_FakeArray): def __init__(self, evaluator, sequence_values, type): + """ + type should be one of "tuple", "list" + """ super(FakeSequence, self).__init__(evaluator, sequence_values, type) self._sequence_values = sequence_values @@ -559,37 +563,56 @@ def py__iter__types(evaluator, types, node=None): return unite(py__iter__(evaluator, types, node)) -def py__getitem__(evaluator, types, index, node): +def py__getitem__(evaluator, types, trailer): + from jedi.evaluate.representation import Class result = set() - # Index handling. - if isinstance(index, (compiled.CompiledObject, Slice)): - index = index.obj + trailer_op, node, trailer_cl = trailer.children + assert trailer_op == "[" + assert trailer_cl == "]" - if type(index) not in (float, int, str, unicode, slice): - # If the index is not clearly defined, we have to get all the - # possiblities. - for typ in list(types): - if isinstance(typ, Array) and typ.type == 'dict': + # special case: PEP0484 typing module, see + # https://github.com/davidhalter/jedi/issues/663 + for typ in list(types): + if isinstance(typ, Class): + typing_module_types = \ + pep0484.get_types_for_typing_module(evaluator, typ, node) + if typing_module_types is not None: types.remove(typ) - result |= typ.dict_values() - return result | py__iter__types(evaluator, types) + result |= typing_module_types - for typ in types: - # The actual getitem call. - try: - getitem = typ.py__getitem__ - except AttributeError: - analysis.add(evaluator, 'type-error-not-subscriptable', node, - message="TypeError: '%s' object is not subscriptable" % typ) - else: + if not types: + # all consumed by special cases + return result + + for index in create_index_types(evaluator, node): + if isinstance(index, (compiled.CompiledObject, Slice)): + index = index.obj + + if type(index) not in (float, int, str, unicode, slice): + # If the index is not clearly defined, we have to get all the + # possiblities. + for typ in list(types): + if isinstance(typ, Array) and typ.type == 'dict': + types.remove(typ) + result |= typ.dict_values() + return result | py__iter__types(evaluator, types) + + for typ in types: + # The actual getitem call. try: - result |= getitem(index) - except IndexError: - result |= py__iter__types(evaluator, set([typ])) - except KeyError: - # Must be a dict. Lists don't raise IndexErrors. - result |= typ.dict_values() + getitem = typ.py__getitem__ + except AttributeError: + analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op, + message="TypeError: '%s' object is not subscriptable" % typ) + else: + try: + result |= getitem(index) + except IndexError: + result |= py__iter__types(evaluator, set([typ])) + except KeyError: + # Must be a dict. Lists don't raise KeyErrors. + result |= typ.dict_values() return result diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py new file mode 100644 index 00000000..f48a5673 --- /dev/null +++ b/jedi/evaluate/jedi_typing.py @@ -0,0 +1,100 @@ +""" +This module is not intended to be used in jedi, rather it will be fed to the +jedi-parser to replace classes in the typing module +""" + +try: + from collections import abc +except ImportError: + # python 2 + import collections as abc + + +def factory(typing_name, indextypes): + class Iterable(abc.Iterable): + def __iter__(self): + while True: + yield indextypes[0]() + + class Iterator(Iterable, abc.Iterator): + def next(self): + """ needed for python 2 """ + return self.__next__() + + def __next__(self): + return indextypes[0]() + + class Sequence(abc.Sequence): + def __getitem__(self, index): + return indextypes[0]() + + class MutableSequence(Sequence, abc.MutableSequence): + pass + + class List(MutableSequence, list): + pass + + class Tuple(Sequence, tuple): + def __getitem__(self, index): + if indextypes[1] == Ellipsis: + # https://www.python.org/dev/peps/pep-0484/#the-typing-module + # Tuple[int, ...] means a tuple of ints of indetermined length + return indextypes[0]() + else: + return indextypes[index]() + + class AbstractSet(Iterable, abc.Set): + pass + + class MutableSet(AbstractSet, abc.MutableSet): + pass + + class KeysView(Iterable, abc.KeysView): + pass + + class ValuesView(abc.ValuesView): + def __iter__(self): + while True: + yield indextypes[1]() + + class ItemsView(abc.ItemsView): + def __iter__(self): + while True: + yield (indextypes[0](), indextypes[1]()) + + class Mapping(Iterable, abc.Mapping): + def __getitem__(self, item): + return indextypes[1]() + + def keys(self): + return KeysView() + + def values(self): + return ValuesView() + + def items(self): + return ItemsView() + + class MutableMapping(Mapping, abc.MutableMapping): + pass + + class Dict(MutableMapping, dict): + pass + + dct = { + "Sequence": Sequence, + "MutableSequence": MutableSequence, + "List": List, + "Iterable": Iterable, + "Iterator": Iterator, + "AbstractSet": AbstractSet, + "MutableSet": MutableSet, + "Mapping": Mapping, + "MutableMapping": MutableMapping, + "Tuple": Tuple, + "KeysView": KeysView, + "ItemsView": ItemsView, + "ValuesView": ValuesView, + "Dict": Dict, + } + return dct[typing_name] diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 4eec2fed..08588d71 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -18,38 +18,53 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ -from itertools import chain +import itertools -from jedi.parser import Parser, load_grammar, ParseError +import os +from jedi.parser import \ + Parser, load_grammar, ParseError, ParserWithRecovery, tree from jedi.evaluate.cache import memoize_default -from jedi.evaluate.compiled import CompiledObject +from jedi.common import unite +from jedi.evaluate import compiled from jedi import debug +from jedi import _compatibility def _evaluate_for_annotation(evaluator, annotation): if annotation is not None: - definitions = set() - for definition in evaluator.eval_element(annotation): - if (isinstance(definition, CompiledObject) and - isinstance(definition.obj, str)): - try: - p = Parser(load_grammar(), definition.obj, start='eval_input') - element = p.get_parsed_node() - except ParseError: - debug.warning('Annotation not parsed: %s' % definition.obj) - else: - module = annotation.get_parent_until() - p.position_modifier.line = module.end_pos[0] - element.parent = module - definitions |= evaluator.eval_element(element) - else: - definitions.add(definition) - return list(chain.from_iterable( + definitions = evaluator.eval_element( + _fix_forward_reference(evaluator, annotation)) + return list(itertools.chain.from_iterable( evaluator.execute(d) for d in definitions)) else: return [] +def _fix_forward_reference(evaluator, node): + evaled_nodes = evaluator.eval_element(node) + if len(evaled_nodes) != 1: + debug.warning("Eval'ed typing index %s should lead to 1 object, " + " not %s" % (node, evaled_nodes)) + return node + evaled_node = list(evaled_nodes)[0] + if isinstance(evaled_node, compiled.CompiledObject) and \ + isinstance(evaled_node.obj, str): + try: + p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj), + start='eval_input') + newnode = p.get_parsed_node() + except ParseError: + debug.warning('Annotation not parsed: %s' % evaled_node.obj) + return node + else: + module = node.get_parent_until() + p.position_modifier.line = module.end_pos[0] + newnode.parent = module + return newnode + else: + return node + + @memoize_default(None, evaluator_is_first_arg=True) def follow_param(evaluator, param): annotation = param.annotation() @@ -60,3 +75,64 @@ def follow_param(evaluator, param): def find_return_types(evaluator, func): annotation = func.py__annotations__().get("return", None) return _evaluate_for_annotation(evaluator, annotation) + + +_typing_module = None + + +def _get_typing_replacement_module(): + """ + The idea is to return our jedi replacement for the PEP-0484 typing module + as discussed at https://github.com/davidhalter/jedi/issues/663 + """ + global _typing_module + if _typing_module is None: + typing_path = \ + os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) + with open(typing_path) as f: + code = _compatibility.unicode(f.read()) + p = ParserWithRecovery(load_grammar(), code) + _typing_module = p.module + return _typing_module + + +def get_types_for_typing_module(evaluator, typ, node): + from jedi.evaluate.iterable import FakeSequence + if not typ.base.get_parent_until().name.value == "typing": + return None + # we assume that any class using [] in a module called + # "typing" with a name for which we have a replacement + # should be replaced by that class. This is not 100% + # airtight but I don't have a better idea to check that it's + # actually the PEP-0484 typing module and not some other + if tree.is_node(node, "subscriptlist"): + nodes = node.children[::2] # skip the commas + else: + nodes = [node] + del node + + nodes = [_fix_forward_reference(evaluator, node) for node in nodes] + + # hacked in Union and Optional, since it's hard to do nicely in parsed code + if typ.name.value == "Union": + return unite(evaluator.eval_element(node) for node in nodes) + if typ.name.value == "Optional": + return evaluator.eval_element(nodes[0]) + + typing = _get_typing_replacement_module() + factories = evaluator.find_types(typing, "factory") + assert len(factories) == 1 + factory = list(factories)[0] + assert factory + function_body_nodes = factory.children[4].children + valid_classnames = set(child.name.value + for child in function_body_nodes + if isinstance(child, tree.Class)) + if typ.name.value not in valid_classnames: + return None + compiled_classname = compiled.create(evaluator, typ.name.value) + + args = FakeSequence(evaluator, nodes, "tuple") + + result = evaluator.execute_evaluated(factory, compiled_classname, args) + return result diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py new file mode 100644 index 00000000..7bee1e64 --- /dev/null +++ b/test/completion/pep0484_typing.py @@ -0,0 +1,261 @@ +""" +Test the typing library, with docstrings. This is needed since annotations +are not supported in python 2.7 else then annotating by comment (and this is +still TODO at 2016-01-23) +""" +import typing +class B: + pass + +def we_can_has_sequence(p, q, r, s, t, u): + """ + :type p: typing.Sequence[int] + :type q: typing.Sequence[B] + :type r: typing.Sequence[int] + :type s: typing.Sequence["int"] + :type t: typing.MutableSequence[dict] + :type u: typing.List[float] + """ + #? ["count"] + p.c + #? int() + p[1] + #? ["count"] + q.c + #? B() + q[1] + #? ["count"] + r.c + #? int() + r[1] + #? ["count"] + s.c + #? int() + s[1] + #? [] + s.a + #? ["append"] + t.a + #? dict() + t[1] + #? ["append"] + u.a + #? float() + u[1] + +def iterators(ps, qs, rs, ts): + """ + :type ps: typing.Iterable[int] + :type qs: typing.Iterator[str] + :type rs: typing.Sequence["ForwardReference"] + :type ts: typing.AbstractSet["float"] + """ + for p in ps: + #? int() + p + #? + next(ps) + a, b = ps + #? int() + a + ##? int() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + # test below is just to make sure that in case it gets fixed by accident + # these tests will be fixed as well the way they should be + #? + b + + for q in qs: + #? str() + q + #? str() + next(qs) + for r in rs: + #? ForwardReference() + r + #? + next(rs) + for t in ts: + #? float() + t + +def sets(p, q): + """ + :type p: typing.AbstractSet[int] + :type q: typing.MutableSet[float] + """ + #? [] + p.a + #? ["add"] + q.a + +def tuple(p, q, r): + """ + :type p: typing.Tuple[int] + :type q: typing.Tuple[int, str, float] + :type r: typing.Tuple[B, ...] + """ + #? int() + p[0] + #? int() + q[0] + #? str() + q[1] + #? float() + q[2] + #? B() + r[0] + #? B() + r[1] + #? B() + r[2] + #? B() + r[10000] + i, s, f = q + #? int() + i + ##? str() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? + s + ##? float() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? + f + +class Key: + pass + +class Value: + pass + +def mapping(p, q, d, r, s, t): + """ + :type p: typing.Mapping[Key, Value] + :type q: typing.MutableMapping[Key, Value] + :type d: typing.Dict[Key, Value] + :type r: typing.KeysView[Key] + :type s: typing.ValuesView[Value] + :type t: typing.ItemsView[Key, Value] + """ + #? [] + p.setd + #? ["setdefault"] + q.setd + #? ["setdefault"] + d.setd + #? Value() + p[1] + for key in p: + #? Key() + key + for key in p.keys(): + #? Key() + key + for value in p.values(): + #? Value() + value + for item in p.items(): + #? Key() + item[0] + #? Value() + item[1] + (key, value) = item + #? Key() + key + #? Value() + value + for key, value in p.items(): + #? Key() + key + #? Value() + value + for key in r: + #? Key() + key + for value in s: + #? Value() + value + for key, value in t: + #? Key() + key + #? Value() + value + +def union(p, q, r, s, t): + """ + :type p: typing.Union[int] + :type q: typing.Union[int, int] + :type r: typing.Union[int, str, "int"] + :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]] + :type t: typing.Union[int, None] + """ + #? int() + p + #? int() + q + #? int() str() + r + #? int() str() float() dict() + s + #? int() + t + +def optional(p): + """ + :type p: typing.Optional[int] + Optional does not do anything special. However it should be recognised + as being of that type. Jedi doesn't do anything with the extra into that + it can be None as well + """ + #? int() + p + +class ForwardReference: + pass + +class TestDict(typing.Dict[str, int]): + def setdud(self): + pass + +def testdict(x): + """ + :type x: TestDict + """ + #? ["setdud", "setdefault"] + x.setd + for key in x.keys(): + #? str() + key + for value in x.values(): + #? int() + value + +x = TestDict() +#? ["setdud", "setdefault"] +x.setd +for key in x.keys(): + #? str() + key +for value in x.values(): + #? int() + value +# python >= 3.2 +""" +docstrings have some auto-import, annotations can use all of Python's +import logic +""" +import typing as t +def union2(x: t.Union[int, str]): + #? int() str() + x + +from typing import Union +def union3(x: Union[int, str]): + #? int() str() + x + +from typing import Union as U +def union4(x: U[int, str]): + #? int() str() + x diff --git a/tox.ini b/tox.ini index 526092fc..1229d735 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,8 @@ deps = docopt # coloroma for colored debug output colorama +# for testing the typing module + typing setenv = # https://github.com/tomchristie/django-rest-framework/issues/1957 # tox corrupts __pycache__, solution from here: