Merge pull request #663 from reinhrst/typing

PEP 484 typing library

jedi/evaluate/__init__.py

@@ -391,8 +391,7 @@ class Evaluator(object):
 
         new_types = set()
         if trailer_op == '[':
-            for trailer_typ in iterable.create_index_types(self, node):
+            new_types |= iterable.py__getitem__(self, types, trailer)
-                new_types |= iterable.py__getitem__(self, types, trailer_typ, trailer_op)
         else:
             for typ in types:
                 debug.dbg('eval_trailer: %s in scope %s', trailer, typ)

jedi/evaluate/iterable.py

@@ -29,6 +29,7 @@ from jedi.evaluate import compiled
 from jedi.evaluate import helpers
 from jedi.evaluate.cache import CachedMetaClass, memoize_default
 from jedi.evaluate import analysis
+from jedi.evaluate import pep0484
 
 
 class IterableWrapper(tree.Base):
@@ -430,6 +431,9 @@ class ImplicitTuple(_FakeArray):
 
 class FakeSequence(_FakeArray):
     def __init__(self, evaluator, sequence_values, type):
+        """
+        type should be one of "tuple", "list"
+        """
         super(FakeSequence, self).__init__(evaluator, sequence_values, type)
         self._sequence_values = sequence_values
 
@@ -559,37 +563,56 @@ def py__iter__types(evaluator, types, node=None):
     return unite(py__iter__(evaluator, types, node))
 
 
-def py__getitem__(evaluator, types, index, node):
+def py__getitem__(evaluator, types, trailer):
+    from jedi.evaluate.representation import Class
     result = set()
 
-    # Index handling.
+    trailer_op, node, trailer_cl = trailer.children
-    if isinstance(index, (compiled.CompiledObject, Slice)):
+    assert trailer_op == "["
-        index = index.obj
+    assert trailer_cl == "]"
 
-    if type(index) not in (float, int, str, unicode, slice):
+    # special case: PEP0484 typing module, see
-        # If the index is not clearly defined, we have to get all the
+    # https://github.com/davidhalter/jedi/issues/663
-        # possiblities.
+    for typ in list(types):
-        for typ in list(types):
+        if isinstance(typ, Class):
-            if isinstance(typ, Array) and typ.type == 'dict':
+            typing_module_types = \
+                pep0484.get_types_for_typing_module(evaluator, typ, node)
+            if typing_module_types is not None:
                 types.remove(typ)
-                result |= typ.dict_values()
+                result |= typing_module_types
-        return result | py__iter__types(evaluator, types)
 
-    for typ in types:
+    if not types:
-        # The actual getitem call.
+        # all consumed by special cases
-        try:
+        return result
-            getitem = typ.py__getitem__
+
-        except AttributeError:
+    for index in create_index_types(evaluator, node):
-            analysis.add(evaluator, 'type-error-not-subscriptable', node,
+        if isinstance(index, (compiled.CompiledObject, Slice)):
-                         message="TypeError: '%s' object is not subscriptable" % typ)
+            index = index.obj
-        else:
+
+        if type(index) not in (float, int, str, unicode, slice):
+            # If the index is not clearly defined, we have to get all the
+            # possiblities.
+            for typ in list(types):
+                if isinstance(typ, Array) and typ.type == 'dict':
+                    types.remove(typ)
+                    result |= typ.dict_values()
+            return result | py__iter__types(evaluator, types)
+
+        for typ in types:
+            # The actual getitem call.
             try:
-                result |= getitem(index)
+                getitem = typ.py__getitem__
-            except IndexError:
+            except AttributeError:
-                result |= py__iter__types(evaluator, set([typ]))
+                analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op,
-            except KeyError:
+                             message="TypeError: '%s' object is not subscriptable" % typ)
-                # Must be a dict. Lists don't raise IndexErrors.
+            else:
-                result |= typ.dict_values()
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= py__iter__types(evaluator, set([typ]))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= typ.dict_values()
     return result
 
 

jedi/evaluate/jedi_typing.py

@@ -0,0 +1,100 @@
+"""
+This module is not intended to be used in jedi, rather it will be fed to the
+jedi-parser to replace classes in the typing module
+"""
+
+try:
+    from collections import abc
+except ImportError:
+    # python 2
+    import collections as abc
+
+
+def factory(typing_name, indextypes):
+    class Iterable(abc.Iterable):
+        def __iter__(self):
+            while True:
+                yield indextypes[0]()
+
+    class Iterator(Iterable, abc.Iterator):
+        def next(self):
+            """ needed for python 2 """
+            return self.__next__()
+
+        def __next__(self):
+            return indextypes[0]()
+
+    class Sequence(abc.Sequence):
+        def __getitem__(self, index):
+            return indextypes[0]()
+
+    class MutableSequence(Sequence, abc.MutableSequence):
+        pass
+
+    class List(MutableSequence, list):
+        pass
+
+    class Tuple(Sequence, tuple):
+        def __getitem__(self, index):
+            if indextypes[1] == Ellipsis:
+                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
+                # Tuple[int, ...] means a tuple of ints of indetermined length
+                return indextypes[0]()
+            else:
+                return indextypes[index]()
+
+    class AbstractSet(Iterable, abc.Set):
+        pass
+
+    class MutableSet(AbstractSet, abc.MutableSet):
+        pass
+
+    class KeysView(Iterable, abc.KeysView):
+        pass
+
+    class ValuesView(abc.ValuesView):
+        def __iter__(self):
+            while True:
+                yield indextypes[1]()
+
+    class ItemsView(abc.ItemsView):
+        def __iter__(self):
+            while True:
+                yield (indextypes[0](), indextypes[1]())
+
+    class Mapping(Iterable, abc.Mapping):
+        def __getitem__(self, item):
+            return indextypes[1]()
+
+        def keys(self):
+            return KeysView()
+
+        def values(self):
+            return ValuesView()
+
+        def items(self):
+            return ItemsView()
+
+    class MutableMapping(Mapping, abc.MutableMapping):
+        pass
+
+    class Dict(MutableMapping, dict):
+        pass
+
+    dct = {
+        "Sequence": Sequence,
+        "MutableSequence": MutableSequence,
+        "List": List,
+        "Iterable": Iterable,
+        "Iterator": Iterator,
+        "AbstractSet": AbstractSet,
+        "MutableSet": MutableSet,
+        "Mapping": Mapping,
+        "MutableMapping": MutableMapping,
+        "Tuple": Tuple,
+        "KeysView": KeysView,
+        "ItemsView": ItemsView,
+        "ValuesView": ValuesView,
+        "Dict": Dict,
+    }
+    return dct[typing_name]

jedi/evaluate/pep0484.py

@@ -18,38 +18,53 @@ x support for type hint comments `# type: (int, str) -> int`. See comment from
     Guido https://github.com/davidhalter/jedi/issues/662
 """
 
-from itertools import chain
+import itertools
 
-from jedi.parser import Parser, load_grammar, ParseError
+import os
+from jedi.parser import \
+    Parser, load_grammar, ParseError, ParserWithRecovery, tree
 from jedi.evaluate.cache import memoize_default
-from jedi.evaluate.compiled import CompiledObject
+from jedi.common import unite
+from jedi.evaluate import compiled
 from jedi import debug
+from jedi import _compatibility
 
 
 def _evaluate_for_annotation(evaluator, annotation):
     if annotation is not None:
-        definitions = set()
+        definitions = evaluator.eval_element(
-        for definition in evaluator.eval_element(annotation):
+            _fix_forward_reference(evaluator, annotation))
-            if (isinstance(definition, CompiledObject) and
+        return list(itertools.chain.from_iterable(
-                    isinstance(definition.obj, str)):
-                try:
-                    p = Parser(load_grammar(), definition.obj, start='eval_input')
-                    element = p.get_parsed_node()
-                except ParseError:
-                    debug.warning('Annotation not parsed: %s' % definition.obj)
-                else:
-                    module = annotation.get_parent_until()
-                    p.position_modifier.line = module.end_pos[0]
-                    element.parent = module
-                    definitions |= evaluator.eval_element(element)
-            else:
-                definitions.add(definition)
-        return list(chain.from_iterable(
             evaluator.execute(d) for d in definitions))
     else:
         return []
 
 
+def _fix_forward_reference(evaluator, node):
+    evaled_nodes = evaluator.eval_element(node)
+    if len(evaled_nodes) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (node, evaled_nodes))
+        return node
+    evaled_node = list(evaled_nodes)[0]
+    if isinstance(evaled_node, compiled.CompiledObject) and \
+            isinstance(evaled_node.obj, str):
+        try:
+            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
+                       start='eval_input')
+            newnode = p.get_parsed_node()
+        except ParseError:
+            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
+            return node
+        else:
+            module = node.get_parent_until()
+            p.position_modifier.line = module.end_pos[0]
+            newnode.parent = module
+            return newnode
+    else:
+        return node
+
+
 @memoize_default(None, evaluator_is_first_arg=True)
 def follow_param(evaluator, param):
     annotation = param.annotation()
@@ -60,3 +75,64 @@ def follow_param(evaluator, param):
 def find_return_types(evaluator, func):
     annotation = func.py__annotations__().get("return", None)
     return _evaluate_for_annotation(evaluator, annotation)
+
+
+_typing_module = None
+
+
+def _get_typing_replacement_module():
+    """
+    The idea is to return our jedi replacement for the PEP-0484 typing module
+    as discussed at https://github.com/davidhalter/jedi/issues/663
+    """
+    global _typing_module
+    if _typing_module is None:
+        typing_path = \
+            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
+        with open(typing_path) as f:
+            code = _compatibility.unicode(f.read())
+        p = ParserWithRecovery(load_grammar(), code)
+        _typing_module = p.module
+    return _typing_module
+
+
+def get_types_for_typing_module(evaluator, typ, node):
+    from jedi.evaluate.iterable import FakeSequence
+    if not typ.base.get_parent_until().name.value == "typing":
+        return None
+    # we assume that any class using [] in a module called
+    # "typing" with a name for which we have a replacement
+    # should be replaced by that class. This is not 100%
+    # airtight but I don't have a better idea to check that it's
+    # actually the PEP-0484 typing module and not some other
+    if tree.is_node(node, "subscriptlist"):
+        nodes = node.children[::2]  # skip the commas
+    else:
+        nodes = [node]
+    del node
+
+    nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
+
+    # hacked in Union and Optional, since it's hard to do nicely in parsed code
+    if typ.name.value == "Union":
+        return unite(evaluator.eval_element(node) for node in nodes)
+    if typ.name.value == "Optional":
+        return evaluator.eval_element(nodes[0])
+
+    typing = _get_typing_replacement_module()
+    factories = evaluator.find_types(typing, "factory")
+    assert len(factories) == 1
+    factory = list(factories)[0]
+    assert factory
+    function_body_nodes = factory.children[4].children
+    valid_classnames = set(child.name.value
+                           for child in function_body_nodes
+                           if isinstance(child, tree.Class))
+    if typ.name.value not in valid_classnames:
+        return None
+    compiled_classname = compiled.create(evaluator, typ.name.value)
+
+    args = FakeSequence(evaluator, nodes, "tuple")
+
+    result = evaluator.execute_evaluated(factory, compiled_classname, args)
+    return result

test/completion/pep0484_typing.py

@@ -0,0 +1,261 @@
+"""
+Test the typing library, with docstrings. This is needed since annotations
+are not supported in python 2.7 else then annotating by comment (and this is
+still TODO at 2016-01-23)
+"""
+import typing
+class B:
+    pass
+
+def we_can_has_sequence(p, q, r, s, t, u):
+    """
+    :type p: typing.Sequence[int]
+    :type q: typing.Sequence[B]
+    :type r: typing.Sequence[int]
+    :type s: typing.Sequence["int"]
+    :type t: typing.MutableSequence[dict]
+    :type u: typing.List[float]
+    """
+    #? ["count"]
+    p.c
+    #? int()
+    p[1]
+    #? ["count"]
+    q.c
+    #? B()
+    q[1]
+    #? ["count"]
+    r.c
+    #? int()
+    r[1]
+    #? ["count"]
+    s.c
+    #? int()
+    s[1]
+    #? []
+    s.a
+    #? ["append"]
+    t.a
+    #? dict()
+    t[1]
+    #? ["append"]
+    u.a
+    #? float()
+    u[1]
+
+def iterators(ps, qs, rs, ts):
+    """
+    :type ps: typing.Iterable[int]
+    :type qs: typing.Iterator[str]
+    :type rs: typing.Sequence["ForwardReference"]
+    :type ts: typing.AbstractSet["float"]
+    """
+    for p in ps:
+        #? int()
+        p
+    #?
+    next(ps)
+    a, b = ps
+    #? int()
+    a
+    ##? int()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    # test below is just to make sure that in case it gets fixed by accident
+    # these tests will be fixed as well the way they should be
+    #?
+    b
+
+    for q in qs:
+        #? str()
+        q
+    #? str()
+    next(qs)
+    for r in rs:
+        #? ForwardReference()
+        r
+    #?
+    next(rs)
+    for t in ts:
+        #? float()
+        t
+
+def sets(p, q):
+    """
+    :type p: typing.AbstractSet[int]
+    :type q: typing.MutableSet[float]
+    """
+    #? []
+    p.a
+    #? ["add"]
+    q.a
+
+def tuple(p, q, r):
+    """
+    :type p: typing.Tuple[int]
+    :type q: typing.Tuple[int, str, float]
+    :type r: typing.Tuple[B, ...]
+    """
+    #? int()
+    p[0]
+    #? int()
+    q[0]
+    #? str()
+    q[1]
+    #? float()
+    q[2]
+    #? B()
+    r[0]
+    #? B()
+    r[1]
+    #? B()
+    r[2]
+    #? B()
+    r[10000]
+    i, s, f = q
+    #? int()
+    i
+    ##? str()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    s
+    ##? float()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    f
+
+class Key:
+    pass
+
+class Value:
+    pass
+
+def mapping(p, q, d, r, s, t):
+    """
+    :type p: typing.Mapping[Key, Value]
+    :type q: typing.MutableMapping[Key, Value]
+    :type d: typing.Dict[Key, Value]
+    :type r: typing.KeysView[Key]
+    :type s: typing.ValuesView[Value]
+    :type t: typing.ItemsView[Key, Value]
+    """
+    #? []
+    p.setd
+    #? ["setdefault"]
+    q.setd
+    #? ["setdefault"]
+    d.setd
+    #? Value()
+    p[1]
+    for key in p:
+        #? Key()
+        key
+    for key in p.keys():
+        #? Key()
+        key
+    for value in p.values():
+        #? Value()
+        value
+    for item in p.items():
+        #? Key()
+        item[0]
+        #? Value()
+        item[1]
+        (key, value) = item
+        #? Key()
+        key
+        #? Value()
+        value
+    for key, value in p.items():
+        #? Key()
+        key
+        #? Value()
+        value
+    for key in r:
+        #? Key()
+        key
+    for value in s:
+        #? Value()
+        value
+    for key, value in t:
+        #? Key()
+        key
+        #? Value()
+        value
+
+def union(p, q, r, s, t):
+    """
+    :type p: typing.Union[int]
+    :type q: typing.Union[int, int]
+    :type r: typing.Union[int, str, "int"]
+    :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]]
+    :type t: typing.Union[int, None]
+    """
+    #? int()
+    p
+    #? int()
+    q
+    #? int() str()
+    r
+    #? int() str() float() dict()
+    s
+    #? int()
+    t
+
+def optional(p):
+    """
+    :type p: typing.Optional[int]
+    Optional does not do anything special. However it should be recognised
+    as being of that type. Jedi doesn't do anything with the extra into that
+    it can be None as well
+    """
+    #? int()
+    p
+
+class ForwardReference:
+    pass
+
+class TestDict(typing.Dict[str, int]):
+    def setdud(self):
+        pass
+
+def testdict(x):
+    """
+    :type x: TestDict
+    """
+    #? ["setdud", "setdefault"]
+    x.setd
+    for key in x.keys():
+        #? str()
+        key
+    for value in x.values():
+        #? int()
+        value
+
+x = TestDict()
+#? ["setdud", "setdefault"]
+x.setd
+for key in x.keys():
+    #? str()
+    key
+for value in x.values():
+    #? int()
+    value
+# python >= 3.2
+"""
+docstrings have some auto-import, annotations can use all of Python's
+import logic
+"""
+import typing as t
+def union2(x: t.Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union
+def union3(x: Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union as U
+def union4(x: U[int, str]):
+    #? int() str()
+    x

tox.ini

@@ -8,6 +8,8 @@ deps =
     docopt
 # coloroma for colored debug output
     colorama
+# for testing the typing module
+    typing
 setenv =
 # https://github.com/tomchristie/django-rest-framework/issues/1957
 # tox corrupts __pycache__, solution from here: