Merge with the linter branch (especially the changes of pep484.

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+python: 3.5
 sudo: false
 env:
     - TOXENV=py26

README.rst

@@ -96,7 +96,7 @@ understands, see: `Features
 <https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
 caveats can be found on the same page.
 
-You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3 or 3.4, but it should also
+You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3, 3.4 or 3.5 but it should also
 understand/parse code older than those versions.
 
 Tips on how to use Jedi efficiently can be found `here

docs/docs/features.rst

@@ -24,8 +24,8 @@ General Features
 - ignores syntax errors and wrong indentation
 - can deal with complex module / function / class structures
 - virtualenv support
-- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings
+- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
-  (:ref:`type hinting <type-hinting>`)
+  and PEP0484-style type hints (:ref:`type hinting <type-hinting>`)
 
 
 Supported Python Features
@@ -126,7 +126,49 @@ Type Hinting
 
 If |jedi| cannot detect the type of a function argument correctly (due to the
 dynamic nature of Python), you can help it by hinting the type using
-one of the following docstring syntax styles:
+one of the following docstring/annotation syntax styles:
+
+**PEP-0484 style**
+
+https://www.python.org/dev/peps/pep-0484/
+
+function annotations (python 3 only; python 2 function annotations with
+comments in planned but not yet implemented)
+
+::
+
+    def myfunction(node: ProgramNode, foo: str) -> None:
+        """Do something with a ``node``.
+
+        """
+        node.| # complete here
+
+
+assignment, for-loop and with-statement type hints (all python versions).
+Note that the type hints must be on the same line as the statement
+
+::
+
+    x = foo()  # type: int
+    x, y = 2, 3  # type: typing.Optional[int], typing.Union[int, str] # typing module is mostly supported
+    for key, value in foo.items():  # type: str, Employee  # note that Employee must be in scope
+        pass
+    with foo() as f:  # type: int
+        print(f + 3)
+
+Most of the features in PEP-0484 are supported including the typing module
+(for python < 3.5 you have to do ``pip install typing`` to use these),
+and forward references.
+
+Things that are missing (and this is not an exhaustive list; some of these
+are planned, others might be hard to implement and provide little worth):
+
+- annotating functions with comments: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code
+- understanding ``typing.cast()``
+- stub files: https://www.python.org/dev/peps/pep-0484/#stub-files
+- ``typing.Callable``
+- ``typing.TypeVar``
+- User defined generic types: https://www.python.org/dev/peps/pep-0484/#user-defined-generic-types
 
 **Sphinx style**
 

jedi/_compatibility.py

@@ -13,6 +13,7 @@ except ImportError:
 
 is_py3 = sys.version_info[0] >= 3
 is_py33 = is_py3 and sys.version_info.minor >= 3
+is_py35 = is_py3 and sys.version_info.minor >= 5
 is_py26 = not is_py3 and sys.version_info[1] < 7
 
 

jedi/api/keywords.py

@@ -1,7 +1,7 @@
 import pydoc
 import keyword
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from jedi import common
 from jedi.evaluate.helpers import FakeName
 from jedi.parser.tree import Leaf
@@ -12,7 +12,12 @@ except ImportError:
     import pydoc_topics
 
 if is_py3:
-    keys = keyword.kwlist
+    if is_py35:
+        # in python 3.5 async and await are not proper keywords, but for
+        # completion pursposes should as as though they are
+        keys = keyword.kwlist + ["async", "await"]
+    else:
+        keys = keyword.kwlist
 else:
     keys = keyword.kwlist + ['None', 'False', 'True']
 

jedi/debug.py

@@ -35,7 +35,12 @@ try:
                 # after every output the stream is reset automatically we don't
                 # need this.
                 initialise.atexit_done = True
-                init()
+                try:
+                    init()
+                except Exception:
+                    # Colorama fails with initializing under vim and is buggy in
+                    # version 0.3.6.
+                    pass
             _inited = True
 
 except ImportError:

jedi/evaluate/__init__.py

@@ -292,7 +292,7 @@ class Evaluator(object):
             types = set([element])  # TODO this is no real evaluation.
         elif element.type == 'expr_stmt':
             types = self.eval_statement(element)
-        elif element.type == 'power':
+        elif element.type in ('power', 'atom_expr'):
             types = self._eval_atom(element.children[0])
             for trailer in element.children[1:]:
                 if trailer == '**':  # has a power operation.
@@ -392,8 +392,7 @@ class Evaluator(object):
 
         new_types = set()
         if trailer_op == '[':
-            for trailer_typ in iterable.create_index_types(self, node):
+            new_types |= iterable.py__getitem__(self, types, trailer)
-                new_types |= iterable.py__getitem__(self, types, trailer_typ, trailer_op)
         else:
             for typ in types:
                 debug.dbg('eval_trailer: %s in scope %s', trailer, typ)

jedi/evaluate/analysis.py

@@ -174,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
     def check_hasattr(node, suite):
         try:
             assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos
-            assert node.type == 'power'
+            assert node.type in ('power', 'atom_expr')
             base = node.children[0]
             assert base.type == 'name' and base.value == 'hasattr'
             trailer = node.children[1]

jedi/evaluate/dynamic.py

@@ -89,7 +89,7 @@ def search_function_call(evaluator, func):
                     parent = parent.parent
 
                 trailer = None
-                if tree.is_node(parent, 'power'):
+                if tree.is_node(parent, 'power', 'atom_expr'):
                     for t in parent.children[1:]:
                         if t == '**':
                             break

jedi/evaluate/finder.py

@@ -311,6 +311,14 @@ class NameFinder(object):
 @memoize_default(set(), evaluator_is_first_arg=True)
 def _name_to_types(evaluator, name, scope):
     typ = name.get_definition()
+    if typ.isinstance(tree.ForStmt):
+        types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.WithStmt):
+        types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name)
+        if types:
+            return types
     if typ.isinstance(tree.ForStmt, tree.CompFor):
         container_types = evaluator.eval_element(typ.children[3])
         for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3])
@@ -362,6 +370,10 @@ def _remove_statements(evaluator, stmt, name):
         check_instance = stmt.instance
         stmt = stmt.var
 
+    pep0484types = \
+        pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name)
+    if pep0484types:
+        return pep0484types
     types |= evaluator.eval_statement(stmt, seek_name=name)
 
     if check_instance is not None:
@@ -455,7 +467,7 @@ def check_flow_information(evaluator, flow, search_name, pos):
 
 def _check_isinstance_type(evaluator, element, search_name):
     try:
-        assert element.type == 'power'
+        assert element.type in ('power', 'atom_expr')
         # this might be removed if we analyze and, etc
         assert len(element.children) == 2
         first, trailer = element.children

jedi/evaluate/helpers.py

@@ -89,7 +89,8 @@ def call_of_name(name, cut_own_trailer=False):
             return name
 
     power = par.parent
-    if tree.is_node(power, 'power') and power.children[0] != name \
+    if tree.is_node(power, 'power', 'atom_expr') \
+            and power.children[0] != name \
             and not (power.children[-2] == '**' and
                      name.start_pos > power.children[-1].start_pos):
         par = power

jedi/evaluate/iterable.py

@@ -29,6 +29,7 @@ from jedi.evaluate import compiled
 from jedi.evaluate import helpers
 from jedi.evaluate.cache import CachedMetaClass, memoize_default
 from jedi.evaluate import analysis
+from jedi.evaluate import pep0484
 
 
 class IterableWrapper(tree.Base):
@@ -430,6 +431,9 @@ class ImplicitTuple(_FakeArray):
 
 class FakeSequence(_FakeArray):
     def __init__(self, evaluator, sequence_values, type):
+        """
+        type should be one of "tuple", "list"
+        """
         super(FakeSequence, self).__init__(evaluator, sequence_values, type)
         self._sequence_values = sequence_values
 
@@ -524,7 +528,7 @@ def unpack_tuple_to_dict(evaluator, types, exprlist):
             analysis.add(evaluator, 'value-error-too-few-values', has_parts,
                          message="ValueError: need more than %s values to unpack" % n)
         return dct
-    elif exprlist.type == 'power':
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
         # Something like ``arr[x], var = ...``.
         # This is something that is not yet supported, would also be difficult
         # to write into a dict.
@@ -559,37 +563,56 @@ def py__iter__types(evaluator, types, node=None):
     return unite(py__iter__(evaluator, types, node))
 
 
-def py__getitem__(evaluator, types, index, node):
+def py__getitem__(evaluator, types, trailer):
+    from jedi.evaluate.representation import Class
     result = set()
 
-    # Index handling.
+    trailer_op, node, trailer_cl = trailer.children
-    if isinstance(index, (compiled.CompiledObject, Slice)):
+    assert trailer_op == "["
-        index = index.obj
+    assert trailer_cl == "]"
 
-    if type(index) not in (float, int, str, unicode, slice):
+    # special case: PEP0484 typing module, see
-        # If the index is not clearly defined, we have to get all the
+    # https://github.com/davidhalter/jedi/issues/663
-        # possiblities.
+    for typ in list(types):
-        for typ in list(types):
+        if isinstance(typ, Class):
-            if isinstance(typ, Array) and typ.type == 'dict':
+            typing_module_types = \
+                pep0484.get_types_for_typing_module(evaluator, typ, node)
+            if typing_module_types is not None:
                 types.remove(typ)
-                result |= typ.dict_values()
+                result |= typing_module_types
-        return result | py__iter__types(evaluator, types)
 
-    for typ in types:
+    if not types:
-        # The actual getitem call.
+        # all consumed by special cases
-        try:
+        return result
-            getitem = typ.py__getitem__
+
-        except AttributeError:
+    for index in create_index_types(evaluator, node):
-            analysis.add(evaluator, 'type-error-not-subscriptable', node,
+        if isinstance(index, (compiled.CompiledObject, Slice)):
-                         message="TypeError: '%s' object is not subscriptable" % typ)
+            index = index.obj
-        else:
+
+        if type(index) not in (float, int, str, unicode, slice):
+            # If the index is not clearly defined, we have to get all the
+            # possiblities.
+            for typ in list(types):
+                if isinstance(typ, Array) and typ.type == 'dict':
+                    types.remove(typ)
+                    result |= typ.dict_values()
+            return result | py__iter__types(evaluator, types)
+
+        for typ in types:
+            # The actual getitem call.
             try:
-                result |= getitem(index)
+                getitem = typ.py__getitem__
-            except IndexError:
+            except AttributeError:
-                result |= py__iter__types(evaluator, set([typ]))
+                analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op,
-            except KeyError:
+                             message="TypeError: '%s' object is not subscriptable" % typ)
-                # Must be a dict. Lists don't raise IndexErrors.
+            else:
-                result |= typ.dict_values()
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= py__iter__types(evaluator, set([typ]))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= typ.dict_values()
     return result
 
 

jedi/evaluate/jedi_typing.py

@@ -0,0 +1,100 @@
+"""
+This module is not intended to be used in jedi, rather it will be fed to the
+jedi-parser to replace classes in the typing module
+"""
+
+try:
+    from collections import abc
+except ImportError:
+    # python 2
+    import collections as abc
+
+
+def factory(typing_name, indextypes):
+    class Iterable(abc.Iterable):
+        def __iter__(self):
+            while True:
+                yield indextypes[0]()
+
+    class Iterator(Iterable, abc.Iterator):
+        def next(self):
+            """ needed for python 2 """
+            return self.__next__()
+
+        def __next__(self):
+            return indextypes[0]()
+
+    class Sequence(abc.Sequence):
+        def __getitem__(self, index):
+            return indextypes[0]()
+
+    class MutableSequence(Sequence, abc.MutableSequence):
+        pass
+
+    class List(MutableSequence, list):
+        pass
+
+    class Tuple(Sequence, tuple):
+        def __getitem__(self, index):
+            if indextypes[1] == Ellipsis:
+                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
+                # Tuple[int, ...] means a tuple of ints of indetermined length
+                return indextypes[0]()
+            else:
+                return indextypes[index]()
+
+    class AbstractSet(Iterable, abc.Set):
+        pass
+
+    class MutableSet(AbstractSet, abc.MutableSet):
+        pass
+
+    class KeysView(Iterable, abc.KeysView):
+        pass
+
+    class ValuesView(abc.ValuesView):
+        def __iter__(self):
+            while True:
+                yield indextypes[1]()
+
+    class ItemsView(abc.ItemsView):
+        def __iter__(self):
+            while True:
+                yield (indextypes[0](), indextypes[1]())
+
+    class Mapping(Iterable, abc.Mapping):
+        def __getitem__(self, item):
+            return indextypes[1]()
+
+        def keys(self):
+            return KeysView()
+
+        def values(self):
+            return ValuesView()
+
+        def items(self):
+            return ItemsView()
+
+    class MutableMapping(Mapping, abc.MutableMapping):
+        pass
+
+    class Dict(MutableMapping, dict):
+        pass
+
+    dct = {
+        "Sequence": Sequence,
+        "MutableSequence": MutableSequence,
+        "List": List,
+        "Iterable": Iterable,
+        "Iterator": Iterator,
+        "AbstractSet": AbstractSet,
+        "MutableSet": MutableSet,
+        "Mapping": Mapping,
+        "MutableMapping": MutableMapping,
+        "Tuple": Tuple,
+        "KeysView": KeysView,
+        "ItemsView": ItemsView,
+        "ValuesView": ValuesView,
+        "Dict": Dict,
+    }
+    return dct[typing_name]

jedi/evaluate/param.py

@@ -47,7 +47,10 @@ class Arguments(tree.Base):
             for el in self.argument_node:
                 yield 0, el
         else:
-            if not tree.is_node(self.argument_node, 'arglist'):
+            if not (tree.is_node(self.argument_node, 'arglist') or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (tree.is_node(self.argument_node, 'argument') and
+                     self.argument_node.children[0] in ('*', '**')))):
                 yield 0, self.argument_node
                 return
 
@@ -57,6 +60,10 @@ class Arguments(tree.Base):
                     continue
                 elif child in ('*', '**'):
                     yield len(child.value), next(iterator)
+                elif tree.is_node(child, 'argument') and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
                 else:
                     yield 0, child
 

jedi/evaluate/pep0484.py

@@ -9,47 +9,74 @@ v Function parameter annotations with builtin/custom type classes
 v Function returntype annotations with builtin/custom type classes
 v Function parameter annotations with strings (forward reference)
 v Function return type annotations with strings (forward reference)
-x Local variable type hints
+v Local variable type hints
 v Assigned types: `Url = str\ndef get(url:Url) -> str:`
-x Type hints in `with` statements
+v Type hints in `with` statements
 x Stub files support
 x support `@no_type_check` and `@no_type_check_decorator`
-x support for type hint comments `# type: (int, str) -> int`. See comment from
+x support for typing.cast() operator
-    Guido https://github.com/davidhalter/jedi/issues/662
+x support for type hint comments for functions, `# type: (int, str) -> int`.
+    See comment from Guido https://github.com/davidhalter/jedi/issues/662
 """
 
-from itertools import chain
+import itertools
 
-from jedi.parser import Parser, load_grammar, ParseError
+import os
+from jedi.parser import \
+    Parser, load_grammar, ParseError, ParserWithRecovery, tree
 from jedi.evaluate.cache import memoize_default
-from jedi.evaluate.compiled import CompiledObject
+from jedi.common import unite
+from jedi.evaluate import compiled
 from jedi import debug
+from jedi import _compatibility
+import re
 
 
-def _evaluate_for_annotation(evaluator, annotation):
+def _evaluate_for_annotation(evaluator, annotation, index=None):
+    """
+    Evaluates a string-node, looking for an annotation
+    If index is not None, the annotation is expected to be a tuple
+    and we're interested in that index
+    """
     if annotation is not None:
-        definitions = set()
+        definitions = evaluator.eval_element(
-        for definition in evaluator.eval_element(annotation):
+            _fix_forward_reference(evaluator, annotation))
-            if (isinstance(definition, CompiledObject) and
+        if index is not None:
-                    isinstance(definition.obj, str)):
+            definitions = list(itertools.chain.from_iterable(
-                try:
+                definition.py__getitem__(index) for definition in definitions
-                    p = Parser(load_grammar(), definition.obj, start_symbol='eval_input')
+                if definition.type == 'tuple' and
-                    element = p.get_parsed_node()
+                len(list(definition.py__iter__())) >= index))
-                except ParseError:
+        return list(itertools.chain.from_iterable(
-                    debug.warning('Annotation not parsed: %s' % definition.obj)
-                else:
-                    module = annotation.get_parent_until()
-                    p.position_modifier.line = module.end_pos[0]
-                    element.parent = module
-                    definitions |= evaluator.eval_element(element)
-            else:
-                definitions.add(definition)
-        return list(chain.from_iterable(
             evaluator.execute(d) for d in definitions))
     else:
         return []
 
 
+def _fix_forward_reference(evaluator, node):
+    evaled_nodes = evaluator.eval_element(node)
+    if len(evaled_nodes) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (node, evaled_nodes))
+        return node
+    evaled_node = list(evaled_nodes)[0]
+    if isinstance(evaled_node, compiled.CompiledObject) and \
+            isinstance(evaled_node.obj, str):
+        try:
+            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
+                       start_symbol='eval_input')
+            newnode = p.get_parsed_node()
+        except ParseError:
+            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
+            return node
+        else:
+            module = node.get_parent_until()
+            p.position_modifier.line = module.end_pos[0]
+            newnode.parent = module
+            return newnode
+    else:
+        return node
+
+
 @memoize_default(None, evaluator_is_first_arg=True)
 def follow_param(evaluator, param):
     annotation = param.annotation()
@@ -60,3 +87,109 @@ def follow_param(evaluator, param):
 def find_return_types(evaluator, func):
     annotation = func.py__annotations__().get("return", None)
     return _evaluate_for_annotation(evaluator, annotation)
+
+
+_typing_module = None
+
+
+def _get_typing_replacement_module():
+    """
+    The idea is to return our jedi replacement for the PEP-0484 typing module
+    as discussed at https://github.com/davidhalter/jedi/issues/663
+    """
+    global _typing_module
+    if _typing_module is None:
+        typing_path = \
+            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
+        with open(typing_path) as f:
+            code = _compatibility.unicode(f.read())
+        p = ParserWithRecovery(load_grammar(), code)
+        _typing_module = p.module
+    return _typing_module
+
+
+def get_types_for_typing_module(evaluator, typ, node):
+    from jedi.evaluate.iterable import FakeSequence
+    if not typ.base.get_parent_until().name.value == "typing":
+        return None
+    # we assume that any class using [] in a module called
+    # "typing" with a name for which we have a replacement
+    # should be replaced by that class. This is not 100%
+    # airtight but I don't have a better idea to check that it's
+    # actually the PEP-0484 typing module and not some other
+    if tree.is_node(node, "subscriptlist"):
+        nodes = node.children[::2]  # skip the commas
+    else:
+        nodes = [node]
+    del node
+
+    nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
+
+    # hacked in Union and Optional, since it's hard to do nicely in parsed code
+    if typ.name.value == "Union":
+        return unite(evaluator.eval_element(node) for node in nodes)
+    if typ.name.value == "Optional":
+        return evaluator.eval_element(nodes[0])
+
+    typing = _get_typing_replacement_module()
+    factories = evaluator.find_types(typing, "factory")
+    assert len(factories) == 1
+    factory = list(factories)[0]
+    assert factory
+    function_body_nodes = factory.children[4].children
+    valid_classnames = set(child.name.value
+                           for child in function_body_nodes
+                           if isinstance(child, tree.Class))
+    if typ.name.value not in valid_classnames:
+        return None
+    compiled_classname = compiled.create(evaluator, typ.name.value)
+
+    args = FakeSequence(evaluator, nodes, "tuple")
+
+    result = evaluator.execute_evaluated(factory, compiled_classname, args)
+    return result
+
+
+def find_type_from_comment_hint_for(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[1], name)
+
+
+def find_type_from_comment_hint_with(evaluator, node, name):
+    assert len(node.children[1].children) == 3, \
+        "Can only be here when children[1] is 'foo() as f'"
+    return _find_type_from_comment_hint(
+        evaluator, node, node.children[1].children[2], name)
+
+
+def find_type_from_comment_hint_assign(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[0], name)
+
+
+def _find_type_from_comment_hint(evaluator, node, varlist, name):
+    index = None
+    if varlist.type in ("testlist_star_expr", "exprlist"):
+        # something like "a, b = 1, 2"
+        index = 0
+        for child in varlist.children:
+            if child == name:
+                break
+            if child.type == "operator":
+                continue
+            index += 1
+        else:
+            return []
+
+    comment = node.get_following_comment_same_line()
+    if comment is None:
+        return []
+    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
+    if not match:
+        return []
+    annotation = tree.String(
+        tree.zero_position_modifier,
+        repr(str(match.group(1).strip())),
+        node.start_pos)
+    annotation.parent = node.parent
+    return _evaluate_for_annotation(evaluator, annotation, index)

jedi/evaluate/sys_path.py

@@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt):
     for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
         try:
             assert operator in ['=', '+=']
-            assert tree.is_node(assignee, 'power') and len(assignee.children) > 1
+            assert tree.is_node(assignee, 'power', 'atom_expr') and \
+                len(assignee.children) > 1
             c = assignee.children
             assert c[0].type == 'name' and c[0].value == 'sys'
             trailer = c[1]
@@ -152,7 +153,7 @@ def _check_module(evaluator, module):
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
-            if tree.is_node(power, 'power'):
+            if tree.is_node(power, 'power', 'atom_expr'):
                 c = power.children
                 if isinstance(c[0], tree.Name) and c[0].value == 'sys' \
                         and tree.is_node(c[1], 'trailer'):

jedi/parser/__init__.py

@@ -46,7 +46,7 @@ def load_grammar(version='3.4'):
     if version in ('3.2', '3.3'):
         version = '3.4'
     elif version == '2.6':
-        version == '2.7'
+        version = '2.7'
 
     file = 'grammar' + version + '.txt'
 

jedi/parser/grammar3.5.txt

@@ -15,15 +15,17 @@
 #       file_input is a module or sequence of commands read from an input file;
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
-single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 
-async_funcdef: ASYNC funcdef
+# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens
+# skipping python3.5 compatibility, in favour of 3.7 solution
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 
 parameters: '(' [typedargslist] ')'
@@ -69,7 +71,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -82,7 +84,9 @@ with_stmt: 'with' with_item (',' with_item)*  ':' suite
 with_item: test ['as' expr]
 # NB compile.c makes sure that the default except clause is last
 except_clause: 'except' [test ['as' NAME]]
-suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
 
 test: or_test ['if' or_test 'else' test] | lambdef
 test_nocond: or_test | lambdef_nocond
@@ -104,7 +108,7 @@ arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom_expr ['**' factor]
-atom_expr: [AWAIT] atom trailer*
+atom_expr: ['await'] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |

jedi/parser/token.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from token import *
 
 
@@ -24,6 +24,11 @@ else:
     tok_name[ELLIPSIS] = 'ELLIPSIS'
     N_TOKENS += 1
 
+if not is_py35:
+    ATEQUAL = N_TOKENS
+    tok_name[ATEQUAL] = 'ATEQUAL'
+    N_TOKENS += 1
+
 
 
 # Map from operator to number (since tokenize doesn't do this)
@@ -68,6 +73,7 @@ opmap_raw = """\
 %= PERCENTEQUAL
 &= AMPEREQUAL
 |= VBAREQUAL
+@= ATEQUAL
 ^= CIRCUMFLEXEQUAL
 <<= LEFTSHIFTEQUAL
 >>= RIGHTSHIFTEQUAL

jedi/parser/tokenize.py

@@ -76,7 +76,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
 # recognized as two instances of =).
 operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
-                 r"[+\-*/%&|^=<>]=?",
+                 r"[+\-*@/%&|^=<>]=?",
                  r"~")
 
 bracket = '[][(){}]'

jedi/parser/tree.py

@@ -589,10 +589,44 @@ class BaseNode(Base):
         c = self.parent.children
         index = c.index(self)
         if index == len(c) - 1:
+            # TODO WTF? recursion?
             return self.get_next_leaf()
         else:
             return c[index + 1]
 
+    def last_leaf(self):
+        try:
+            return self.children[-1].last_leaf()
+        except AttributeError:
+            return self.children[-1]
+
+    def get_following_comment_same_line(self):
+        """
+        returns (as string) any comment that appears on the same line,
+        after the node, including the #
+        """
+        try:
+            if self.isinstance(ForStmt):
+                whitespace = self.children[5].first_leaf().prefix
+            elif self.isinstance(WithStmt):
+                whitespace = self.children[3].first_leaf().prefix
+            else:
+                whitespace = self.last_leaf().get_next_leaf().prefix
+        except AttributeError:
+            return None
+        except ValueError:
+            # TODO in some particular cases, the tree doesn't seem to be linked
+            # correctly
+            return None
+        if "#" not in whitespace:
+            return None
+        comment = whitespace[whitespace.index("#"):]
+        if "\r" in comment:
+            comment = comment[:comment.index("\r")]
+        if "\n" in comment:
+            comment = comment[:comment.index("\n")]
+        return comment
+
     @utf8_repr
     def __repr__(self):
         code = self.get_code().replace('\n', ' ').strip()
@@ -1471,7 +1505,7 @@ def _defined_names(current):
             names += _defined_names(child)
     elif is_node(current, 'atom'):
         names += _defined_names(current.children[1])
-    elif is_node(current, 'power'):
+    elif is_node(current, 'power', 'atom_expr'):
         if current.children[-2] != '**':  # Just if there's no operation
             trailer = current.children[-1]
             if trailer.children[0] == '.':

setup.py

@@ -43,6 +43,7 @@ setup(name='jedi',
           'Programming Language :: Python :: 3.2',
           'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',

test/completion/pep0484_comments.py

@@ -0,0 +1,105 @@
+a = 3  # type: str
+#? str()
+a
+
+b = 3  # type: str but I write more
+#? int()
+b
+
+c = 3  # type: str # I comment more
+#? str()
+c
+
+d = "It should not read comments from the next line"
+# type: int
+#? str()
+d
+
+# type: int
+e = "It should not read comments from the previous line"
+#? str()
+e
+
+class BB: pass
+
+def test(a, b):
+    a = a  # type: BB
+    c = a  # type: str
+    d = a
+    # type: str
+    e = a                 # type: str           # Should ignore long whitespace
+
+    #? BB()
+    a
+    #? str()
+    c
+    #? BB()
+    d
+    #? str()
+    e
+
+a,b = 1, 2 # type: str, float
+#? str()
+a
+#? float()
+b
+
+class Employee:
+    pass
+
+from typing import List
+x = []   # type: List[Employee]
+#? Employee()
+x[1]
+x, y, z = [], [], []  # type: List[int], List[int], List[str]
+#? int()
+y[2]
+x, y, z = [], [], []  # type: (List[float], List[float], List[BB])
+for zi in z:
+    #? BB()
+    zi
+
+x = [
+   1,
+   2,
+]  # type: List[str]
+
+#? str()
+x[1]
+
+
+for bar in foo():  # type: str
+    #? str()
+    bar
+
+for bar, baz in foo():  # type: int, float
+    #? int()
+    bar
+    #? float()
+    baz
+
+for bar, baz in foo():
+    # type: str, str
+    """ type hinting on next line should not work """
+    #?
+    bar
+    #?
+    baz
+
+with foo():  # type: int
+    ...
+
+with foo() as f:  # type: str
+    #? str()
+    f
+
+with foo() as f:
+    # type: str
+    """ type hinting on next line should not work """
+    #?
+    f
+
+aaa = some_extremely_long_function_name_that_doesnt_leave_room_for_hints() \
+    # type: float # We should be able to put hints on the next line with a \
+#? float()
+aaa

test/completion/pep0484_typing.py

@@ -0,0 +1,263 @@
+"""
+Test the typing library, with docstrings. This is needed since annotations
+are not supported in python 2.7 else then annotating by comment (and this is
+still TODO at 2016-01-23)
+"""
+# There's no Python 2.6 typing module.
+# python >= 2.7
+import typing
+class B:
+    pass
+
+def we_can_has_sequence(p, q, r, s, t, u):
+    """
+    :type p: typing.Sequence[int]
+    :type q: typing.Sequence[B]
+    :type r: typing.Sequence[int]
+    :type s: typing.Sequence["int"]
+    :type t: typing.MutableSequence[dict]
+    :type u: typing.List[float]
+    """
+    #? ["count"]
+    p.c
+    #? int()
+    p[1]
+    #? ["count"]
+    q.c
+    #? B()
+    q[1]
+    #? ["count"]
+    r.c
+    #? int()
+    r[1]
+    #? ["count"]
+    s.c
+    #? int()
+    s[1]
+    #? []
+    s.a
+    #? ["append"]
+    t.a
+    #? dict()
+    t[1]
+    #? ["append"]
+    u.a
+    #? float()
+    u[1]
+
+def iterators(ps, qs, rs, ts):
+    """
+    :type ps: typing.Iterable[int]
+    :type qs: typing.Iterator[str]
+    :type rs: typing.Sequence["ForwardReference"]
+    :type ts: typing.AbstractSet["float"]
+    """
+    for p in ps:
+        #? int()
+        p
+    #?
+    next(ps)
+    a, b = ps
+    #? int()
+    a
+    ##? int()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    # test below is just to make sure that in case it gets fixed by accident
+    # these tests will be fixed as well the way they should be
+    #?
+    b
+
+    for q in qs:
+        #? str()
+        q
+    #? str()
+    next(qs)
+    for r in rs:
+        #? ForwardReference()
+        r
+    #?
+    next(rs)
+    for t in ts:
+        #? float()
+        t
+
+def sets(p, q):
+    """
+    :type p: typing.AbstractSet[int]
+    :type q: typing.MutableSet[float]
+    """
+    #? []
+    p.a
+    #? ["add"]
+    q.a
+
+def tuple(p, q, r):
+    """
+    :type p: typing.Tuple[int]
+    :type q: typing.Tuple[int, str, float]
+    :type r: typing.Tuple[B, ...]
+    """
+    #? int()
+    p[0]
+    #? int()
+    q[0]
+    #? str()
+    q[1]
+    #? float()
+    q[2]
+    #? B()
+    r[0]
+    #? B()
+    r[1]
+    #? B()
+    r[2]
+    #? B()
+    r[10000]
+    i, s, f = q
+    #? int()
+    i
+    ##? str()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    s
+    ##? float()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    f
+
+class Key:
+    pass
+
+class Value:
+    pass
+
+def mapping(p, q, d, r, s, t):
+    """
+    :type p: typing.Mapping[Key, Value]
+    :type q: typing.MutableMapping[Key, Value]
+    :type d: typing.Dict[Key, Value]
+    :type r: typing.KeysView[Key]
+    :type s: typing.ValuesView[Value]
+    :type t: typing.ItemsView[Key, Value]
+    """
+    #? []
+    p.setd
+    #? ["setdefault"]
+    q.setd
+    #? ["setdefault"]
+    d.setd
+    #? Value()
+    p[1]
+    for key in p:
+        #? Key()
+        key
+    for key in p.keys():
+        #? Key()
+        key
+    for value in p.values():
+        #? Value()
+        value
+    for item in p.items():
+        #? Key()
+        item[0]
+        #? Value()
+        item[1]
+        (key, value) = item
+        #? Key()
+        key
+        #? Value()
+        value
+    for key, value in p.items():
+        #? Key()
+        key
+        #? Value()
+        value
+    for key in r:
+        #? Key()
+        key
+    for value in s:
+        #? Value()
+        value
+    for key, value in t:
+        #? Key()
+        key
+        #? Value()
+        value
+
+def union(p, q, r, s, t):
+    """
+    :type p: typing.Union[int]
+    :type q: typing.Union[int, int]
+    :type r: typing.Union[int, str, "int"]
+    :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]]
+    :type t: typing.Union[int, None]
+    """
+    #? int()
+    p
+    #? int()
+    q
+    #? int() str()
+    r
+    #? int() str() float() dict()
+    s
+    #? int()
+    t
+
+def optional(p):
+    """
+    :type p: typing.Optional[int]
+    Optional does not do anything special. However it should be recognised
+    as being of that type. Jedi doesn't do anything with the extra into that
+    it can be None as well
+    """
+    #? int()
+    p
+
+class ForwardReference:
+    pass
+
+class TestDict(typing.Dict[str, int]):
+    def setdud(self):
+        pass
+
+def testdict(x):
+    """
+    :type x: TestDict
+    """
+    #? ["setdud", "setdefault"]
+    x.setd
+    for key in x.keys():
+        #? str()
+        key
+    for value in x.values():
+        #? int()
+        value
+
+x = TestDict()
+#? ["setdud", "setdefault"]
+x.setd
+for key in x.keys():
+    #? str()
+    key
+for value in x.values():
+    #? int()
+    value
+# python >= 3.2
+"""
+docstrings have some auto-import, annotations can use all of Python's
+import logic
+"""
+import typing as t
+def union2(x: t.Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union
+def union3(x: Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union as U
+def union4(x: U[int, str]):
+    #? int() str()
+    x

test/run.py

@@ -249,14 +249,16 @@ def skip_python_version(line):
             map(int, match.group(2).split(".")))
         operation = getattr(operator, comp_map[match.group(1)])
         if not operation(sys.version_info, minimal_python_version):
-            return "Minimal python version %s" % match.group(1)
+            return "Minimal python version %s %s" % (match.group(1), match.group(2))
 
     return None
 
 
-def collect_file_tests(lines, lines_to_execute):
+def collect_file_tests(path, lines, lines_to_execute):
-    makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column,
+    def makecase(t):
-                                             start, line, path=None, skip=skip)
+        return IntegrationTestCase(t, correct, line_nr, column,
+                                   start, line, path=path, skip=skip)
+
     start = None
     correct = None
     test_type = None
@@ -325,9 +327,8 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
             else:
                 source = unicode(open(path).read(), 'UTF-8')
 
-            for case in collect_file_tests(StringIO(source),
+            for case in collect_file_tests(path, StringIO(source),
                                            lines_to_execute):
-                case.path = path
                 case.source = source
                 if skip:
                     case.skip = skip

test/test_parser/test_pgen2.py

@@ -9,6 +9,7 @@ test_grammar.py files from both Python 2 and Python 3.
 from textwrap import dedent
 
 
+from jedi._compatibility import unicode
 from jedi.parser import Parser, load_grammar, ParseError
 import pytest
 
@@ -18,7 +19,7 @@ from test.helpers import TestCase
 def parse(code, version='3.4'):
     code = dedent(code) + "\n\n"
     grammar = load_grammar(version=version)
-    return Parser(grammar, code, 'file_input').get_parsed_node()
+    return Parser(grammar, unicode(code), 'file_input').get_parsed_node()
 
 
 class TestDriver(TestCase):
@@ -45,8 +46,8 @@ class GrammarTest(TestCase):
 class TestMatrixMultiplication(GrammarTest):
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_matrix_multiplication_operator(self):
-        parse("a @ b")
+        parse("a @ b", "3.5")
-        parse("a @= b")
+        parse("a @= b", "3.5")
 
 
 class TestYieldFrom(GrammarTest):
@@ -61,7 +62,7 @@ class TestAsyncAwait(GrammarTest):
     def test_await_expr(self):
         parse("""async def foo():
                              await x
-                      """)
+                      """, "3.5")
 
         parse("""async def foo():
 
@@ -70,46 +71,56 @@ class TestAsyncAwait(GrammarTest):
             def foo(): pass
 
             await x
-        """)
+        """, "3.5")
 
-        parse("""async def foo(): return await a""")
+        parse("""async def foo(): return await a""", "3.5")
 
         parse("""def foo():
             def foo(): pass
             async def foo(): await x
-        """)
+        """, "3.5")
 
-        self.invalid_syntax("await x")
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_await_expr_invalid(self):
+        self.invalid_syntax("await x", version="3.5")
         self.invalid_syntax("""def foo():
-                                   await x""")
+                                   await x""", version="3.5")
 
         self.invalid_syntax("""def foo():
             def foo(): pass
             async def foo(): pass
             await x
-        """)
+        """, version="3.5")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
     def test_async_var(self):
-        parse("""async = 1""")
+        parse("""async = 1""", "3.5")
-        parse("""await = 1""")
+        parse("""await = 1""", "3.5")
-        parse("""def async(): pass""")
+        parse("""def async(): pass""", "3.5")
-
-    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
-    def test_async_with(self):
-        parse("""async def foo():
-                             async for a in b: pass""")
-
-        self.invalid_syntax("""def foo():
-                                   async for a in b: pass""")
 
     @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
     def test_async_for(self):
         parse("""async def foo():
-                             async with a: pass""")
+                             async for a in b: pass""", "3.5")
 
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_for_invalid(self):
         self.invalid_syntax("""def foo():
-                                   async with a: pass""")
+                                   async for a in b: pass""", version="3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_async_with(self):
+        parse("""async def foo():
+                             async with a: pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_with_invalid(self):
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""", version="3.5")
 
 
 class TestRaiseChanges(GrammarTest):
@@ -232,6 +243,9 @@ class TestParserIdempotency(TestCase):
 
 
 class TestLiterals(GrammarTest):
+    # It's not possible to get the same result when using \xaa in Python 2/3,
+    # because it's treated differently.
+    @pytest.mark.skipif('sys.version_info[0] < 3')
     def test_multiline_bytes_literals(self):
         s = """
             md5test(b"\xaa" * 80,
@@ -250,6 +264,7 @@ class TestLiterals(GrammarTest):
             '''
         parse(s)
 
+    @pytest.mark.skipif('sys.version_info[0] < 3')
     def test_multiline_str_literals(self):
         s = """
             md5test("\xaa" * 80,

tox.ini

@@ -18,10 +18,26 @@ commands =
 deps =
     unittest2
     {[testenv]deps}
+[testenv:py27]
+deps =
+# for testing the typing module
+    typing
+    {[testenv]deps}
 [testenv:py32]
 deps =
-    pip<8.0.0
+    typing
-    virtualenv<14.0.0
+    {[testenv]deps}
+[testenv:py33]
+deps =
+    typing
+    {[testenv]deps}
+[testenv:py34]
+deps =
+    typing
+    {[testenv]deps}
+[testenv:py35]
+deps =
+    typing
     {[testenv]deps}
 [testenv:cov]
 deps =