Merge branch 'pep484' into linter

AUTHORS.txt

@@ -35,5 +35,6 @@ Ian Lee (@IanLee1521) <IanLee1521@gmail.com>
 Farkhad Khatamov (@hatamov) <comsgn@gmail.com>
 Kevin Kelley (@kelleyk) <kelleyk@kelleyk.net>
 Sid Shanker (@squidarth) <sid.p.shanker@gmail.com>
+Reinoud Elhorst (@reinhrst)
 
 Note: (@user) means a github user name.

jedi/api/__init__.py

@@ -322,13 +322,8 @@ class Script(object):
 
     @memoize_default()
     def _get_under_cursor_stmt(self, cursor_txt, start_pos=None):
-        tokenizer = source_tokens(cursor_txt)
+        stmt = Parser(self._grammar, cursor_txt, 'eval_input').get_parsed_node()
-        r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer)
+        if stmt is None:
-        try:
-            # Take the last statement available that is not an endmarker.
-            # And because it's a simple_stmt, we need to get the first child.
-            stmt = r.module.children[-2].children[0]
-        except (AttributeError, IndexError):
             return None
 
         user_stmt = self._parser.user_stmt()

jedi/evaluate/__init__.py

@@ -305,6 +305,8 @@ class Evaluator(object):
                 types = set(chain.from_iterable(self.find_types(typ, next_name)
                                                 for typ in types))
             types = types
+        elif element.type == 'eval_input':
+            types = self._eval_element_not_cached(element.children[0])
         else:
             types = precedence.calculate_children(self, element.children)
         debug.dbg('eval_element result %s', types)

jedi/evaluate/compiled/fake.py

@@ -8,7 +8,7 @@ import os
 import inspect
 
 from jedi._compatibility import is_py3, builtins, unicode
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser import tree as pt
 from jedi.evaluate.helpers import FakeName
 
@@ -31,7 +31,7 @@ def _load_faked_module(module):
             modules[module_name] = None
             return
         grammar = load_grammar('grammar3.4')
-        module = Parser(grammar, unicode(source), module_name).module
+        module = ParserWithRecovery(grammar, unicode(source), module_name).module
         modules[module_name] = module
 
         if module_name == 'builtins' and not is_py3:
@@ -68,7 +68,11 @@ def get_module(obj):
             # Happens for example in `(_ for _ in []).send.__module__`.
             return builtins
         else:
-            return __import__(imp_plz)
+            try:
+                return __import__(imp_plz)
+            except ImportError:
+                # __module__ can be something arbitrary that doesn't exist.
+                return builtins
 
 
 def _faked(module, obj, name):

jedi/evaluate/docstrings.py

@@ -20,7 +20,7 @@ from itertools import chain
 from textwrap import dedent
 
 from jedi.evaluate.cache import memoize_default
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.common import indent_block
 from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated
 
@@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module):
     # Take the default grammar here, if we load the Python 2.7 grammar here, it
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
-    p = Parser(load_grammar(), code % indent_block(string))
+    p = ParserWithRecovery(load_grammar(), code % indent_block(string))
     try:
         pseudo_cls = p.module.subscopes[0]
         # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,

jedi/evaluate/finder.py

@@ -23,6 +23,7 @@ from jedi.evaluate import representation as er
 from jedi.evaluate import dynamic
 from jedi.evaluate import compiled
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import iterable
 from jedi.evaluate import imports
 from jedi.evaluate import analysis
@@ -386,10 +387,11 @@ def _eval_param(evaluator, param, scope):
             and func.instance.is_generated and str(func.name) == '__init__':
         param = func.var.params[param.position_nr]
 
-    # Add docstring knowledge.
+    # Add pep0484 and docstring knowledge.
+    pep0484_hints = pep0484.follow_param(evaluator, param)
     doc_params = docstrings.follow_param(evaluator, param)
-    if doc_params:
+    if pep0484_hints or doc_params:
-        return doc_params
+        return list(set(pep0484_hints) | set(doc_params))
 
     if isinstance(param, ExecutedParam):
         return res_new | param.eval(evaluator)
@@ -485,8 +487,8 @@ def global_names_dict_generator(evaluator, scope, position):
     the current scope is function:
 
     >>> from jedi._compatibility import u, no_unicode_pprint
-    >>> from jedi.parser import Parser, load_grammar
+    >>> from jedi.parser import ParserWithRecovery, load_grammar
-    >>> parser = Parser(load_grammar(), u('''
+    >>> parser = ParserWithRecovery(load_grammar(), u('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None

jedi/evaluate/pep0484.py

@@ -0,0 +1,59 @@
+"""
+PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
+through function annotations. There is a strong suggestion in this document
+that only the type of type hinting defined in PEP0484 should be allowed
+as annotations in future python versions.
+
+The (initial / probably incomplete) implementation todo list for pep-0484:
+v Function parameter annotations with builtin/custom type classes
+v Function returntype annotations with builtin/custom type classes
+v Function parameter annotations with strings (forward reference)
+v Function return type annotations with strings (forward reference)
+x Local variable type hints
+v Assigned types: `Url = str\ndef get(url:Url) -> str:`
+x Type hints in `with` statements
+x Stub files support
+x support `@no_type_check` and `@no_type_check_decorator`
+x support for type hint comments `# type: (int, str) -> int`. See comment from
+    Guido https://github.com/davidhalter/jedi/issues/662
+"""
+
+from itertools import chain
+
+from jedi.parser import Parser, load_grammar
+from jedi.evaluate.cache import memoize_default
+from jedi.evaluate.compiled import CompiledObject
+from jedi import debug
+
+
+def _evaluate_for_annotation(evaluator, annotation):
+    if annotation is not None:
+        definitions = set()
+        for definition in evaluator.eval_element(annotation):
+            if (isinstance(definition, CompiledObject) and
+                    isinstance(definition.obj, str)):
+                p = Parser(load_grammar(), definition.obj, start='eval_input')
+                element = p.get_parsed_node()
+                if element is None:
+                    debug.warning('Annotation not parsed: %s' % definition.obj)
+                else:
+                    element.parent = annotation.parent
+                    definitions |= evaluator.eval_element(element)
+            else:
+                definitions.add(definition)
+        return list(chain.from_iterable(
+            evaluator.execute(d) for d in definitions))
+    else:
+        return []
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def follow_param(evaluator, param):
+    annotation = param.annotation()
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def find_return_types(evaluator, func):
+    annotation = func.py__annotations__().get("return", None)
+    return _evaluate_for_annotation(evaluator, annotation)

jedi/evaluate/representation.py

@@ -49,6 +49,7 @@ from jedi.evaluate import compiled
 from jedi.evaluate import recursion
 from jedi.evaluate import iterable
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import helpers
 from jedi.evaluate import param
 from jedi.evaluate import flow_analysis
@@ -583,6 +584,20 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
         else:
             return FunctionExecution(self._evaluator, self, params).get_return_types()
 
+    @memoize_default()
+    def py__annotations__(self):
+        parser_func = self.base
+        return_annotation = parser_func.annotation()
+        if return_annotation:
+            dct = {'return': return_annotation}
+        else:
+            dct = {}
+        for function_param in parser_func.params:
+            param_annotation = function_param.annotation()
+            if param_annotation is not None:
+                dct[function_param.name.value] = param_annotation
+        return dct
+
     def py__class__(self):
         return compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS')
 
@@ -642,6 +657,7 @@ class FunctionExecution(Executed):
         else:
             returns = self.returns
             types = set(docstrings.find_return_types(self._evaluator, func))
+            types |= set(pep0484.find_return_types(self._evaluator, func))
 
         for r in returns:
             check = flow_analysis.break_check(self._evaluator, self, r)

jedi/evaluate/stdlib.py

@@ -14,7 +14,7 @@ from jedi.common import unite
 from jedi.evaluate import compiled
 from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
 from jedi import debug
 from jedi.evaluate import precedence
@@ -243,7 +243,7 @@ def collections_namedtuple(evaluator, obj, arguments):
     )
 
     # Parse source
-    generated_class = Parser(evaluator.grammar, unicode(source)).module.subscopes[0]
+    generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0]
     return set([er.Class(evaluator, generated_class)])
 
 

jedi/evaluate/sys_path.py

@@ -5,7 +5,7 @@ from jedi.evaluate.site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.evaluate.cache import memoize_default
 from jedi import debug
 from jedi import common
@@ -209,7 +209,7 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
             debug.dbg('Error trying to read buildout_script: %s', buildout_script)
             return
 
-        p = Parser(evaluator.grammar, source, buildout_script)
+        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
         cache.save_parser(buildout_script, p)
         return p.module
 

jedi/parser/__init__.py

@@ -81,92 +81,89 @@ class ParserSyntaxError(object):
 
 
 class Parser(object):
-    """
+    AST_MAPPING = {
-    This class is used to parse a Python file, it then divides them into a
+        'expr_stmt': pt.ExprStmt,
-    class structure of different scopes.
+        'classdef': pt.Class,
+        'funcdef': pt.Function,
+        'file_input': pt.Module,
+        'import_name': pt.ImportName,
+        'import_from': pt.ImportFrom,
+        'break_stmt': pt.KeywordStatement,
+        'continue_stmt': pt.KeywordStatement,
+        'return_stmt': pt.ReturnStmt,
+        'raise_stmt': pt.KeywordStatement,
+        'yield_expr': pt.YieldExpr,
+        'del_stmt': pt.KeywordStatement,
+        'pass_stmt': pt.KeywordStatement,
+        'global_stmt': pt.GlobalStmt,
+        'nonlocal_stmt': pt.KeywordStatement,
+        'print_stmt': pt.KeywordStatement,
+        'assert_stmt': pt.AssertStmt,
+        'if_stmt': pt.IfStmt,
+        'with_stmt': pt.WithStmt,
+        'for_stmt': pt.ForStmt,
+        'while_stmt': pt.WhileStmt,
+        'try_stmt': pt.TryStmt,
+        'comp_for': pt.CompFor,
+        'decorator': pt.Decorator,
+        'lambdef': pt.Lambda,
+        'old_lambdef': pt.Lambda,
+        'lambdef_nocond': pt.Lambda,
+    }
 
-    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    class ParserError(Exception):
-    :param source: The codebase for the parser. Must be unicode.
+        pass
-    :param module_path: The path of the module in the file system, may be None.
-    :type module_path: str
-    :param top_module: Use this module as a parent instead of `self.module`.
-    """
-    def __init__(self, grammar, source, module_path=None, tokenizer=None):
-        self._ast_mapping = {
-            'expr_stmt': pt.ExprStmt,
-            'classdef': pt.Class,
-            'funcdef': pt.Function,
-            'file_input': pt.Module,
-            'import_name': pt.ImportName,
-            'import_from': pt.ImportFrom,
-            'break_stmt': pt.KeywordStatement,
-            'continue_stmt': pt.KeywordStatement,
-            'return_stmt': pt.ReturnStmt,
-            'raise_stmt': pt.KeywordStatement,
-            'yield_expr': pt.YieldExpr,
-            'del_stmt': pt.KeywordStatement,
-            'pass_stmt': pt.KeywordStatement,
-            'global_stmt': pt.GlobalStmt,
-            'nonlocal_stmt': pt.KeywordStatement,
-            'print_stmt': pt.KeywordStatement,
-            'assert_stmt': pt.AssertStmt,
-            'if_stmt': pt.IfStmt,
-            'with_stmt': pt.WithStmt,
-            'for_stmt': pt.ForStmt,
-            'while_stmt': pt.WhileStmt,
-            'try_stmt': pt.TryStmt,
-            'comp_for': pt.CompFor,
-            'decorator': pt.Decorator,
-            'lambdef': pt.Lambda,
-            'old_lambdef': pt.Lambda,
-            'lambdef_nocond': pt.Lambda,
-        }
 
-        self.syntax_errors = []
+    def __init__(self, grammar, source, start, tokenizer=None):
+        start_number = grammar.symbol2number[start]
 
-        self._global_names = []
-        self._omit_dedent_list = []
-        self._indent_counter = 0
-        self._last_failed_start_pos = (0, 0)
-
-        # TODO do print absolute import detection here.
-        #try:
-        #    del python_grammar_no_print_statement.keywords["print"]
-        #except KeyError:
-        #    pass  # Doesn't exist in the Python 3 grammar.
-
-        #if self.options["print_function"]:
-        #    python_grammar = pygram.python_grammar_no_print_statement
-        #else:
         self._used_names = {}
         self._scope_names_stack = [{}]
         self._error_statement_stacks = []
-
+        self._last_failed_start_pos = (0, 0)
-        added_newline = False
+        self._global_names = []
-        # The Python grammar needs a newline at the end of each statement.
-        if not source.endswith('\n'):
-            source += '\n'
-            added_newline = True
 
         # For the fast parser.
         self.position_modifier = pt.PositionModifier()
-        p = PgenParser(grammar, self.convert_node, self.convert_leaf,
-                       self.error_recovery)
-        tokenizer = tokenizer or tokenize.source_tokens(source)
-        self.module = p.parse(self._tokenize(tokenizer))
-        if self.module.type != 'file_input':
-            # If there's only one statement, we get back a non-module. That's
-            # not what we want, we want a module, so we add it here:
-            self.module = self.convert_node(grammar,
-                                            grammar.symbol2number['file_input'],
-                                            [self.module])
 
-        if added_newline:
+        added_newline = False
-            self.remove_last_newline()
+        # The Python grammar needs a newline at the end of each statement.
-        self.module.used_names = self._used_names
+        if not source.endswith('\n') and start == 'file_input':
-        self.module.path = module_path
+            source += '\n'
-        self.module.global_names = self._global_names
+            added_newline = True
-        self.module.error_statement_stacks = self._error_statement_stacks
+
+        p = PgenParser(grammar, self.convert_node, self.convert_leaf,
+                       self.error_recovery, start_number)
+        if tokenizer is None:
+            tokenizer = tokenize.source_tokens(source)
+        try:
+            self._parsed = p.parse(self._tokenize(tokenizer))
+        except Parser.ParserError:
+            self._parsed = None
+        else:
+            if start == 'file_input' != self._parsed.type:
+                # If there's only one statement, we get back a non-module. That's
+                # not what we want, we want a module, so we add it here:
+                self._parsed = self.convert_node(grammar,
+                                                 grammar.symbol2number['file_input'],
+                                                 [self._parsed])
+
+            if added_newline:
+                self.remove_last_newline()
+
+    def get_parsed_node(self):
+        return self._parsed
+
+    def _tokenize(self, tokenizer):
+        for typ, value, start_pos, prefix in tokenizer:
+            if typ == ERRORTOKEN:
+                raise Parser.ParserError
+            elif typ == OP:
+                typ = token.opmap[value]
+            yield typ, value, prefix, start_pos
+
+    def error_recovery(self, *args, **kwargs):
+        raise Parser.ParserError
 
     def convert_node(self, grammar, type, children):
         """
@@ -178,7 +175,7 @@ class Parser(object):
         """
         symbol = grammar.number2symbol[type]
         try:
-            new_node = self._ast_mapping[symbol](children)
+            new_node = Parser.AST_MAPPING[symbol](children)
         except KeyError:
             new_node = pt.Node(symbol, children)
 
@@ -231,6 +228,83 @@ class Parser(object):
         else:
             return pt.Operator(self.position_modifier, value, start_pos, prefix)
 
+    def remove_last_newline(self):
+        """
+        In all of this we need to work with _start_pos, because if we worked
+        with start_pos, we would need to check the position_modifier as well
+        (which is accounted for in the start_pos property).
+        """
+        endmarker = self._parsed.children[-1]
+        # The newline is either in the endmarker as a prefix or the previous
+        # leaf as a newline token.
+        if endmarker.prefix.endswith('\n'):
+            endmarker.prefix = endmarker.prefix[:-1]
+            last_line = re.sub('.*\n', '', endmarker.prefix)
+            endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line)
+        else:
+            try:
+                newline = endmarker.get_previous()
+            except IndexError:
+                return  # This means that the parser is empty.
+            while True:
+                if newline.value == '':
+                    # Must be a DEDENT, just continue.
+                    try:
+                        newline = newline.get_previous()
+                    except IndexError:
+                        # If there's a statement that fails to be parsed, there
+                        # will be no previous leaf. So just ignore it.
+                        break
+                elif newline.value != '\n':
+                    # This may happen if error correction strikes and removes
+                    # a whole statement including '\n'.
+                    break
+                else:
+                    newline.value = ''
+                    if self._last_failed_start_pos > newline._start_pos:
+                        # It may be the case that there was a syntax error in a
+                        # function. In that case error correction removes the
+                        # right newline. So we use the previously assigned
+                        # _last_failed_start_pos variable to account for that.
+                        endmarker._start_pos = self._last_failed_start_pos
+                    else:
+                        endmarker._start_pos = newline._start_pos
+                    break
+
+
+class ParserWithRecovery(Parser):
+    """
+    This class is used to parse a Python file, it then divides them into a
+    class structure of different scopes.
+
+    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    :param source: The codebase for the parser. Must be unicode.
+    :param module_path: The path of the module in the file system, may be None.
+    :type module_path: str
+    """
+    def __init__(self, grammar, source, module_path=None, tokenizer=None):
+        self.syntax_errors = []
+
+        self._omit_dedent_list = []
+        self._indent_counter = 0
+
+        # TODO do print absolute import detection here.
+        #try:
+        #    del python_grammar_no_print_statement.keywords["print"]
+        #except KeyError:
+        #    pass  # Doesn't exist in the Python 3 grammar.
+
+        #if self.options["print_function"]:
+        #    python_grammar = pygram.python_grammar_no_print_statement
+        #else:
+        super(ParserWithRecovery, self).__init__(grammar, source, 'file_input', tokenizer)
+
+        self.module = self._parsed
+        self.module.used_names = self._used_names
+        self.module.path = module_path
+        self.module.global_names = self._global_names
+        self.module.error_statement_stacks = self._error_statement_stacks
+
     def error_recovery(self, grammar, stack, typ, value, start_pos, prefix,
                        add_token_callback):
         """
@@ -349,46 +423,3 @@ class Parser(object):
 
     def __repr__(self):
         return "<%s: %s>" % (type(self).__name__, self.module)
-
-    def remove_last_newline(self):
-        """
-        In all of this we need to work with _start_pos, because if we worked
-        with start_pos, we would need to check the position_modifier as well
-        (which is accounted for in the start_pos property).
-        """
-        endmarker = self.module.children[-1]
-        # The newline is either in the endmarker as a prefix or the previous
-        # leaf as a newline token.
-        if endmarker.prefix.endswith('\n'):
-            endmarker.prefix = endmarker.prefix[:-1]
-            last_line = re.sub('.*\n', '', endmarker.prefix)
-            endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line)
-        else:
-            try:
-                newline = endmarker.get_previous()
-            except IndexError:
-                return  # This means that the parser is empty.
-            while True:
-                if newline.value == '':
-                    # Must be a DEDENT, just continue.
-                    try:
-                        newline = newline.get_previous()
-                    except IndexError:
-                        # If there's a statement that fails to be parsed, there
-                        # will be no previous leaf. So just ignore it.
-                        break
-                elif newline.value != '\n':
-                    # This may happen if error correction strikes and removes
-                    # a whole statement including '\n'.
-                    break
-                else:
-                    newline.value = ''
-                    if self._last_failed_start_pos > newline._start_pos:
-                        # It may be the case that there was a syntax error in a
-                        # function. In that case error correction removes the
-                        # right newline. So we use the previously assigned
-                        # _last_failed_start_pos variable to account for that.
-                        endmarker._start_pos = self._last_failed_start_pos
-                    else:
-                        endmarker._start_pos = newline._start_pos
-                    break

jedi/parser/fast.py

@@ -8,7 +8,7 @@ from itertools import chain
 
 from jedi._compatibility import use_metaclass
 from jedi import settings
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
 from jedi import cache
 from jedi import debug
@@ -52,8 +52,9 @@ class FastModule(tree.Module):
         return "<fast.%s: %s@%s-%s>" % (type(self).__name__, self.name,
                                         self.start_pos[0], self.end_pos[0])
 
-    # To avoid issues with with the `parser.Parser`, we need setters that do
+    # To avoid issues with with the `parser.ParserWithRecovery`, we need
-    # nothing, because if pickle comes along and sets those values.
+    # setters that do nothing, because if pickle comes along and sets those
+    # values.
     @global_names.setter
     def global_names(self, value):
         pass
@@ -99,10 +100,10 @@ class CachedFastParser(type):
     """ This is a metaclass for caching `FastParser`. """
     def __call__(self, grammar, source, module_path=None):
         if not settings.fast_parser:
-            return Parser(grammar, source, module_path)
+            return ParserWithRecovery(grammar, source, module_path)
 
         pi = cache.parser_cache.get(module_path, None)
-        if pi is None or isinstance(pi.parser, Parser):
+        if pi is None or isinstance(pi.parser, ParserWithRecovery):
             p = super(CachedFastParser, self).__call__(grammar, source, module_path)
         else:
             p = pi.parser  # pi is a `cache.ParserCacheItem`
@@ -432,7 +433,7 @@ class FastParser(use_metaclass(CachedFastParser)):
         else:
             tokenizer = FastTokenizer(parser_code)
             self.number_parsers_used += 1
-            p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
+            p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
 
             end = line_offset + p.module.end_pos[0]
             used_lines = self._lines[line_offset:end - 1]

jedi/parser/pgen2/parse.py

@@ -60,7 +60,7 @@ class PgenParser(object):
 
     """
 
-    def __init__(self, grammar, convert_node, convert_leaf, error_recovery):
+    def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start):
         """Constructor.
 
         The grammar argument is a grammar.Grammar instance; see the
@@ -90,8 +90,6 @@ class PgenParser(object):
         self.convert_node = convert_node
         self.convert_leaf = convert_leaf
 
-        # Prepare for parsing.
-        start = self.grammar.start
         # Each stack entry is a tuple: (dfa, state, node).
         # A node is a tuple: (type, children),
         # where children is a list of nodes or None

jedi/parser/tokenize.py

@@ -149,7 +149,7 @@ ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except',
 
 def source_tokens(source):
     """Generate tokens from a the source code (string)."""
-    source = source + '\n'  # end with \n, because the parser needs it
+    source = source
     readline = StringIO(source).readline
     return generate_tokens(readline)
 
@@ -165,6 +165,7 @@ def generate_tokens(readline):
     paren_level = 0  # count parentheses
     indents = [0]
     lnum = 0
+    max = 0
     numchars = '0123456789'
     contstr = ''
     contline = None
@@ -282,9 +283,12 @@ def generate_tokens(readline):
                     paren_level -= 1
                 yield OP, token, spos, prefix
 
-    end_pos = (lnum, max - 1)
+    if new_line:
+        end_pos = lnum + 1, 0
+    else:
+        end_pos = lnum, max
     # As the last position we just take the maximally possible position. We
     # remove -1 for the last new line.
     for indent in indents[1:]:
         yield DEDENT, '', end_pos, ''
-    yield ENDMARKER, '', end_pos, prefix
+    yield ENDMARKER, '', end_pos, additional_prefix

jedi/parser/tree.py

@@ -14,8 +14,8 @@ The easiest way to play with this module is to use :class:`parsing.Parser`.
 :attr:`parsing.Parser.module` holds an instance of :class:`Module`:
 
 >>> from jedi._compatibility import u
->>> from jedi.parser import Parser, load_grammar
+>>> from jedi.parser import ParserWithRecovery, load_grammar
->>> parser = Parser(load_grammar(), u('import os'), 'example.py')
+>>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py')
 >>> submodule = parser.module
 >>> submodule
 <Module: example.py@1-1>
@@ -873,7 +873,10 @@ class Function(ClassOrFunc):
 
     def annotation(self):
         try:
-            return self.children[6]  # 6th element: def foo(...) -> bar
+            if self.children[3] == "->":
+                return self.children[4]
+            assert self.children[3] == ":"
+            return None
         except IndexError:
             return None
 
@@ -952,6 +955,10 @@ class Lambda(Function):
     def is_generator(self):
         return False
 
+    def annotation(self):
+        # lambda functions do not support annotations
+        return None
+
     @property
     def yields(self):
         return []
@@ -1404,8 +1411,14 @@ class Param(BaseNode):
             return None
 
     def annotation(self):
-        # Generate from tfpdef.
+        tfpdef = self._tfpdef()
-        raise NotImplementedError
+        if is_node(tfpdef, 'tfpdef'):
+            assert tfpdef.children[1] == ":"
+            assert len(tfpdef.children) == 3
+            annotation = tfpdef.children[2]
+            return annotation
+        else:
+            return None
 
     def _tfpdef(self):
         """

jedi/parser/user_context.py

@@ -4,7 +4,7 @@ import keyword
 
 from jedi import cache
 from jedi import common
-from jedi.parser import tokenize, Parser
+from jedi.parser import tokenize, ParserWithRecovery
 from jedi._compatibility import u
 from jedi.parser.fast import FastParser
 from jedi.parser import tree
@@ -284,7 +284,7 @@ class UserContextParser(object):
             # Don't pickle that module, because the main module is changing quickly
             cache.save_parser(self._path, parser, pickling=False)
         else:
-            parser = Parser(self._grammar, self._source, self._path)
+            parser = ParserWithRecovery(self._grammar, self._source, self._path)
         self._parser_done_callback(parser)
         return parser
 

test/completion/pep0484.py

@@ -0,0 +1,161 @@
+""" Pep-0484 type hinting """
+
+# python >= 3.2
+
+
+class A():
+    pass
+
+
+def function_parameters(a: A, b, c: str, d: int, e: str, f: str, g: int=4):
+    """
+    :param e: if docstring and annotation agree, only one should be returned
+    :type e: str
+    :param f: if docstring and annotation disagree, both should be returned
+    :type f: int
+    """
+    #? A()
+    a
+    #?
+    b
+    #? str()
+    c
+    #? int()
+    d
+    #? str()
+    e
+    #? int() str()
+    f
+    # int()
+    g
+
+
+def return_unspecified():
+    pass
+
+#?
+return_unspecified()
+
+
+def return_none() -> None:
+    """
+    Return type None means the same as no return type as far as jedi
+    is concerned
+    """
+    pass
+
+#?
+return_none()
+
+
+def return_str() -> str:
+    pass
+
+#? str()
+return_str()
+
+
+def return_custom_class() -> A:
+    pass
+
+#? A()
+return_custom_class()
+
+
+def return_annotation_and_docstring() -> str:
+    """
+    :rtype: int
+    """
+    pass
+
+#? str() int()
+return_annotation_and_docstring()
+
+
+def return_annotation_and_docstring_different() -> str:
+    """
+    :rtype: str
+    """
+    pass
+
+#? str()
+return_annotation_and_docstring_different()
+
+
+def annotation_forward_reference(b: "B") -> "B":
+    #? B()
+    b
+
+#? B()
+annotation_forward_reference(1)
+#? ["test_element"]
+annotation_forward_reference(1).t
+
+class B:
+    test_element = 1
+    pass
+
+
+class SelfReference:
+    test_element = 1
+    def test_method(self, x: "SelfReference") -> "SelfReference":
+        #? SelfReference()
+        x
+        #? ["test_element", "test_method"]
+        self.t
+        #? ["test_element", "test_method"]
+        x.t
+        #? ["test_element", "test_method"]
+        self.test_method(1).t
+
+#? SelfReference()
+SelfReference().test_method()
+
+def function_with_non_pep_0484_annotation(
+        x: "I can put anything here",
+        xx: "",
+        yy: "\r\n\0;+*&^564835(---^&*34",
+        y: 3 + 3,
+        zz: float) -> int("42"):
+    # infers int from function call
+    #? int()
+    x
+    # infers int from function call
+    #? int()
+    xx
+    # infers int from function call
+    #? int()
+    yy
+    # infers str from function call
+    #? str()
+    y
+    #? float()
+    zz
+#?
+function_with_non_pep_0484_annotation(1, 2, 3, "force string")
+
+def function_forward_reference_dynamic(
+        x: return_str_type(),
+        y: "return_str_type()") -> None:
+    # technically should not be resolvable since out of scope,
+    # but jedi is not smart enough for that
+    #? str()
+    x
+    #? str()
+    y
+
+def return_str_type():
+    return str
+
+
+X = str
+def function_with_assined_class_in_reference(x: X, y: "Y"):
+    #? str()
+    x
+    #? int()
+    y
+Y = int
+
+def just_because_we_can(x: "flo" + "at"):
+    #? float()
+    x

test/run.py

@@ -111,6 +111,7 @@ Tests look like this::
 """
 import os
 import re
+import sys
 from ast import literal_eval
 from io import StringIO
 from functools import reduce
@@ -127,7 +128,7 @@ TEST_USAGES = 3
 
 class IntegrationTestCase(object):
     def __init__(self, test_type, correct, line_nr, column, start, line,
-                 path=None):
+                 path=None, skip=None):
         self.test_type = test_type
         self.correct = correct
         self.line_nr = line_nr
@@ -135,7 +136,7 @@ class IntegrationTestCase(object):
         self.start = start
         self.line = line
         self.path = path
-        self.skip = None
+        self.skip = skip
 
     @property
     def module_name(self):
@@ -234,10 +235,11 @@ class IntegrationTestCase(object):
 
 def collect_file_tests(lines, lines_to_execute):
     makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column,
-                                             start, line)
+                                             start, line, path=None, skip=skip)
     start = None
     correct = None
     test_type = None
+    skip = None
     for line_nr, line in enumerate(lines, 1):
         if correct is not None:
             r = re.match('^(\d+)\s*(.*)$', correct)
@@ -257,6 +259,15 @@ def collect_file_tests(lines, lines_to_execute):
                 yield makecase(TEST_DEFINITIONS)
             correct = None
         else:
+            # check for python minimal version number
+            match = re.match(r" *# *python *>= *(\d+(?:\.\d+)?)$", line)
+            if match:
+                minimal_python_version = tuple(
+                    map(int, match.group(1).split(".")))
+                if sys.version_info >= minimal_python_version:
+                    skip = None
+                else:
+                    skip = "Minimal python version %s" % match.groups(1)
             try:
                 r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]*)', line)
                 # test_type is ? for completion and ! for goto_assignments

test/test_evaluate/test_absolute_import.py

@@ -4,7 +4,7 @@ Python 2.X)
 """
 import jedi
 from jedi._compatibility import u
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from .. import helpers
 
 
@@ -12,7 +12,7 @@ def test_explicit_absolute_imports():
     """
     Detect modules with ``from __future__ import absolute_import``.
     """
-    parser = Parser(load_grammar(), u("from __future__ import absolute_import"), "test.py")
+    parser = ParserWithRecovery(load_grammar(), u("from __future__ import absolute_import"), "test.py")
     assert parser.module.has_explicit_absolute_import
 
 
@@ -20,7 +20,7 @@ def test_no_explicit_absolute_imports():
     """
      Detect modules without ``from __future__ import absolute_import``.
     """
-    parser = Parser(load_grammar(), u("1"), "test.py")
+    parser = ParserWithRecovery(load_grammar(), u("1"), "test.py")
     assert not parser.module.has_explicit_absolute_import
 
 
@@ -30,7 +30,7 @@ def test_dont_break_imports_without_namespaces():
     assume that all imports have non-``None`` namespaces.
     """
     src = u("from __future__ import absolute_import\nimport xyzzy")
-    parser = Parser(load_grammar(), src, "test.py")
+    parser = ParserWithRecovery(load_grammar(), src, "test.py")
     assert parser.module.has_explicit_absolute_import
 
 

test/test_evaluate/test_annotations.py

@@ -8,8 +8,8 @@ import pytest
 def test_simple_annotations():
     """
     Annotations only exist in Python 3.
-    At the moment we ignore them. So they should be parsed and not interfere
+    If annotations adhere to PEP-0484, we use them (they override inference),
-    with anything.
+    else they are parsed but ignored
     """
 
     source = dedent("""\
@@ -27,3 +27,34 @@ def test_simple_annotations():
 
     annot_ret('')""")
     assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['str']
+
+    source = dedent("""\
+    def annot(a:int):
+        return a
+
+    annot('')""")
+
+    assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['int']
+
+
+@pytest.mark.skipif('sys.version_info[0] < 3')
+@pytest.mark.parametrize('reference', [
+    'assert 1',
+    '1',
+    'def x(): pass',
+    '1, 2',
+    r'1\n'
+])
+def test_illegal_forward_references(reference):
+    source = 'def foo(bar: "%s"): bar' % reference
+
+    assert not jedi.Script(source).goto_definitions()
+
+
+@pytest.mark.skipif('sys.version_info[0] < 3')
+def test_lambda_forward_references():
+    source = 'def foo(bar: "lambda: 3"): bar'
+
+    # For now just receiving the 3 is ok. I'm doubting that this is what we
+    # want. We also execute functions. Should we only execute classes?
+    assert jedi.Script(source).goto_definitions()

test/test_evaluate/test_buildout_detection.py

@@ -7,7 +7,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file,
                                     sys_path_with_modifications,
                                     _check_module)
 from jedi.evaluate import Evaluator
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 from ..helpers import cwd_at
 
@@ -37,7 +37,7 @@ def test_append_on_non_sys_path():
         d = Dummy()
         d.path.append('foo')"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert len(paths) > 0
     assert 'foo' not in paths
@@ -48,7 +48,7 @@ def test_path_from_invalid_sys_path_assignment():
         import sys
         sys.path = 'invalid'"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert len(paths) > 0
     assert 'invalid' not in paths
@@ -60,7 +60,7 @@ def test_sys_path_with_modifications():
         import os
     """))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
     paths = sys_path_with_modifications(Evaluator(grammar), p.module)
     assert '/tmp/.buildout/eggs/important_package.egg' in paths
@@ -83,7 +83,7 @@ def test_path_from_sys_path_assignment():
         if __name__ == '__main__':
             sys.exit(important_package.main())"""))
     grammar = load_grammar()
-    p = Parser(grammar, SRC)
+    p = ParserWithRecovery(grammar, SRC)
     paths = _check_module(Evaluator(grammar), p.module)
     assert 1 not in paths
     assert '/home/test/.buildout/eggs/important_package.egg' in paths

test/test_evaluate/test_sys_path.py

@@ -5,14 +5,14 @@ import sys
 import pytest
 
 from jedi._compatibility import unicode
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.evaluate import sys_path, Evaluator
 
 
 def test_paths_from_assignment():
     def paths(src):
         grammar = load_grammar()
-        stmt = Parser(grammar, unicode(src)).module.statements[0]
+        stmt = ParserWithRecovery(grammar, unicode(src)).module.statements[0]
         return set(sys_path._paths_from_assignment(Evaluator(grammar), stmt))
 
     assert paths('sys.path[0:0] = ["a"]') == set(['a'])

test/test_new_parser.py

@@ -1,11 +1,11 @@
 from jedi._compatibility import u
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 
 def test_basic_parsing():
     def compare(string):
         """Generates the AST object and then regenerates the code."""
-        assert Parser(load_grammar(), string).module.get_code() == string
+        assert ParserWithRecovery(load_grammar(), string).module.get_code() == string
 
     compare(u('\na #pass\n'))
     compare(u('wblabla* 1\t\n'))

test/test_parser/test_get_code.py

@@ -3,7 +3,7 @@ import difflib
 import pytest
 
 from jedi._compatibility import u
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 code_basic_features = u('''
 """A mod docstring"""
@@ -44,7 +44,7 @@ def diff_code_assert(a, b, n=4):
 def test_basic_parsing():
     """Validate the parsing features"""
 
-    prs = Parser(load_grammar(), code_basic_features)
+    prs = ParserWithRecovery(load_grammar(), code_basic_features)
     diff_code_assert(
         code_basic_features,
         prs.module.get_code()
@@ -53,7 +53,7 @@ def test_basic_parsing():
 
 def test_operators():
     src = u('5  * 3')
-    prs = Parser(load_grammar(), src)
+    prs = ParserWithRecovery(load_grammar(), src)
     diff_code_assert(src, prs.module.get_code())
 
 
@@ -82,7 +82,7 @@ def method_with_docstring():
     """class docstr"""
     pass
 ''')
-    assert Parser(load_grammar(), s).module.get_code() == s
+    assert ParserWithRecovery(load_grammar(), s).module.get_code() == s
 
 
 def test_end_newlines():
@@ -92,7 +92,7 @@ def test_end_newlines():
     line the parser needs.
     """
     def test(source, end_pos):
-        module = Parser(load_grammar(), u(source)).module
+        module = ParserWithRecovery(load_grammar(), u(source)).module
         assert module.get_code() == source
         assert module.end_pos == end_pos
 

test/test_parser/test_parser.py

@@ -3,7 +3,7 @@ import sys
 
 import jedi
 from jedi._compatibility import u, is_py3
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser.user_context import UserContextParser
 from jedi.parser import tree as pt
 from textwrap import dedent
@@ -23,7 +23,7 @@ def test_user_statement_on_import():
 class TestCallAndName():
     def get_call(self, source):
         # Get the simple_stmt and then the first one.
-        simple_stmt = Parser(load_grammar(), u(source)).module.children[0]
+        simple_stmt = ParserWithRecovery(load_grammar(), u(source)).module.children[0]
         return simple_stmt.children[0]
 
     def test_name_and_call_positions(self):
@@ -58,7 +58,7 @@ class TestCallAndName():
 
 class TestSubscopes():
     def get_sub(self, source):
-        return Parser(load_grammar(), u(source)).module.subscopes[0]
+        return ParserWithRecovery(load_grammar(), u(source)).module.subscopes[0]
 
     def test_subscope_names(self):
         name = self.get_sub('class Foo: pass').name
@@ -74,7 +74,7 @@ class TestSubscopes():
 
 class TestImports():
     def get_import(self, source):
-        return Parser(load_grammar(), source).module.imports[0]
+        return ParserWithRecovery(load_grammar(), source).module.imports[0]
 
     def test_import_names(self):
         imp = self.get_import(u('import math\n'))
@@ -89,13 +89,13 @@ class TestImports():
 
 
 def test_module():
-    module = Parser(load_grammar(), u('asdf'), 'example.py').module
+    module = ParserWithRecovery(load_grammar(), u('asdf'), 'example.py').module
     name = module.name
     assert str(name) == 'example'
     assert name.start_pos == (1, 0)
     assert name.end_pos == (1, 7)
 
-    module = Parser(load_grammar(), u('asdf')).module
+    module = ParserWithRecovery(load_grammar(), u('asdf')).module
     name = module.name
     assert str(name) == ''
     assert name.start_pos == (1, 0)
@@ -108,7 +108,7 @@ def test_end_pos():
                  def func():
                      y = None
                  '''))
-    parser = Parser(load_grammar(), s)
+    parser = ParserWithRecovery(load_grammar(), s)
     scope = parser.module.subscopes[0]
     assert scope.start_pos == (3, 0)
     assert scope.end_pos == (5, 0)
@@ -121,7 +121,7 @@ def test_carriage_return_statements():
         # this is a namespace package
     '''))
     source = source.replace('\n', '\r\n')
-    stmt = Parser(load_grammar(), source).module.statements[0]
+    stmt = ParserWithRecovery(load_grammar(), source).module.statements[0]
     assert '#' not in stmt.get_code()
 
 
@@ -129,7 +129,7 @@ def test_incomplete_list_comprehension():
     """ Shouldn't raise an error, same bug as #418. """
     # With the old parser this actually returned a statement. With the new
     # parser only valid statements generate one.
-    assert Parser(load_grammar(), u('(1 for def')).module.statements == []
+    assert ParserWithRecovery(load_grammar(), u('(1 for def')).module.statements == []
 
 
 def test_hex_values_in_docstring():
@@ -141,7 +141,7 @@ def test_hex_values_in_docstring():
             return 1
         '''
 
-    doc = Parser(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc
+    doc = ParserWithRecovery(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc
     if is_py3:
         assert doc == '\xff'
     else:
@@ -160,7 +160,7 @@ def test_error_correction_with():
 
 
 def test_newline_positions():
-    endmarker = Parser(load_grammar(), u('a\n')).module.children[-1]
+    endmarker = ParserWithRecovery(load_grammar(), u('a\n')).module.children[-1]
     assert endmarker.end_pos == (2, 0)
     new_line = endmarker.get_previous()
     assert new_line.start_pos == (1, 1)
@@ -174,7 +174,7 @@ def test_end_pos_error_correction():
     end_pos, even if something breaks in the parser (error correction).
     """
     s = u('def x():\n .')
-    m = Parser(load_grammar(), s).module
+    m = ParserWithRecovery(load_grammar(), s).module
     func = m.children[0]
     assert func.type == 'funcdef'
     # This is not exactly correct, but ok, because it doesn't make a difference
@@ -191,7 +191,7 @@ def test_param_splitting():
     def check(src, result):
         # Python 2 tuple params should be ignored for now.
         grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
-        m = Parser(grammar, u(src)).module
+        m = ParserWithRecovery(grammar, u(src)).module
         if is_py3:
             assert not m.subscopes
         else:
@@ -211,5 +211,5 @@ def test_unicode_string():
 
 def test_backslash_dos_style():
     grammar = load_grammar()
-    m = Parser(grammar, u('\\\r\n')).module
+    m = ParserWithRecovery(grammar, u('\\\r\n')).module
     assert m

test/test_parser/test_parser_tree.py

@@ -5,17 +5,18 @@ from textwrap import dedent
 import pytest
 
 from jedi._compatibility import u, unicode
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser import tree as pt
 
 
 class TestsFunctionAndLambdaParsing(object):
 
     FIXTURES = [
-        ('def my_function(x, y, z):\n    return x + y * z\n', {
+        ('def my_function(x, y, z) -> str:\n    return x + y * z\n', {
             'name': 'my_function',
             'call_sig': 'my_function(x, y, z)',
             'params': ['x', 'y', 'z'],
+            'annotation': "str",
         }),
         ('lambda x, y, z: x + y * z\n', {
             'name': '<lambda>',
@@ -26,7 +27,7 @@ class TestsFunctionAndLambdaParsing(object):
 
     @pytest.fixture(params=FIXTURES)
     def node(self, request):
-        parsed = Parser(load_grammar(), dedent(u(request.param[0])))
+        parsed = ParserWithRecovery(load_grammar(), dedent(u(request.param[0])))
         request.keywords['expected'] = request.param[1]
         return parsed.module.subscopes[0]
 
@@ -55,7 +56,11 @@ class TestsFunctionAndLambdaParsing(object):
             assert not node.yields
 
     def test_annotation(self, node, expected):
-        assert node.annotation() is expected.get('annotation', None)
+        expected_annotation = expected.get('annotation', None)
+        if expected_annotation is None:
+            assert node.annotation() is None
+        else:
+            assert node.annotation().value == expected_annotation
 
     def test_get_call_signature(self, node, expected):
         assert node.get_call_signature() == expected['call_sig']

test/test_parser/test_tokenize.py

@@ -7,7 +7,7 @@ import pytest
 
 from jedi._compatibility import u, is_py3
 from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT
-from jedi.parser import Parser, load_grammar, tokenize
+from jedi.parser import ParserWithRecovery, load_grammar, tokenize
 
 
 from ..helpers import unittest
@@ -15,7 +15,7 @@ from ..helpers import unittest
 
 class TokenTest(unittest.TestCase):
     def test_end_pos_one_line(self):
-        parsed = Parser(load_grammar(), dedent(u('''
+        parsed = ParserWithRecovery(load_grammar(), dedent(u('''
         def testit():
             a = "huhu"
         ''')))
@@ -23,7 +23,7 @@ class TokenTest(unittest.TestCase):
         assert tok.end_pos == (3, 14)
 
     def test_end_pos_multi_line(self):
-        parsed = Parser(load_grammar(), dedent(u('''
+        parsed = ParserWithRecovery(load_grammar(), dedent(u('''
         def testit():
             a = """huhu
         asdfasdf""" + "h"
@@ -108,7 +108,7 @@ class TokenTest(unittest.TestCase):
         ]
 
         for s in string_tokens:
-            parsed = Parser(load_grammar(), u('''a = %s\n''' % s))
+            parsed = ParserWithRecovery(load_grammar(), u('''a = %s\n''' % s))
             simple_stmt = parsed.module.children[0]
             expr_stmt = simple_stmt.children[0]
             assert len(expr_stmt.children) == 3

test/test_regression.py

@@ -15,7 +15,7 @@ from jedi._compatibility import u
 from jedi import Script
 from jedi import api
 from jedi.evaluate import imports
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 
 #jedi.set_debug_function()
 
@@ -102,7 +102,7 @@ class TestRegression(TestCase):
     def test_end_pos_line(self):
         # jedi issue #150
         s = u("x()\nx( )\nx(  )\nx (  )")
-        parser = Parser(load_grammar(), s)
+        parser = ParserWithRecovery(load_grammar(), s)
         for i, s in enumerate(parser.module.statements):
             assert s.end_pos == (i + 1, i + 3)