Merge remote-tracking branch 'upstream/dev' into bugfix/performances_degradation

# Conflicts:
#	jedi/evaluate/compiled/fake.py

.gitignore

@@ -1,6 +1,5 @@
 *~
-*.swp
-*.swo
+*.sw?
 *.pyc
 .ropeproject
 .tox

.travis.yml

@@ -1,11 +1,12 @@
 language: python
+python: 3.5
 sudo: false
 env:
     - TOXENV=py26
     - TOXENV=py27
-    - TOXENV=py32
     - TOXENV=py33
     - TOXENV=py34
+    - TOXENV=py35
     - TOXENV=pypy
     - TOXENV=cov
     - TOXENV=sith
@@ -14,8 +15,9 @@ matrix:
         - env: TOXENV=cov
         - env: TOXENV=sith
         - env: TOXENV=pypy
+python: 3.5
 install:
-    - pip install --quiet --use-mirrors tox
+    - pip install --quiet tox 'virtualenv<14.0.0' 'pip<8.0.0'
 script:
     - tox
 after_script:
@@ -23,3 +25,4 @@ after_script:
       pip install --quiet --use-mirrors coveralls;
       coveralls;
       fi
+

AUTHORS.txt

@@ -34,5 +34,9 @@ Phillip Berndt (@phillipberndt) <phillip.berndt@gmail.com>
 Ian Lee (@IanLee1521) <IanLee1521@gmail.com>
 Farkhad Khatamov (@hatamov) <comsgn@gmail.com>
 Kevin Kelley (@kelleyk) <kelleyk@kelleyk.net>
+Sid Shanker (@squidarth) <sid.p.shanker@gmail.com>
+Reinoud Elhorst (@reinhrst)
+Guido van Rossum (@gvanrossum) <guido@python.org>
+Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,6 +3,14 @@
 Changelog
 ---------
 
+0.10.0 (2016-06-)
++++++++++++++++++
+
+- Actual semantic completions for the complete Python syntax.
+- Basic type inference for ``yield from`` PEP 380.
+- PEP 484 support (most of the important features of it). Thanks Claude! (@reinhrst)
+- Again a lot of internal changes.
+
 0.9.0 (2015-04-10)
 ++++++++++++++++++
 

LICENSE.txt

@@ -1,3 +1,15 @@
+All contributions towards Jedi are MIT licensed.
+
+Some Python files have been taken from the standard library and are therefore
+PSF licensed. Modifications on these files are dual licensed (both MIT and
+PSF). These files are:
+
+- jedi/parser/pgen2
+- jedi/parser/tokenize.py
+- jedi/parser/token.py
+- test/test_parser/test_pgen2.py
+
+-------------------------------------------------------------------------------
 The MIT License (MIT)
 
 Copyright (c) <2013> <David Halter and others, see AUTHORS.txt>
@@ -19,3 +31,52 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+-------------------------------------------------------------------------------
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved"
+are retained in Python alone or in any derivative version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

README.rst

@@ -30,19 +30,23 @@ implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
 which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
 It's really easy.
 
-Jedi can currently be used with the following editors:
+Jedi can currently be used with the following editors/projects:
 
-- Vim (jedi-vim_, YouCompleteMe_)
-- Emacs (Jedi.el_, elpy_, anaconda-mode_, ycmd_)
+- Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_)
+- Emacs (Jedi.el_, company-mode_, elpy_, anaconda-mode_, ycmd_)
 - Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3])
-- SynWrite_
 - TextMate_ (Not sure if it's actually working)
 - Kate_ version 4.13+ supports it natively, you have to enable it, though. [`proof
   <https://projects.kde.org/projects/kde/applications/kate/repository/show?rev=KDE%2F4.13>`_]
-
-And it powers the following projects:
-
+- Atom_ (autocomplete-python_)
+- SourceLair_
+- `GNOME Builder`_ (with support for GObject Introspection)
+- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=donjayamanne.python>`_)
+- Gedit (gedi_)
 - wdb_ - Web Debugger
+- `Eric IDE`_ (Available as a plugin)
+
+and many more!
 
 
 Here are some pictures taken from jedi-vim_:
@@ -91,11 +95,11 @@ understands, see: `Features
 <https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
 caveats can be found on the same page.
 
-You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3 or 3.4, but it should also
+You can run Jedi on cPython 2.6, 2.7, 3.3, 3.4 or 3.5 but it should also
 understand/parse code older than those versions.
 
 Tips on how to use Jedi efficiently can be found `here
-<https://jedi.readthedocs.org/en/latest/docs/recipes.html>`_.
+<https://jedi.readthedocs.org/en/latest/docs/features.html#recipes>`_.
 
 API
 ---
@@ -173,15 +177,34 @@ For more detailed information visit the `testing documentation
 <https://jedi.readthedocs.org/en/latest/docs/testing.html>`_
 
 
+Acknowledgements
+================
+
+- Takafumi Arakaki (@tkf) for creating a solid test environment and a lot of
+  other things.
+- Danilo Bargen (@dbrgn) for general housekeeping and being a good friend :).
+- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2
+  (originally used in lib2to3).
+
+
+
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
 .. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
+.. _company-mode: https://github.com/syohex/emacs-company-jedi
 .. _elpy: https://github.com/jorgenschaefer/elpy
 .. _anaconda-mode: https://github.com/proofit404/anaconda-mode
 .. _ycmd: https://github.com/abingham/emacs-ycmd
 .. _sublimejedi: https://github.com/srusskih/SublimeJEDI
 .. _anaconda: https://github.com/DamnWidget/anaconda
-.. _SynWrite: http://uvviewsoft.com/synjedi/
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
 .. _Kate: http://kate-editor.org
+.. _Atom: https://atom.io/
+.. _autocomplete-python: https://atom.io/packages/autocomplete-python
+.. _SourceLair: https://www.sourcelair.com
+.. _GNOME Builder: https://wiki.gnome.org/Apps/Builder
+.. _Visual Studio Code: https://code.visualstudio.com/
+.. _gedi: https://github.com/isamert/gedi
+.. _Eric IDE: http://eric-ide.python-projects.org

docs/docs/features.rst

@@ -20,12 +20,12 @@ make it work.
 General Features
 ----------------
 
-- python 2.6+ and 3.2+ support
+- python 2.6+ and 3.3+ support
 - ignores syntax errors and wrong indentation
 - can deal with complex module / function / class structures
 - virtualenv support
-- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings
-  (:ref:`type hinting <type-hinting>`)
+- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
+  and PEP0484-style type hints (:ref:`type hinting <type-hinting>`)
 
 
 Supported Python Features
@@ -34,8 +34,8 @@ Supported Python Features
 |jedi| supports many of the widely used Python features:
 
 - builtins
-- multiple returns or yields
-- tuple assignments / array indexing / dictionary indexing
+- returns, yields, yield from
+- tuple assignments / array indexing / dictionary indexing / star unpacking
 - with-statement / exception handling
 - ``*args`` / ``**kwargs``
 - decorators / lambdas / closures
@@ -64,6 +64,7 @@ Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
+- implicit namespace packages (Python 3.3+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 
@@ -125,7 +126,49 @@ Type Hinting
 
 If |jedi| cannot detect the type of a function argument correctly (due to the
 dynamic nature of Python), you can help it by hinting the type using
-one of the following docstring syntax styles:
+one of the following docstring/annotation syntax styles:
+
+**PEP-0484 style**
+
+https://www.python.org/dev/peps/pep-0484/
+
+function annotations (python 3 only; python 2 function annotations with
+comments in planned but not yet implemented)
+
+::
+
+    def myfunction(node: ProgramNode, foo: str) -> None:
+        """Do something with a ``node``.
+
+        """
+        node.| # complete here
+
+
+assignment, for-loop and with-statement type hints (all python versions).
+Note that the type hints must be on the same line as the statement
+
+::
+
+    x = foo()  # type: int
+    x, y = 2, 3  # type: typing.Optional[int], typing.Union[int, str] # typing module is mostly supported
+    for key, value in foo.items():  # type: str, Employee  # note that Employee must be in scope
+        pass
+    with foo() as f:  # type: int
+        print(f + 3)
+
+Most of the features in PEP-0484 are supported including the typing module
+(for python < 3.5 you have to do ``pip install typing`` to use these),
+and forward references.
+
+Things that are missing (and this is not an exhaustive list; some of these
+are planned, others might be hard to implement and provide little worth):
+
+- annotating functions with comments: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code
+- understanding ``typing.cast()``
+- stub files: https://www.python.org/dev/peps/pep-0484/#stub-files
+- ``typing.Callable``
+- ``typing.TypeVar``
+- User defined generic types: https://www.python.org/dev/peps/pep-0484/#user-defined-generic-types
 
 **Sphinx style**
 

docs/docs/plugin-api.rst

@@ -47,14 +47,14 @@ Completions:
    >>> script = jedi.Script(source, 1, 19, '')
    >>> script
    <jedi.api.Script object at 0x2121b10>
-   >>> completions = script.complete()
+   >>> completions = script.completions()
    >>> completions
    [<Completion: load>, <Completion: loads>]
    >>> completions[1]
    <Completion: loads>
    >>> completions[1].complete
    'oads'
-   >>> completions[1].word
+   >>> completions[1].name
    'loads'
 
 Definitions / Goto:

docs/docs/static_analsysis.rst

@@ -0,0 +1,106 @@
+
+This file is the start of the documentation of how static analysis works.
+
+Below is a list of parser names that are used within nodes_to_execute.
+
+------------ cared for:
+global_stmt
+exec_stmt  # no priority
+assert_stmt
+if_stmt
+while_stmt
+for_stmt
+try_stmt
+(except_clause)
+with_stmt
+(with_item)
+(with_var)
+print_stmt
+del_stmt
+return_stmt
+raise_stmt
+yield_expr
+file_input
+funcdef
+param
+old_lambdef
+lambdef
+import_name
+import_from
+(import_as_name)
+(dotted_as_name)
+(import_as_names)
+(dotted_as_names)
+(dotted_name)
+classdef
+comp_for
+(comp_if) ?
+decorator
+
+----------- add basic
+test
+or_test
+and_test
+not_test
+expr
+xor_expr
+and_expr
+shift_expr
+arith_expr
+term
+factor
+power
+atom
+comparison
+expr_stmt
+testlist
+testlist1
+testlist_safe
+
+----------- special care:
+# mostly depends on how we handle the other ones.
+testlist_star_expr  # should probably just work with expr_stmt
+star_expr
+exprlist  # just ignore? then names are just resolved. Strange anyway, bc expr is not really allowed in the list, typically.
+
+----------- ignore:
+suite
+subscriptlist
+subscript
+simple_stmt
+?? sliceop     # can probably just be added.
+testlist_comp  # prob ignore and care about it with atom.
+dictorsetmaker
+trailer
+decorators
+decorated
+# always execute function arguments? -> no problem with stars.
+# Also arglist and argument are different in different grammars.
+arglist
+argument
+
+
+----------- remove:
+tname      # only exists in current Jedi parser. REMOVE!
+tfpdef     # python 2: tuple assignment; python 3: annotation
+vfpdef     # reduced in python 3 and therefore not existing.
+tfplist    # not in 3
+vfplist    # not in 3
+
+--------- not existing with parser reductions.
+small_stmt
+import_stmt
+flow_stmt
+compound_stmt
+stmt
+pass_stmt
+break_stmt
+continue_stmt
+comp_op
+augassign
+old_test
+typedargslist  # afaik becomes [param]
+varargslist    # dito
+vname
+comp_iter
+test_nocond

docs/docs/usage.rst

@@ -21,6 +21,7 @@ Vim:
 
 - jedi-vim_
 - YouCompleteMe_
+- deoplete-jedi_
 
 Emacs:
 
@@ -47,14 +48,33 @@ Kate:
   <https://projects.kde.org/projects/kde/applications/kate/repository/entry/addons/kate/pate/src/plugins/python_autocomplete_jedi.py?rev=KDE%2F4.13>`__,
   you have to enable it, though.
 
+Atom:
 
-.. _other-software:
+- autocomplete-python_
 
-Other Software Using Jedi
--------------------------
+SourceLair:
 
-- wdb_ - Web Debugger
+- SourceLair_
 
+GNOME Builder:
+
+- `GNOME Builder`_ `supports it natively
+  <https://git.gnome.org/browse/gnome-builder/tree/plugins/jedi>`__,
+  and is enabled by default.
+
+Gedit:
+
+- gedi_
+
+Eric IDE:
+
+- `Eric IDE`_ (Available as a plugin)
+
+Web Debugger:
+
+- wdb_
+
+and many more!
 
 .. _repl-completion:
 
@@ -77,6 +97,7 @@ Using a custom ``$HOME/.pythonrc.py``
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
 .. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
 .. _elpy: https://github.com/jorgenschaefer/elpy
 .. _anaconda-mode: https://github.com/proofit404/anaconda-mode
@@ -86,3 +107,8 @@ Using a custom ``$HOME/.pythonrc.py``
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
 .. _kate: http://kate-editor.org/
+.. _autocomplete-python: https://atom.io/packages/autocomplete-python
+.. _SourceLair: https://www.sourcelair.com
+.. _GNOME Builder: https://wiki.gnome.org/Apps/Builder/
+.. _gedi: https://github.com/isamert/gedi
+.. _Eric IDE: http://eric-ide.python-projects.org

jedi/__main__.py

@@ -1,18 +1,13 @@
-from sys import argv
+import sys
 from os.path import join, dirname, abspath, isdir
 
 
-if len(argv) == 2 and argv[1] == 'repl':
-    # don't want to use __main__ only for repl yet, maybe we want to use it for
-    # something else. So just use the keyword ``repl`` for now.
-    print(join(dirname(abspath(__file__)), 'api', 'replstartup.py'))
-elif len(argv) > 1 and argv[1] == 'linter':
+def _start_linter():
     """
     This is a pre-alpha API. You're not supposed to use it at all, except for
     testing. It will very likely change.
     """
     import jedi
-    import sys
 
     if '--debug' in sys.argv:
         jedi.set_debug_function()
@@ -37,7 +32,17 @@ elif len(argv) > 1 and argv[1] == 'linter':
                     print(error)
         except Exception:
             if '--pdb' in sys.argv:
+                import traceback
+                traceback.print_exc()
                 import pdb
                 pdb.post_mortem()
             else:
                 raise
+
+
+if len(sys.argv) == 2 and sys.argv[1] == 'repl':
+    # don't want to use __main__ only for repl yet, maybe we want to use it for
+    # something else. So just use the keyword ``repl`` for now.
+    print(join(dirname(abspath(__file__)), 'api', 'replstartup.py'))
+elif len(sys.argv) > 1 and sys.argv[1] == 'linter':
+    _start_linter()

jedi/_compatibility.py

@@ -13,6 +13,8 @@ except ImportError:
 
 is_py3 = sys.version_info[0] >= 3
 is_py33 = is_py3 and sys.version_info.minor >= 3
+is_py34 = is_py3 and sys.version_info.minor >= 4
+is_py35 = is_py3 and sys.version_info.minor >= 5
 is_py26 = not is_py3 and sys.version_info[1] < 7
 
 
@@ -25,7 +27,7 @@ def find_module_py33(string, path=None):
         except ValueError as e:
             # See #491. Importlib might raise a ValueError, to avoid this, we
             # just raise an ImportError to fix the issue.
-            raise ImportError("Originally ValueError: " + str(e))
+            raise ImportError("Originally  " + repr(e))
 
     if loader is None:
         raise ImportError("Couldn't find a loader for {0}".format(string))

jedi/api/__init__.py

@@ -3,38 +3,37 @@ The API basically only provides one class. You can create a :class:`Script` and
 use its methods.
 
 Additionally you can add a debug function with :func:`set_debug_function`.
+Alternatively, if you don't need a custom function and are happy with printing
+debug messages to stdout, simply call :func:`set_debug_function` without
+arguments.
 
 .. warning:: Please, note that Jedi is **not thread safe**.
 """
-import re
 import os
 import warnings
 import sys
-from itertools import chain
 
-from jedi._compatibility import unicode, builtins
-from jedi.parser import Parser, load_grammar
-from jedi.parser.tokenize import source_tokens
+from jedi._compatibility import unicode
+from jedi.parser import load_grammar
 from jedi.parser import tree
-from jedi.parser.user_context import UserContext, UserContextParser
+from jedi.parser.fast import FastParser
+from jedi.parser.utils import save_parser
 from jedi import debug
 from jedi import settings
 from jedi import common
 from jedi import cache
-from jedi.api import keywords
 from jedi.api import classes
 from jedi.api import interpreter
 from jedi.api import usages
 from jedi.api import helpers
+from jedi.api.completion import Completion
 from jedi.evaluate import Evaluator
 from jedi.evaluate import representation as er
-from jedi.evaluate import compiled
 from jedi.evaluate import imports
-from jedi.evaluate.cache import memoize_default
-from jedi.evaluate.helpers import FakeName, get_module_names
-from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names
-from jedi.evaluate import analysis
+from jedi.evaluate.param import try_iter_content
+from jedi.evaluate.helpers import get_module_names
 from jedi.evaluate.sys_path import get_venv_path
+from jedi.evaluate.iterable import unpack_tuple_to_dict
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -75,8 +74,8 @@ class Script(object):
     :type source: str
     :param line: The line to perform actions on (starting with 1).
     :type line: int
-    :param col: The column of the cursor (starting with 0).
-    :type col: int
+    :param column: The column of the cursor (starting with 0).
+    :type column: int
     :param path: The path of the file in the file system, or ``''`` if
         it hasn't been saved yet.
     :type path: str or None
@@ -101,31 +100,30 @@ class Script(object):
             encoding = source_encoding
 
         self._orig_path = path
-        self.path = None if path is None else os.path.abspath(path)
+        # An empty path (also empty string) should always result in no path.
+        self.path = os.path.abspath(path) if path else None
 
         if source is None:
+            # TODO add a better warning than the traceback!
             with open(path) as f:
                 source = f.read()
 
-        self.source = common.source_to_unicode(source, encoding)
-        lines = common.splitlines(self.source)
-        line = max(len(lines), 1) if line is None else line
-        if not (0 < line <= len(lines)):
+        self._source = common.source_to_unicode(source, encoding)
+        self._code_lines = common.splitlines(self._source)
+        line = max(len(self._code_lines), 1) if line is None else line
+        if not (0 < line <= len(self._code_lines)):
             raise ValueError('`line` parameter is not in a valid range.')
 
-        line_len = len(lines[line - 1])
+        line_len = len(self._code_lines[line - 1])
         column = line_len if column is None else column
         if not (0 <= column <= line_len):
             raise ValueError('`column` parameter is not in a valid range.')
         self._pos = line, column
+        self._path = path
 
         cache.clear_time_caches()
         debug.reset_time()
-        self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
-        self._user_context = UserContext(self.source, self._pos)
-        self._parser = UserContextParser(self._grammar, self.source, path,
-                                         self._pos, self._user_context,
-                                         self._parsed_callback)
+        self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
         if sys_path is None:
             venv = os.getenv('VIRTUAL_ENV')
             if venv:
@@ -133,9 +131,14 @@ class Script(object):
         self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
         debug.speed('init')
 
-    def _parsed_callback(self, parser):
+    def _get_module(self):
+        cache.invalidate_star_import_cache(self._path)
+        parser = FastParser(self._grammar, self._source, self.path)
+        save_parser(self.path, parser, pickling=False)
+
         module = self._evaluator.wrap(parser.module)
         imports.add_module(self._evaluator, unicode(module.name), module)
+        return parser.module
 
     @property
     def source_path(self):
@@ -158,187 +161,14 @@ class Script(object):
         :return: Completion objects, sorted by name and __ comes last.
         :rtype: list of :class:`classes.Completion`
         """
-        def get_completions(user_stmt, bs):
-            # TODO this closure is ugly. it also doesn't work with
-            # simple_complete (used for Interpreter), somehow redo.
-            module = self._evaluator.wrap(self._parser.module())
-            names, level, only_modules, unfinished_dotted = \
-                helpers.check_error_statements(module, self._pos)
-            completion_names = []
-            if names is not None:
-                imp_names = tuple(str(n) for n in names if n.end_pos < self._pos)
-                i = imports.Importer(self._evaluator, imp_names, module, level)
-                completion_names = i.completion_names(self._evaluator, only_modules)
-
-            # TODO this paragraph is necessary, but not sure it works.
-            context = self._user_context.get_context()
-            if not next(context).startswith('.'):  # skip the path
-                if next(context) == 'from':
-                    # completion is just "import" if before stands from ..
-                    if unfinished_dotted:
-                        return completion_names
-                    else:
-                        return keywords.keyword_names('import')
-
-            if isinstance(user_stmt, tree.Import):
-                module = self._parser.module()
-                completion_names += imports.completion_names(self._evaluator,
-                                                             user_stmt, self._pos)
-                return completion_names
-
-            if names is None and not isinstance(user_stmt, tree.Import):
-                if not path and not dot:
-                    # add keywords
-                    completion_names += keywords.keyword_names(all=True)
-                    # TODO delete? We should search for valid parser
-                    # transformations.
-                completion_names += self._simple_complete(path, dot, like)
-            return completion_names
-
         debug.speed('completions start')
-        path = self._user_context.get_path_until_cursor()
-        # Dots following an int are not the start of a completion but a float
-        # literal.
-        if re.search(r'^\d\.$', path):
-            return []
-        path, dot, like = helpers.completion_parts(path)
-
-        user_stmt = self._parser.user_stmt_with_whitespace()
-
-        b = compiled.builtin
-        completion_names = get_completions(user_stmt, b)
-
-        if not dot:
-            # add named params
-            for call_sig in self.call_signatures():
-                # Allow protected access, because it's a public API.
-                module = call_sig._name.get_parent_until()
-                # Compiled modules typically don't allow keyword arguments.
-                if not isinstance(module, compiled.CompiledObject):
-                    for p in call_sig.params:
-                        # Allow access on _definition here, because it's a
-                        # public API and we don't want to make the internal
-                        # Name object public.
-                        if p._definition.stars == 0:  # no *args/**kwargs
-                            completion_names.append(p._name)
-
-        needs_dot = not dot and path
-
-        comps = []
-        comp_dct = {}
-        for c in set(completion_names):
-            n = str(c)
-            if settings.case_insensitive_completion \
-                    and n.lower().startswith(like.lower()) \
-                    or n.startswith(like):
-                if isinstance(c.parent, (tree.Function, tree.Class)):
-                    # TODO I think this is a hack. It should be an
-                    #   er.Function/er.Class before that.
-                    c = self._evaluator.wrap(c.parent).name
-                new = classes.Completion(self._evaluator, c, needs_dot, len(like))
-                k = (new.name, new.complete)  # key
-                if k in comp_dct and settings.no_completion_duplicates:
-                    comp_dct[k]._same_name_completions.append(new)
-                else:
-                    comp_dct[k] = new
-                    comps.append(new)
-
+        completion = Completion(
+            self._evaluator, self._get_module(), self._code_lines,
+            self._pos, self.call_signatures
+        )
+        completions = completion.completions()
         debug.speed('completions end')
-
-        return sorted(comps, key=lambda x: (x.name.startswith('__'),
-                                            x.name.startswith('_'),
-                                            x.name.lower()))
-
-    def _simple_complete(self, path, dot, like):
-        if not path and not dot:
-            scope = self._parser.user_scope()
-            if not scope.is_scope():  # Might be a flow (if/while/etc).
-                scope = scope.get_parent_scope()
-            names_dicts = global_names_dict_generator(
-                self._evaluator,
-                self._evaluator.wrap(scope),
-                self._pos
-            )
-            completion_names = []
-            for names_dict, pos in names_dicts:
-                names = list(chain.from_iterable(names_dict.values()))
-                if not names:
-                    continue
-                completion_names += filter_definition_names(names, self._parser.user_stmt(), pos)
-        elif self._get_under_cursor_stmt(path) is None:
-            return []
-        else:
-            scopes = list(self._prepare_goto(path, True))
-            completion_names = []
-            debug.dbg('possible completion scopes: %s', scopes)
-            for s in scopes:
-                names = []
-                for names_dict in s.names_dicts(search_global=False):
-                    names += chain.from_iterable(names_dict.values())
-
-                completion_names += filter_definition_names(names, self._parser.user_stmt())
-        return completion_names
-
-    def _prepare_goto(self, goto_path, is_completion=False):
-        """
-        Base for completions/goto. Basically it returns the resolved scopes
-        under cursor.
-        """
-        debug.dbg('start: %s in %s', goto_path, self._parser.user_scope())
-
-        user_stmt = self._parser.user_stmt_with_whitespace()
-        if not user_stmt and len(goto_path.split('\n')) > 1:
-            # If the user_stmt is not defined and the goto_path is multi line,
-            # something's strange. Most probably the backwards tokenizer
-            # matched to much.
-            return []
-
-        if isinstance(user_stmt, tree.Import):
-            i, _ = helpers.get_on_import_stmt(self._evaluator, self._user_context,
-                                              user_stmt, is_completion)
-            if i is None:
-                return []
-            scopes = [i]
-        else:
-            # just parse one statement, take it and evaluate it
-            eval_stmt = self._get_under_cursor_stmt(goto_path)
-            if eval_stmt is None:
-                return []
-
-            module = self._evaluator.wrap(self._parser.module())
-            names, level, _, _ = helpers.check_error_statements(module, self._pos)
-            if names:
-                names = [str(n) for n in names]
-                i = imports.Importer(self._evaluator, names, module, level)
-                return i.follow()
-
-            scopes = self._evaluator.eval_element(eval_stmt)
-
-        return scopes
-
-    @memoize_default()
-    def _get_under_cursor_stmt(self, cursor_txt, start_pos=None):
-        tokenizer = source_tokens(cursor_txt)
-        r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer)
-        try:
-            # Take the last statement available that is not an endmarker.
-            # And because it's a simple_stmt, we need to get the first child.
-            stmt = r.module.children[-2].children[0]
-        except (AttributeError, IndexError):
-            return None
-
-        user_stmt = self._parser.user_stmt()
-        if user_stmt is None:
-            # Set the start_pos to a pseudo position, that doesn't exist but
-            # works perfectly well (for both completions in docstrings and
-            # statements).
-            pos = start_pos or self._pos
-        else:
-            pos = user_stmt.start_pos
-
-        stmt.move(pos[0] - 1, pos[1])  # Moving the offset.
-        stmt.parent = self._parser.user_scope()
-        return stmt
+        return completions
 
     def goto_definitions(self):
         """
@@ -352,39 +182,18 @@ class Script(object):
 
         :rtype: list of :class:`classes.Definition`
         """
-        def resolve_import_paths(scopes):
-            for s in scopes.copy():
-                if isinstance(s, imports.ImportWrapper):
-                    scopes.remove(s)
-                    scopes.update(resolve_import_paths(set(s.follow())))
-            return scopes
+        leaf = self._get_module().name_for_position(self._pos)
+        if leaf is None:
+            leaf = self._get_module().get_leaf_for_position(self._pos)
+            if leaf is None:
+                return []
+        definitions = helpers.evaluate_goto_definition(self._evaluator, leaf)
 
-        goto_path = self._user_context.get_path_under_cursor()
-        context = self._user_context.get_context()
-        definitions = set()
-        if next(context) in ('class', 'def'):
-            definitions = set([self._evaluator.wrap(self._parser.user_scope())])
-        else:
-            # Fetch definition of callee, if there's no path otherwise.
-            if not goto_path:
-                definitions = set(signature._definition
-                                  for signature in self.call_signatures())
-
-        if re.match('\w[\w\d_]*$', goto_path) and not definitions:
-            user_stmt = self._parser.user_stmt()
-            if user_stmt is not None and user_stmt.type == 'expr_stmt':
-                for name in user_stmt.get_defined_names():
-                    if name.start_pos <= self._pos <= name.end_pos:
-                        # TODO scaning for a name and then using it should be
-                        # the default.
-                        definitions = set(self._evaluator.goto_definition(name))
-
-        if not definitions and goto_path:
-            definitions = set(self._prepare_goto(goto_path))
-
-        definitions = resolve_import_paths(definitions)
         names = [s.name for s in definitions]
         defs = [classes.Definition(self._evaluator, name) for name in names]
+        # The additional set here allows the definitions to become unique in an
+        # API sense. In the internals we want to separate more things than in
+        # the API.
         return helpers.sorted_definitions(set(defs))
 
     def goto_assignments(self):
@@ -400,72 +209,14 @@ class Script(object):
         d = [classes.Definition(self._evaluator, d) for d in set(results)]
         return helpers.sorted_definitions(d)
 
-    def _goto(self, add_import_name=False):
+    def _goto(self):
         """
         Used for goto_assignments and usages.
-
-        :param add_import_name: Add the the name (if import) to the result.
         """
-        def follow_inexistent_imports(defs):
-            """ Imports can be generated, e.g. following
-            `multiprocessing.dummy` generates an import dummy in the
-            multiprocessing module. The Import doesn't exist -> follow.
-            """
-            definitions = set(defs)
-            for d in defs:
-                if isinstance(d.parent, tree.Import) \
-                        and d.start_pos == (0, 0):
-                    i = imports.ImportWrapper(self._evaluator, d.parent).follow(is_goto=True)
-                    definitions.remove(d)
-                    definitions |= follow_inexistent_imports(i)
-            return definitions
-
-        goto_path = self._user_context.get_path_under_cursor()
-        context = self._user_context.get_context()
-        user_stmt = self._parser.user_stmt()
-        user_scope = self._parser.user_scope()
-
-        stmt = self._get_under_cursor_stmt(goto_path)
-        if stmt is None:
+        name = self._get_module().name_for_position(self._pos)
+        if name is None:
             return []
-
-        if user_scope is None:
-            last_name = None
-        else:
-            # Try to use the parser if possible.
-            last_name = user_scope.name_for_position(self._pos)
-
-        if last_name is None:
-            last_name = stmt
-            while not isinstance(last_name, tree.Name):
-                try:
-                    last_name = last_name.children[-1]
-                except AttributeError:
-                    # Doesn't have a name in it.
-                    return []
-
-        if next(context) in ('class', 'def'):
-            # The cursor is on a class/function name.
-            user_scope = self._parser.user_scope()
-            definitions = set([user_scope.name])
-        elif isinstance(user_stmt, tree.Import):
-            s, name = helpers.get_on_import_stmt(self._evaluator,
-                                                 self._user_context, user_stmt)
-
-            definitions = self._evaluator.goto(name)
-        else:
-            # The Evaluator.goto function checks for definitions, but since we
-            # use a reverse tokenizer, we have new name_part objects, so we
-            # have to check the user_stmt here for positions.
-            if isinstance(user_stmt, tree.ExprStmt) \
-                    and isinstance(last_name.parent, tree.ExprStmt):
-                for name in user_stmt.get_defined_names():
-                    if name.start_pos <= self._pos <= name.end_pos:
-                        return [name]
-
-            defs = self._evaluator.goto(last_name)
-            definitions = follow_inexistent_imports(defs)
-        return definitions
+        return list(self._evaluator.goto(name))
 
     def usages(self, additional_module_paths=()):
         """
@@ -481,8 +232,8 @@ class Script(object):
         temp, settings.dynamic_flow_information = \
             settings.dynamic_flow_information, False
         try:
-            user_stmt = self._parser.user_stmt()
-            definitions = self._goto(add_import_name=True)
+            user_stmt = self._get_module().get_statement_for_position(self._pos)
+            definitions = self._goto()
             if not definitions and isinstance(user_stmt, tree.Import):
                 # For not defined imports (goto doesn't find something, we take
                 # the name as a definition. This is enough, because every name
@@ -503,7 +254,7 @@ class Script(object):
                                                                definitions)
 
             module = set([d.get_parent_until() for d in definitions])
-            module.add(self._parser.module())
+            module.add(self._get_module())
             names = usages.usages(self._evaluator, definitions, module)
 
             for d in set(definitions):
@@ -525,50 +276,59 @@ class Script(object):
 
             abs()# <-- cursor is here
 
-        This would return ``None``.
+        This would return an empty list..
 
         :rtype: list of :class:`classes.CallSignature`
         """
-        call_txt, call_index, key_name, start_pos = self._user_context.call_signature()
-        if call_txt is None:
-            return []
-
-        stmt = self._get_under_cursor_stmt(call_txt, start_pos)
-        if stmt is None:
+        call_signature_details = \
+            helpers.get_call_signature_details(self._get_module(), self._pos)
+        if call_signature_details is None:
             return []
 
         with common.scale_speed_settings(settings.scale_call_signatures):
-            origins = cache.cache_call_signatures(self._evaluator, stmt,
-                                                  self.source, self._pos)
+            definitions = helpers.cache_call_signatures(
+                self._evaluator,
+                call_signature_details.bracket_leaf,
+                self._code_lines,
+                self._pos
+            )
         debug.speed('func_call followed')
 
-        return [classes.CallSignature(self._evaluator, o.name, stmt, call_index, key_name)
-                for o in origins if hasattr(o, 'py__call__')]
+        return [classes.CallSignature(self._evaluator, d.name,
+                                      call_signature_details.bracket_leaf.start_pos,
+                                      call_signature_details.call_index,
+                                      call_signature_details.keyword_name_str)
+                for d in definitions if hasattr(d, 'py__call__')]
 
     def _analysis(self):
-        def check_types(types):
-            for typ in types:
-                try:
-                    f = typ.iter_content
-                except AttributeError:
-                    pass
+        self._evaluator.is_analysis = True
+        self._evaluator.analysis_modules = [self._get_module()]
+        try:
+            for node in self._get_module().nodes_to_execute():
+                if node.type in ('funcdef', 'classdef'):
+                    if node.type == 'classdef':
+                        continue
+                        raise NotImplementedError
+                    er.Function(self._evaluator, node).get_decorated_func()
+                elif isinstance(node, tree.Import):
+                    import_names = set(node.get_defined_names())
+                    if node.is_nested():
+                        import_names |= set(path[-1] for path in node.paths())
+                    for n in import_names:
+                        imports.ImportWrapper(self._evaluator, n).follow()
+                elif node.type == 'expr_stmt':
+                    types = self._evaluator.eval_element(node)
+                    for testlist in node.children[:-1:2]:
+                        # Iterate tuples.
+                        unpack_tuple_to_dict(self._evaluator, types, testlist)
                 else:
-                    check_types(f())
+                    try_iter_content(self._evaluator.goto_definitions(node))
+                self._evaluator.reset_recursion_limitations()
 
-        #statements = set(chain(*self._parser.module().used_names.values()))
-        nodes, imp_names, decorated_funcs = \
-            analysis.get_module_statements(self._parser.module())
-        # Sort the statements so that the results are reproducible.
-        for n in imp_names:
-            imports.ImportWrapper(self._evaluator, n).follow()
-        for node in sorted(nodes, key=lambda obj: obj.start_pos):
-            check_types(self._evaluator.eval_element(node))
-
-        for dec_func in decorated_funcs:
-            er.Function(self._evaluator, dec_func).get_decorated_func()
-
-        ana = [a for a in self._evaluator.analysis if self.path == a.path]
-        return sorted(set(ana), key=lambda x: x.line)
+            ana = [a for a in self._evaluator.analysis if self.path == a.path]
+            return sorted(set(ana), key=lambda x: x.line)
+        finally:
+            self._evaluator.is_analysis = False
 
 
 class Interpreter(Script):
@@ -582,7 +342,7 @@ class Interpreter(Script):
 
     >>> from os.path import join
     >>> namespace = locals()
-    >>> script = Interpreter('join().up', [namespace])
+    >>> script = Interpreter('join("").up', [namespace])
     >>> print(script.completions()[0].name)
     upper
     """
@@ -601,61 +361,19 @@ class Interpreter(Script):
         If `line` and `column` are None, they are assumed be at the end of
         `source`.
         """
-        if type(namespaces) is not list or len(namespaces) == 0 or \
-           any([type(x) is not dict for x in namespaces]):
-            raise TypeError("namespaces must be a non-empty list of dict")
+        try:
+            namespaces = [dict(n) for n in namespaces]
+        except Exception:
+            raise TypeError("namespaces must be a non-empty list of dicts.")
 
         super(Interpreter, self).__init__(source, **kwds)
         self.namespaces = namespaces
 
-        # Don't use the fast parser, because it does crazy stuff that we don't
-        # need in our very simple and small code here (that is always
-        # changing).
-        self._parser = UserContextParser(self._grammar, self.source,
-                                         self._orig_path, self._pos,
-                                         self._user_context, self._parsed_callback,
-                                         use_fast_parser=False)
-        interpreter.add_namespaces_to_parser(self._evaluator, namespaces,
-                                             self._parser.module())
+        parser_module = super(Interpreter, self)._get_module()
+        self._module = interpreter.MixedModule(self._evaluator, parser_module, self.namespaces)
 
-    def _simple_complete(self, path, dot, like):
-        user_stmt = self._parser.user_stmt_with_whitespace()
-        is_simple_path = not path or re.search('^[\w][\w\d.]*$', path)
-        if isinstance(user_stmt, tree.Import) or not is_simple_path:
-            return super(Interpreter, self)._simple_complete(path, dot, like)
-        else:
-            class NamespaceModule(object):
-                def __getattr__(_, name):
-                    for n in self.namespaces:
-                        try:
-                            return n[name]
-                        except KeyError:
-                            pass
-                    raise AttributeError()
-
-                def __dir__(_):
-                    gen = (n.keys() for n in self.namespaces)
-                    return list(set(chain.from_iterable(gen)))
-
-            paths = path.split('.') if path else []
-
-            namespaces = (NamespaceModule(), builtins)
-            for p in paths:
-                old, namespaces = namespaces, []
-                for n in old:
-                    try:
-                        namespaces.append(getattr(n, p))
-                    except Exception:
-                        pass
-
-            completion_names = []
-            for namespace in namespaces:
-                for name in dir(namespace):
-                    if name.lower().startswith(like.lower()):
-                        scope = self._parser.module()
-                        n = FakeName(name, scope)
-                        completion_names.append(n)
-            return completion_names
+    def _get_module(self):
+        return self._module
 
 
 def defined_names(source, path=None, encoding='utf-8'):
@@ -701,7 +419,7 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False,
     # Set line/column to a random position, because they don't matter.
     script = Script(source, line=1, column=0, path=path, encoding=encoding)
     defs = [classes.Definition(script._evaluator, name_part)
-            for name_part in get_module_names(script._parser.module(), all_scopes)]
+            for name_part in get_module_names(script._get_module(), all_scopes)]
     return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
 
 
@@ -722,6 +440,8 @@ def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
     """
     Define a callback debug function to get all the debug messages.
 
+    If you don't specify any arguments, debug messages will be printed to stdout.
+
     :param func_cb: The callback function for debug messages, with n params.
     """
     debug.debug_function = func_cb

jedi/api/classes.py

@@ -16,6 +16,7 @@ from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
 from jedi.evaluate import imports
 from jedi.evaluate import compiled
+from jedi.evaluate.compiled import mixed
 from jedi.api import keywords
 from jedi.evaluate.finder import filter_definition_names
 
@@ -148,7 +149,7 @@ class BaseDefinition(object):
         if isinstance(stripped, er.InstanceElement):
             stripped = stripped.var
 
-        if isinstance(stripped, compiled.CompiledObject):
+        if isinstance(stripped, (compiled.CompiledObject, mixed.MixedObject)):
             return stripped.api_type()
         elif isinstance(stripped, iterable.Array):
             return 'instance'
@@ -322,7 +323,7 @@ class BaseDefinition(object):
         elif self._definition.isinstance(tree.Import):
             return imports.ImportWrapper(self._evaluator, self._name).follow()
         else:
-            return [self._definition]
+            return set([self._definition])
 
     @property
     @memoize_default()
@@ -331,7 +332,7 @@ class BaseDefinition(object):
         Raises an ``AttributeError``if the definition is not callable.
         Otherwise returns a list of `Definition` that represents the params.
         """
-        followed = self._follow_statements_imports()
+        followed = list(self._follow_statements_imports())
         if not followed or not hasattr(followed[0], 'py__call__'):
             raise AttributeError()
         followed = followed[0]  # only check the first one.
@@ -365,33 +366,31 @@ class Completion(BaseDefinition):
     `Completion` objects are returned from :meth:`api.Script.completions`. They
     provide additional information about a completion.
     """
-    def __init__(self, evaluator, name, needs_dot, like_name_length):
+    def __init__(self, evaluator, name, stack, like_name_length):
         super(Completion, self).__init__(evaluator, name)
 
-        self._needs_dot = needs_dot
         self._like_name_length = like_name_length
+        self._stack = stack
 
         # Completion objects with the same Completion name (which means
         # duplicate items in the completion)
         self._same_name_completions = []
 
     def _complete(self, like_name):
-        dot = '.' if self._needs_dot else ''
         append = ''
         if settings.add_bracket_after_function \
                 and self.type == 'Function':
             append = '('
 
-        if settings.add_dot_after_module:
-            if isinstance(self._definition, tree.Module):
-                append += '.'
-        if isinstance(self._definition, tree.Param):
-            append += '='
+        if isinstance(self._definition, tree.Param) and self._stack is not None:
+            node_names = list(self._stack.get_node_names(self._evaluator.grammar))
+            if 'trailer' in node_names and 'argument' not in node_names:
+                append += '='
 
         name = str(self._name)
         if like_name:
             name = name[self._like_name_length:]
-        return dot + name + append
+        return name + append
 
     @property
     def complete(self):
@@ -449,7 +448,7 @@ class Completion(BaseDefinition):
                 followed = self._follow_statements_imports()
                 if followed:
                     # TODO: Use all of the followed objects as input to Documentation.
-                    definition = followed[0]
+                    definition = list(followed)[0]
 
         if raw:
             return _Help(definition).raw()
@@ -629,11 +628,11 @@ class CallSignature(Definition):
     It knows what functions you are currently in. e.g. `isinstance(` would
     return the `isinstance` function. without `(` it would return nothing.
     """
-    def __init__(self, evaluator, executable_name, call_stmt, index, key_name):
+    def __init__(self, evaluator, executable_name, bracket_start_pos, index, key_name_str):
         super(CallSignature, self).__init__(evaluator, executable_name)
         self._index = index
-        self._key_name = key_name
-        self._call_stmt = call_stmt
+        self._key_name_str = key_name_str
+        self._bracket_start_pos = bracket_start_pos
 
     @property
     def index(self):
@@ -641,9 +640,9 @@ class CallSignature(Definition):
         The Param index of the current call.
         Returns None if the index cannot be found in the curent call.
         """
-        if self._key_name is not None:
+        if self._key_name_str is not None:
             for i, param in enumerate(self.params):
-                if self._key_name == param.name:
+                if self._key_name_str == param.name:
                     return i
             if self.params and self.params[-1]._name.get_definition().stars == 2:
                 return i
@@ -665,7 +664,7 @@ class CallSignature(Definition):
         The indent of the bracket that is responsible for the last function
         call.
         """
-        return self._call_stmt.end_pos
+        return self._bracket_start_pos
 
     @property
     def call_name(self):

jedi/api/completion.py

@@ -0,0 +1,231 @@
+from itertools import chain
+
+from jedi.parser import token
+from jedi.parser import tree
+from jedi import debug
+from jedi import settings
+from jedi.api import classes
+from jedi.api import helpers
+from jedi.evaluate import imports
+from jedi.api import keywords
+from jedi.evaluate import compiled
+from jedi.evaluate.helpers import call_of_leaf
+from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names
+
+
+def get_call_signature_param_names(call_signatures):
+    # add named params
+    for call_sig in call_signatures:
+        # Allow protected access, because it's a public API.
+        module = call_sig._name.get_parent_until()
+        # Compiled modules typically don't allow keyword arguments.
+        if not isinstance(module, compiled.CompiledObject):
+            for p in call_sig.params:
+                # Allow access on _definition here, because it's a
+                # public API and we don't want to make the internal
+                # Name object public.
+                if p._definition.stars == 0:  # no *args/**kwargs
+                    yield p._name
+
+
+def filter_names(evaluator, completion_names, stack, like_name):
+    comp_dct = {}
+    for name in set(completion_names):
+        if settings.case_insensitive_completion \
+                and str(name).lower().startswith(like_name.lower()) \
+                or str(name).startswith(like_name):
+
+            if isinstance(name.parent, (tree.Function, tree.Class)):
+                # TODO I think this is a hack. It should be an
+                #   er.Function/er.Class before that.
+                name = evaluator.wrap(name.parent).name
+            new = classes.Completion(
+                evaluator,
+                name,
+                stack,
+                len(like_name)
+            )
+            k = (new.name, new.complete)  # key
+            if k in comp_dct and settings.no_completion_duplicates:
+                comp_dct[k]._same_name_completions.append(new)
+            else:
+                comp_dct[k] = new
+                yield new
+
+
+def get_user_scope(module, position):
+    """
+    Returns the scope in which the user resides. This includes flows.
+    """
+    user_stmt = module.get_statement_for_position(position)
+    if user_stmt is None:
+        def scan(scope):
+            for s in scope.children:
+                if s.start_pos <= position <= s.end_pos:
+                    if isinstance(s, (tree.Scope, tree.Flow)):
+                        return scan(s) or s
+                    elif s.type in ('suite', 'decorated'):
+                        return scan(s)
+            return None
+
+        return scan(module) or module
+    else:
+        return user_stmt.get_parent_scope(include_flows=True)
+
+
+class Completion:
+    def __init__(self, evaluator, module, code_lines, position, call_signatures_method):
+        self._evaluator = evaluator
+        self._module = evaluator.wrap(module)
+        self._code_lines = code_lines
+
+        # The first step of completions is to get the name
+        self._like_name = helpers.get_on_completion_name(code_lines, position)
+        # The actual cursor position is not what we need to calculate
+        # everything. We want the start of the name we're on.
+        self._position = position[0], position[1] - len(self._like_name)
+        self._call_signatures_method = call_signatures_method
+
+    def completions(self):
+        completion_names = self._get_context_completions()
+
+        completions = filter_names(self._evaluator, completion_names,
+                                   self.stack, self._like_name)
+
+        return sorted(completions, key=lambda x: (x.name.startswith('__'),
+                                                  x.name.startswith('_'),
+                                                  x.name.lower()))
+
+    def _get_context_completions(self):
+        """
+        Analyzes the context that a completion is made in and decides what to
+        return.
+
+        Technically this works by generating a parser stack and analysing the
+        current stack for possible grammar nodes.
+
+        Possible enhancements:
+        - global/nonlocal search global
+        - yield from / raise from <- could be only exceptions/generators
+        - In args: */**: no completion
+        - In params (also lambda): no completion before =
+        """
+
+        grammar = self._evaluator.grammar
+
+        try:
+            self.stack = helpers.get_stack_at_position(
+                grammar, self._code_lines, self._module, self._position
+            )
+        except helpers.OnErrorLeaf as e:
+            self.stack = None
+            if e.error_leaf.value == '.':
+                # After ErrorLeaf's that are dots, we will not do any
+                # completions since this probably just confuses the user.
+                return []
+            # If we don't have a context, just use global completion.
+
+            return self._global_completions()
+
+        allowed_keywords, allowed_tokens = \
+            helpers.get_possible_completion_types(grammar, self.stack)
+
+        completion_names = list(self._get_keyword_completion_names(allowed_keywords))
+
+        if token.NAME in allowed_tokens:
+            # This means that we actually have to do type inference.
+
+            symbol_names = list(self.stack.get_node_names(grammar))
+
+            nodes = list(self.stack.get_nodes())
+
+            if "import_stmt" in symbol_names:
+                level = 0
+                only_modules = True
+                level, names = self._parse_dotted_names(nodes)
+                if "import_from" in symbol_names:
+                    if 'import' in nodes:
+                        only_modules = False
+                else:
+                    assert "import_name" in symbol_names
+
+                completion_names += self._get_importer_names(
+                    names,
+                    level,
+                    only_modules
+                )
+            elif nodes and nodes[-1] in ('as', 'def', 'class'):
+                # No completions for ``with x as foo`` and ``import x as foo``.
+                # Also true for defining names as a class or function.
+                return []
+            elif symbol_names[-1] == 'trailer' and nodes[-1] == '.':
+                dot = self._module.get_leaf_for_position(self._position)
+                atom_expr = call_of_leaf(dot.get_previous_leaf())
+                completion_names += self._trailer_completions(atom_expr)
+            else:
+                completion_names += self._global_completions()
+
+            if 'trailer' in symbol_names:
+                call_signatures = self._call_signatures_method()
+                completion_names += get_call_signature_param_names(call_signatures)
+
+        return completion_names
+
+    def _get_keyword_completion_names(self, keywords_):
+        for k in keywords_:
+            yield keywords.keyword(self._evaluator, k).name
+
+    def _global_completions(self):
+        scope = get_user_scope(self._module, self._position)
+        if not scope.is_scope():  # Might be a flow (if/while/etc).
+            scope = scope.get_parent_scope()
+        scope = self._evaluator.wrap(scope)
+        debug.dbg('global completion scope: %s', scope)
+        names_dicts = global_names_dict_generator(
+            self._evaluator,
+            scope,
+            self._position
+        )
+        completion_names = []
+        for names_dict, pos in names_dicts:
+            names = list(chain.from_iterable(names_dict.values()))
+            if not names:
+                continue
+            completion_names += filter_definition_names(
+                names, self._module.get_statement_for_position(self._position), pos
+            )
+        return completion_names
+
+    def _trailer_completions(self, atom_expr):
+        scopes = self._evaluator.eval_element(atom_expr)
+        completion_names = []
+        debug.dbg('trailer completion scopes: %s', scopes)
+        for s in scopes:
+            names = []
+            for names_dict in s.names_dicts(search_global=False):
+                names += chain.from_iterable(names_dict.values())
+
+            completion_names += filter_definition_names(
+                names, self._module.get_statement_for_position(self._position)
+            )
+        return completion_names
+
+    def _parse_dotted_names(self, nodes):
+        level = 0
+        names = []
+        for node in nodes[1:]:
+            if node in ('.', '...'):
+                if not names:
+                    level += len(node.value)
+            elif node.type == 'dotted_name':
+                names += node.children[::2]
+            elif node.type == 'name':
+                names.append(node)
+            else:
+                break
+        return level, names
+
+    def _get_importer_names(self, names, level=0, only_modules=True):
+        names = [str(n) for n in names]
+        i = imports.Importer(self._evaluator, names, self._module, level)
+        return i.completion_names(self._evaluator, only_modules=only_modules)

jedi/api/helpers.py

@@ -2,18 +2,17 @@
 Helpers for the API
 """
 import re
+from collections import namedtuple
+from textwrap import dedent
 
-from jedi.parser import tree as pt
-from jedi.evaluate import imports
+from jedi._compatibility import u
+from jedi.evaluate.helpers import call_of_leaf
+from jedi import parser
+from jedi.parser import tokenize, token
+from jedi.cache import time_cache
 
 
-def completion_parts(path_until_cursor):
-    """
-    Returns the parts for the completion
-    :return: tuple - (path, dot, like)
-    """
-    match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path_until_cursor, flags=re.S)
-    return match.groups()
+CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
 
 
 def sorted_definitions(defs):
@@ -21,58 +20,254 @@ def sorted_definitions(defs):
     return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0))
 
 
-def get_on_import_stmt(evaluator, user_context, user_stmt, is_like_search=False):
+def get_on_completion_name(lines, position):
+    line = lines[position[0] - 1]
+    # The first step of completions is to get the name
+    return re.search(
+        r'(?!\d)\w+$|$', line[:position[1]]
+    ).group(0)
+
+
+def _get_code(code_lines, start_pos, end_pos):
+    # Get relevant lines.
+    lines = code_lines[start_pos[0] - 1:end_pos[0]]
+    # Remove the parts at the end of the line.
+    lines[-1] = lines[-1][:end_pos[1]]
+    # Remove first line indentation.
+    lines[0] = lines[0][start_pos[1]:]
+    return '\n'.join(lines)
+
+
+class OnErrorLeaf(Exception):
+    @property
+    def error_leaf(self):
+        return self.args[0]
+
+
+def get_stack_at_position(grammar, code_lines, module, pos):
     """
-    Resolve the user statement, if it is an import. Only resolve the
-    parts until the user position.
+    Returns the possible node names (e.g. import_from, xor_test or yield_stmt).
     """
-    name = user_stmt.name_for_position(user_context.position)
-    if name is None:
-        return None, None
+    user_stmt = module.get_statement_for_position(pos)
 
-    i = imports.ImportWrapper(evaluator, name)
-    return i, name
+    if user_stmt is not None and user_stmt.type in ('indent', 'dedent'):
+        code = u('')
+    else:
+        if user_stmt is None:
+            user_stmt = module.get_leaf_for_position(pos, include_prefixes=True)
+        if pos <= user_stmt.start_pos:
+            try:
+                leaf = user_stmt.get_previous_leaf()
+            except IndexError:
+                pass
+            else:
+                user_stmt = module.get_statement_for_position(leaf.start_pos)
+
+        if user_stmt.type == 'error_leaf' or user_stmt.type == 'string':
+            # Error leafs cannot be parsed, completion in strings is also
+            # impossible.
+            raise OnErrorLeaf(user_stmt)
+
+        start_pos = user_stmt.start_pos
+        if user_stmt.first_leaf() == '@':
+            # TODO this once again proves that just using user_stmt.get_code
+            #      would probably be nicer than _get_code.
+            # Remove the indent to have a statement that is aligned (properties
+            # on the same line as function)
+            start_pos = start_pos[0], 0
+
+        code = _get_code(code_lines, start_pos, pos)
+        if code == ';':
+            # ; cannot be parsed.
+            code = u('')
+
+        # Remove whitespace at the end. Necessary, because the tokenizer will parse
+        # an error token (there's no new line at the end in our case). This doesn't
+        # alter any truth about the valid tokens at that position.
+        code = code.rstrip('\t ')
+        # Remove as many indents from **all** code lines as possible.
+        code = dedent(code)
+
+    class EndMarkerReached(Exception):
+        pass
+
+    def tokenize_without_endmarker(code):
+        tokens = tokenize.source_tokens(code, use_exact_op_types=True)
+        for token_ in tokens:
+            if token_[0] == token.ENDMARKER:
+                raise EndMarkerReached()
+            elif token_[0] == token.DEDENT:
+                # Ignore those. Error statements should not contain them, if
+                # they do it's for cases where an indentation happens and
+                # before the endmarker we still see them.
+                pass
+            else:
+                yield token_
+
+    p = parser.Parser(grammar, code, start_parsing=False)
+    try:
+        p.parse(tokenizer=tokenize_without_endmarker(code))
+    except EndMarkerReached:
+        return Stack(p.stack)
 
 
-def check_error_statements(module, pos):
-    for error_statement in module.error_statement_stacks:
-        if error_statement.first_type in ('import_from', 'import_name') \
-                and error_statement.first_pos < pos <= error_statement.next_start_pos:
-            return importer_from_error_statement(error_statement, pos)
-    return None, 0, False, False
+class Stack(list):
+    def get_node_names(self, grammar):
+        for dfa, state, (node_number, nodes) in self:
+            yield grammar.number2symbol[node_number]
 
-
-def importer_from_error_statement(error_statement, pos):
-    def check_dotted(children):
-        for name in children[::2]:
-            if name.start_pos <= pos:
-                yield name
-
-    names = []
-    level = 0
-    only_modules = True
-    unfinished_dotted = False
-    for typ, nodes in error_statement.stack:
-        if typ == 'dotted_name':
-            names += check_dotted(nodes)
-            if nodes[-1] == '.':
-                # An unfinished dotted_name
-                unfinished_dotted = True
-        elif typ == 'import_name':
-            if nodes[0].start_pos <= pos <= nodes[0].end_pos:
-                # We are on the import.
-                return None, 0, False, False
-        elif typ == 'import_from':
+    def get_nodes(self):
+        for dfa, state, (node_number, nodes) in self:
             for node in nodes:
-                if node.start_pos >= pos:
-                    break
-                elif isinstance(node, pt.Node) and node.type == 'dotted_name':
-                    names += check_dotted(node.children)
-                elif node in ('.', '...'):
-                    level += len(node.value)
-                elif isinstance(node, pt.Name):
-                    names.append(node)
-                elif node == 'import':
-                    only_modules = False
+                yield node
 
-    return names, level, only_modules, unfinished_dotted
+
+def get_possible_completion_types(grammar, stack):
+    def add_results(label_index):
+        try:
+            grammar_labels.append(inversed_tokens[label_index])
+        except KeyError:
+            try:
+                keywords.append(inversed_keywords[label_index])
+            except KeyError:
+                t, v = grammar.labels[label_index]
+                assert t >= 256
+                # See if it's a symbol and if we're in its first set
+                inversed_keywords
+                itsdfa = grammar.dfas[t]
+                itsstates, itsfirst = itsdfa
+                for first_label_index in itsfirst.keys():
+                    add_results(first_label_index)
+
+    inversed_keywords = dict((v, k) for k, v in grammar.keywords.items())
+    inversed_tokens = dict((v, k) for k, v in grammar.tokens.items())
+
+    keywords = []
+    grammar_labels = []
+
+    def scan_stack(index):
+        dfa, state, node = stack[index]
+        states, first = dfa
+        arcs = states[state]
+
+        for label_index, new_state in arcs:
+            if label_index == 0:
+                # An accepting state, check the stack below.
+                scan_stack(index - 1)
+            else:
+                add_results(label_index)
+
+    scan_stack(-1)
+
+    return keywords, grammar_labels
+
+
+def evaluate_goto_definition(evaluator, leaf):
+    if leaf.type == 'name':
+        # In case of a name we can just use goto_definition which does all the
+        # magic itself.
+        return evaluator.goto_definitions(leaf)
+
+    node = None
+    parent = leaf.parent
+    if parent.type == 'atom':
+        node = leaf.parent
+    elif parent.type == 'trailer':
+        node = call_of_leaf(leaf)
+
+    if node is None:
+        return []
+    return evaluator.eval_element(node)
+
+
+CallSignatureDetails = namedtuple(
+    'CallSignatureDetails',
+    ['bracket_leaf', 'call_index', 'keyword_name_str']
+)
+
+
+def _get_index_and_key(nodes, position):
+    """
+    Returns the amount of commas and the keyword argument string.
+    """
+    nodes_before = [c for c in nodes if c.start_pos < position]
+    if nodes_before[-1].type == 'arglist':
+        nodes_before = [c for c in nodes_before[-1].children if c.start_pos < position]
+
+    key_str = None
+
+    if nodes_before:
+        last = nodes_before[-1]
+        if last.type == 'argument' and last.children[1].end_pos <= position:
+            # Checked if the argument
+            key_str = last.children[0].value
+        elif last == '=':
+            key_str = nodes_before[-2].value
+
+    return nodes_before.count(','), key_str
+
+
+def _get_call_signature_details_from_error_node(node, position):
+    for index, element in reversed(list(enumerate(node.children))):
+        # `index > 0` means that it's a trailer and not an atom.
+        if element == '(' and element.end_pos <= position and index > 0:
+            # It's an error node, we don't want to match too much, just
+            # until the parentheses is enough.
+            children = node.children[index:]
+            name = element.get_previous_leaf()
+            if name.type == 'name' or name.parent.type in ('trailer', 'atom'):
+                return CallSignatureDetails(
+                    element,
+                    *_get_index_and_key(children, position)
+                )
+
+
+def get_call_signature_details(module, position):
+    leaf = module.get_leaf_for_position(position, include_prefixes=True)
+    if leaf == ')':
+        if leaf.end_pos == position:
+            leaf = leaf.get_next_leaf()
+    # Now that we know where we are in the syntax tree, we start to look at
+    # parents for possible function definitions.
+    node = leaf.parent
+    while node is not None:
+        if node.type in ('funcdef', 'classdef'):
+            # Don't show call signatures if there's stuff before it that just
+            # makes it feel strange to have a call signature.
+            return None
+
+        for n in node.children:
+            if n.start_pos < position and n.type == 'error_node':
+                result = _get_call_signature_details_from_error_node(n, position)
+                if result is not None:
+                    return result
+
+        if node.type == 'trailer' and node.children[0] == '(':
+            leaf = node.get_previous_leaf()
+            return CallSignatureDetails(
+                node.children[0], *_get_index_and_key(node.children, position))
+
+        node = node.parent
+
+    return None
+
+
+@time_cache("call_signatures_validity")
+def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos):
+    """This function calculates the cache key."""
+    index = user_pos[0] - 1
+
+    before_cursor = code_lines[index][:user_pos[1]]
+    other_lines = code_lines[bracket_leaf.start_pos[0]:index]
+    whole = '\n'.join(other_lines + [before_cursor])
+    before_bracket = re.match(r'.*\(', whole, re.DOTALL)
+
+    module_path = bracket_leaf.get_parent_until().path
+    if module_path is None:
+        yield None  # Don't cache!
+    else:
+        yield (module_path, before_bracket, bracket_leaf.start_pos)
+    yield evaluate_goto_definition(
+        evaluator,
+        bracket_leaf.get_previous_leaf()
+    )

jedi/api/interpreter.py

@@ -1,30 +1,43 @@
 """
 TODO Some parts of this module are still not well documented.
 """
-import inspect
-import re
+import copy
 
-from jedi._compatibility import builtins
-from jedi import debug
-from jedi.common import source_to_unicode
 from jedi.cache import underscore_memoization
-from jedi.evaluate import compiled
-from jedi.evaluate.compiled.fake import get_module
-from jedi.parser import tree as pt
-from jedi.parser import load_grammar
-from jedi.parser.fast import FastParser
 from jedi.evaluate import helpers
-from jedi.evaluate import iterable
-from jedi.evaluate import representation as er
+from jedi.evaluate.representation import ModuleWrapper
+from jedi.evaluate.compiled import mixed
 
 
-def add_namespaces_to_parser(evaluator, namespaces, parser_module):
-    for namespace in namespaces:
-        for key, value in namespace.items():
-            # Name lookups in an ast tree work by checking names_dict.
-            # Therefore we just add fake names to that and we're done.
-            arr = parser_module.names_dict.setdefault(key, [])
-            arr.append(LazyName(evaluator, parser_module, key, value))
+class MixedModule(object):
+    resets_positions = True
+    type = 'mixed_module'
+
+    def __init__(self, evaluator, parser_module, namespaces):
+        self._evaluator = evaluator
+        self._namespaces = namespaces
+
+        self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces]
+        self._wrapped_module = ModuleWrapper(evaluator, parser_module)
+        # Usually we are dealing with very small code sizes when it comes to
+        # interpreter modules. In this case we just copy the whole syntax tree
+        # to be able to modify it.
+        self._parser_module = copy.deepcopy(parser_module)
+
+        for child in self._parser_module.children:
+            child.parent = self
+
+    def names_dicts(self, search_global):
+        for names_dict in self._wrapped_module.names_dicts(search_global):
+            yield names_dict
+
+        for namespace_obj in self._namespace_objects:
+            m = mixed.MixedObject(self._evaluator, namespace_obj, self._parser_module.name)
+            for names_dict in m.names_dicts(False):
+                yield names_dict
+
+    def __getattr__(self, name):
+        return getattr(self._parser_module, name)
 
 
 class LazyName(helpers.FakeName):
@@ -43,66 +56,11 @@ class LazyName(helpers.FakeName):
     def parent(self):
         """
         Creating fake statements for the interpreter.
+
+        Here we are trying to link back to Python code, if possible. This means
+        we try to find the python module for a name (not the builtin).
         """
-        obj = self._value
-        parser_path = []
-        if inspect.ismodule(obj):
-            module = obj
-        else:
-            names = []
-            try:
-                o = obj.__objclass__
-                names.append(obj.__name__)
-                obj = o
-            except AttributeError:
-                pass
-
-            try:
-                module_name = obj.__module__
-                names.insert(0, obj.__name__)
-            except AttributeError:
-                # Unfortunately in some cases like `int` there's no __module__
-                module = builtins
-            else:
-                # TODO this import is wrong. Yields x for x.y.z instead of z
-                module = __import__(module_name)
-            parser_path = names
-        raw_module = get_module(self._value)
-
-        found = []
-        try:
-            path = module.__file__
-        except AttributeError:
-            pass
-        else:
-            path = re.sub('c$', '', path)
-            if path.endswith('.py'):
-                # cut the `c` from `.pyc`
-                with open(path) as f:
-                    source = source_to_unicode(f.read())
-                mod = FastParser(load_grammar(), source, path[:-1]).module
-                if parser_path:
-                    assert len(parser_path) == 1
-                    found = self._evaluator.find_types(mod, parser_path[0], search_global=True)
-                else:
-                    found = [self._evaluator.wrap(mod)]
-
-                if not found:
-                    debug.warning('Possibly an interpreter lookup for Python code failed %s',
-                                  parser_path)
-
-        if not found:
-            evaluated = compiled.CompiledObject(obj)
-            if evaluated == builtins:
-                # The builtins module is special and always cached.
-                evaluated = compiled.builtin
-            found = [evaluated]
-
-        content = iterable.AlreadyEvaluated(found)
-        stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier,
-                                              '=', (0, 0), ''), content])
-        stmt.parent = self._module
-        return stmt
+        return mixed.create(self._evaluator, self._value)
 
     @parent.setter
     def parent(self, value):

jedi/api/keywords.py

@@ -1,11 +1,10 @@
 import pydoc
 import keyword
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from jedi import common
-from jedi.evaluate import compiled
 from jedi.evaluate.helpers import FakeName
-
+from jedi.parser.tree import Leaf
 try:
     from pydoc_data import topics as pydoc_topics
 except ImportError:
@@ -13,36 +12,75 @@ except ImportError:
     import pydoc_topics
 
 if is_py3:
-    keys = keyword.kwlist
+    if is_py35:
+        # in python 3.5 async and await are not proper keywords, but for
+        # completion pursposes should as as though they are
+        keys = keyword.kwlist + ["async", "await"]
+    else:
+        keys = keyword.kwlist
 else:
     keys = keyword.kwlist + ['None', 'False', 'True']
 
 
-def keywords(string='', pos=(0, 0), all=False):
-    if all:
-        return set([Keyword(k, pos) for k in keys])
+def has_inappropriate_leaf_keyword(pos, module):
+    relevant_errors = filter(
+        lambda error: error.first_pos[0] == pos[0],
+        module.error_statement_stacks)
+
+    for error in relevant_errors:
+        if error.next_token in keys:
+            return True
+
+    return False
+
+
+def completion_names(evaluator, stmt, pos, module):
+    keyword_list = all_keywords(evaluator)
+
+    if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module):
+        keyword_list = filter(
+            lambda keyword: not keyword.only_valid_as_leaf,
+            keyword_list
+        )
+    return [keyword.name for keyword in keyword_list]
+
+
+def all_keywords(evaluator, pos=(0, 0)):
+    return set([Keyword(evaluator, k, pos) for k in keys])
+
+
+def keyword(evaluator, string, pos=(0, 0)):
     if string in keys:
-        return set([Keyword(string, pos)])
-    return set()
+        return Keyword(evaluator, string, pos)
+    else:
+        return None
 
 
-def keyword_names(*args, **kwargs):
-    return [k.name for k in keywords(*args, **kwargs)]
+def get_operator(evaluator, string, pos):
+    return Keyword(evaluator, string, pos)
 
 
-def get_operator(string, pos):
-    return Keyword(string, pos)
+keywords_only_valid_as_leaf = (
+    'continue',
+    'break',
+)
 
 
 class Keyword(object):
-    def __init__(self, name, pos):
+    type = 'completion_keyword'
+
+    def __init__(self, evaluator, name, pos):
         self.name = FakeName(name, self, pos)
         self.start_pos = pos
-        self.parent = compiled.builtin
+        self.parent = evaluator.BUILTINS
 
     def get_parent_until(self):
         return self.parent
 
+    @property
+    def only_valid_as_leaf(self):
+        return self.name.value in keywords_only_valid_as_leaf
+
     @property
     def names(self):
         """ For a `parsing.Name` like comparision """

jedi/cache.py

@@ -14,36 +14,13 @@ there are global variables, which are holding the cache information. Some of
 these variables are being cleaned after every API usage.
 """
 import time
-import os
-import sys
-import json
-import hashlib
-import gc
-import inspect
-import shutil
-import re
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
 
 from jedi import settings
-from jedi import common
-from jedi import debug
+from jedi.parser.utils import parser_cache
+from jedi.parser.utils import underscore_memoization
 
 _time_caches = {}
 
-# for fast_parser, should not be deleted
-parser_cache = {}
-
-
-class ParserCacheItem(object):
-    def __init__(self, parser, change_time=None):
-        self.parser = parser
-        if change_time is None:
-            change_time = time.time()
-        self.change_time = change_time
-
 
 def clear_time_caches(delete_all=False):
     """ Jedi caches many things, that should be completed after each completion
@@ -70,11 +47,12 @@ def clear_time_caches(delete_all=False):
 
 def time_cache(time_add_setting):
     """
-    s
     This decorator works as follows: Call it with a setting and after that
     use the function with a callable that returns the key.
     But: This function is only called if the key is not available. After a
     certain amount of time (`time_add_setting`) the cache is invalid.
+
+    If the given key is None, the function will not be cached.
     """
     def _temp(key_func):
         dct = {}
@@ -99,56 +77,6 @@ def time_cache(time_add_setting):
     return _temp
 
 
-@time_cache("call_signatures_validity")
-def cache_call_signatures(evaluator, call, source, user_pos):
-    """This function calculates the cache key."""
-    index = user_pos[0] - 1
-    lines = common.splitlines(source)
-
-    before_cursor = lines[index][:user_pos[1]]
-    other_lines = lines[call.start_pos[0]:index]
-    whole = '\n'.join(other_lines + [before_cursor])
-    before_bracket = re.match(r'.*\(', whole, re.DOTALL)
-
-    module_path = call.get_parent_until().path
-    yield None if module_path is None else (module_path, before_bracket, call.start_pos)
-    yield evaluator.eval_element(call)
-
-
-def underscore_memoization(func):
-    """
-    Decorator for methods::
-
-        class A(object):
-            def x(self):
-                if self._x:
-                    self._x = 10
-                return self._x
-
-    Becomes::
-
-        class A(object):
-            @underscore_memoization
-            def x(self):
-                return 10
-
-    A now has an attribute ``_x`` written by this decorator.
-    """
-    name = '_' + func.__name__
-
-    def wrapper(self):
-        try:
-            return getattr(self, name)
-        except AttributeError:
-            result = func(self)
-            if inspect.isgenerator(result):
-                result = list(result)
-            setattr(self, name, result)
-            return result
-
-    return wrapper
-
-
 def memoize_method(method):
     """A normal memoize function."""
     def wrapper(self, *args, **kwargs):
@@ -192,6 +120,14 @@ def _invalidate_star_import_cache_module(module, only_main=False):
     else:
         del _time_caches['star_import_cache_validity'][module]
 
+        # This stuff was part of load_parser. However since we're most likely
+        # not going to use star import caching anymore, just ignore it.
+        #else:
+            # In case there is already a module cached and this module
+            # has to be reparsed, we also need to invalidate the import
+            # caches.
+        #    _invalidate_star_import_cache_module(parser_cache_item.parser.module)
+
 
 def invalidate_star_import_cache(path):
     """On success returns True."""
@@ -201,148 +137,3 @@ def invalidate_star_import_cache(path):
         pass
     else:
         _invalidate_star_import_cache_module(parser_cache_item.parser.module)
-
-
-def load_parser(path):
-    """
-    Returns the module or None, if it fails.
-    """
-    p_time = os.path.getmtime(path) if path else None
-    try:
-        parser_cache_item = parser_cache[path]
-        if not path or p_time <= parser_cache_item.change_time:
-            return parser_cache_item.parser
-        else:
-            # In case there is already a module cached and this module
-            # has to be reparsed, we also need to invalidate the import
-            # caches.
-            _invalidate_star_import_cache_module(parser_cache_item.parser.module)
-    except KeyError:
-        if settings.use_filesystem_cache:
-            return ParserPickling.load_parser(path, p_time)
-
-
-def save_parser(path, parser, pickling=True):
-    try:
-        p_time = None if path is None else os.path.getmtime(path)
-    except OSError:
-        p_time = None
-        pickling = False
-
-    item = ParserCacheItem(parser, p_time)
-    parser_cache[path] = item
-    if settings.use_filesystem_cache and pickling:
-        ParserPickling.save_parser(path, item)
-
-
-class ParserPickling(object):
-
-    version = 24
-    """
-    Version number (integer) for file system cache.
-
-    Increment this number when there are any incompatible changes in
-    parser representation classes.  For example, the following changes
-    are regarded as incompatible.
-
-    - Class name is changed.
-    - Class is moved to another module.
-    - Defined slot of the class is changed.
-    """
-
-    def __init__(self):
-        self.__index = None
-        self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
-        """
-        Short name for distinguish Python implementations and versions.
-
-        It's like `sys.implementation.cache_tag` but for Python < 3.3
-        we generate something similar.  See:
-        http://docs.python.org/3/library/sys.html#sys.implementation
-
-        .. todo:: Detect interpreter (e.g., PyPy).
-        """
-
-    def load_parser(self, path, original_changed_time):
-        try:
-            pickle_changed_time = self._index[path]
-        except KeyError:
-            return None
-        if original_changed_time is not None \
-                and pickle_changed_time < original_changed_time:
-            # the pickle file is outdated
-            return None
-
-        with open(self._get_hashed_path(path), 'rb') as f:
-            try:
-                gc.disable()
-                parser_cache_item = pickle.load(f)
-            finally:
-                gc.enable()
-
-        debug.dbg('pickle loaded: %s', path)
-        parser_cache[path] = parser_cache_item
-        return parser_cache_item.parser
-
-    def save_parser(self, path, parser_cache_item):
-        self.__index = None
-        try:
-            files = self._index
-        except KeyError:
-            files = {}
-            self._index = files
-
-        with open(self._get_hashed_path(path), 'wb') as f:
-            pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
-            files[path] = parser_cache_item.change_time
-
-        self._flush_index()
-
-    @property
-    def _index(self):
-        if self.__index is None:
-            try:
-                with open(self._get_path('index.json')) as f:
-                    data = json.load(f)
-            except (IOError, ValueError):
-                self.__index = {}
-            else:
-                # 0 means version is not defined (= always delete cache):
-                if data.get('version', 0) != self.version:
-                    self.clear_cache()
-                    self.__index = {}
-                else:
-                    self.__index = data['index']
-        return self.__index
-
-    def _remove_old_modules(self):
-        # TODO use
-        change = False
-        if change:
-            self._flush_index(self)
-            self._index  # reload index
-
-    def _flush_index(self):
-        data = {'version': self.version, 'index': self._index}
-        with open(self._get_path('index.json'), 'w') as f:
-            json.dump(data, f)
-        self.__index = None
-
-    def clear_cache(self):
-        shutil.rmtree(self._cache_directory())
-
-    def _get_hashed_path(self, path):
-        return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
-
-    def _get_path(self, file):
-        dir = self._cache_directory()
-        if not os.path.exists(dir):
-            os.makedirs(dir)
-        return os.path.join(dir, file)
-
-    def _cache_directory(self):
-        return os.path.join(settings.cache_directory, self.py_tag)
-
-
-# is a singleton
-ParserPickling = ParserPickling()

jedi/common.py

@@ -3,6 +3,7 @@ import sys
 import contextlib
 import functools
 import re
+from itertools import chain
 from ast import literal_eval
 
 from jedi._compatibility import unicode, reraise
@@ -134,7 +135,7 @@ def source_to_unicode(source, encoding=None):
             return possible_encoding.group(1)
         else:
             # the default if nothing else has been set -> PEP 263
-            return encoding if encoding is not None else 'iso-8859-1'
+            return encoding if encoding is not None else 'utf-8'
 
     if isinstance(source, unicode):
         # only cast str/bytes
@@ -152,3 +153,8 @@ def splitlines(string):
     Also different: Returns ``['']`` for an empty string input.
     """
     return re.split('\n|\r\n', string)
+
+
+def unite(iterable):
+    """Turns a two dimensional array into a one dimensional."""
+    return set(chain.from_iterable(iterable))

jedi/debug.py

@@ -3,23 +3,52 @@ import inspect
 import os
 import time
 
+def _lazy_colorama_init():
+    """
+    Lazily init colorama if necessary, not to screw up stdout is debug not
+    enabled.
+
+    This version of the function does nothing.
+    """
+    pass
+
+_inited=False
+
 try:
     if os.name == 'nt':
-        # does not work on Windows, as pyreadline and colorama interfere
+        # Does not work on Windows, as pyreadline and colorama interfere
         raise ImportError
     else:
         # Use colorama for nicer console output.
         from colorama import Fore, init
         from colorama import initialise
-        # pytest resets the stream at the end - causes troubles. Since after
-        # every output the stream is reset automatically we don't need this.
-        initialise.atexit_done = True
-        init()
+        def _lazy_colorama_init():
+            """
+            Lazily init colorama if necessary, not to screw up stdout is
+            debug not enabled.
+
+            This version of the function does init colorama.
+            """
+            global _inited
+            if not _inited:
+                # pytest resets the stream at the end - causes troubles. Since
+                # after every output the stream is reset automatically we don't
+                # need this.
+                initialise.atexit_done = True
+                try:
+                    init()
+                except Exception:
+                    # Colorama fails with initializing under vim and is buggy in
+                    # version 0.3.6.
+                    pass
+            _inited = True
+
 except ImportError:
     class Fore(object):
         RED = ''
         GREEN = ''
         YELLOW = ''
+        MAGENTA = ''
         RESET = ''
 
 NOTICE = object()
@@ -32,15 +61,15 @@ enable_notice = False
 
 # callback, interface: level, str
 debug_function = None
-ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser']
-_debug_indent = -1
+ignored_modules = ['jedi.parser']
+_debug_indent = 0
 _start_time = time.time()
 
 
 def reset_time():
     global _start_time, _debug_indent
     _start_time = time.time()
-    _debug_indent = -1
+    _debug_indent = 0
 
 
 def increase_indent(func):
@@ -49,44 +78,53 @@ def increase_indent(func):
         global _debug_indent
         _debug_indent += 1
         try:
-            result = func(*args, **kwargs)
+            return func(*args, **kwargs)
         finally:
             _debug_indent -= 1
-        return result
     return wrapper
 
 
-def dbg(message, *args):
+def dbg(message, *args, **kwargs):
     """ Looks at the stack, to see if a debug message should be printed. """
+    if kwargs:
+        # Python 2 compatibility, because it doesn't understand default args
+        # after *args.
+        color = kwargs.get('color')
+        if color is None:
+            raise TypeError("debug.dbg doesn't support more named arguments than color")
+    else:
+        color = 'GREEN'
+
     if debug_function and enable_notice:
         frm = inspect.stack()[1]
         mod = inspect.getmodule(frm[0])
         if not (mod.__name__ in ignored_modules):
             i = ' ' * _debug_indent
-            debug_function(NOTICE, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
+            _lazy_colorama_init()
+            debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
 
 
 def warning(message, *args):
     if debug_function and enable_warning:
         i = ' ' * _debug_indent
-        debug_function(WARNING, i + 'warning: ' + message % tuple(u(repr(a)) for a in args))
+        debug_function('RED', i + 'warning: ' + message % tuple(u(repr(a)) for a in args))
 
 
 def speed(name):
     if debug_function and enable_speed:
         now = time.time()
         i = ' ' * _debug_indent
-        debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - _start_time))
+        debug_function('YELLOW', i + 'speed: ' + '%s %s' % (name, now - _start_time))
 
 
-def print_to_stdout(level, str_out):
-    """ The default debug function """
-    if level == NOTICE:
-        col = Fore.GREEN
-    elif level == WARNING:
-        col = Fore.RED
-    else:
-        col = Fore.YELLOW
+def print_to_stdout(color, str_out):
+    """
+    The default debug function that prints to standard out.
+
+    :param str color: A string that is an attribute of ``colorama.Fore``.
+    """
+    col = getattr(Fore, color)
+    _lazy_colorama_init()
     if not is_py3:
         str_out = str_out.encode(encoding, 'replace')
     print(col + str_out + Fore.RESET)

jedi/evaluate/__init__.py

@@ -85,10 +85,13 @@ class Evaluator(object):
         self.memoize_cache = {}  # for memoize decorators
         # To memorize modules -> equals `sys.modules`.
         self.modules = {}  # like `sys.modules`.
-        self.compiled_cache = {}  # see `compiled.create()`
-        self.recursion_detector = recursion.RecursionDetector()
-        self.execution_recursion_detector = recursion.ExecutionRecursionDetector()
+        self.compiled_cache = {}  # see `evaluate.compiled.create()`
+        self.mixed_cache = {}  # see `evaluate.compiled.mixed.create()`
         self.analysis = []
+        self.predefined_if_name_dict_dict = {}
+        self.dynamic_params_depth = 0
+        self.is_analysis = False
+
         if sys_path is None:
             sys_path = sys.path
         self.sys_path = copy.copy(sys_path)
@@ -97,16 +100,28 @@ class Evaluator(object):
         except ValueError:
             pass
 
+        self.reset_recursion_limitations()
+
+        # Constants
+        self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
+
+    def reset_recursion_limitations(self):
+        self.recursion_detector = recursion.RecursionDetector(self)
+        self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
+
     def wrap(self, element):
-        if isinstance(element, tree.Class):
+        if isinstance(element, (er.Wrapper, er.InstanceElement,
+            er.ModuleWrapper, er.FunctionExecution, er.Instance, compiled.CompiledObject)) or element is None:
+            # TODO this is so ugly, please refactor.
+            return element
+
+        if element.type == 'classdef':
             return er.Class(self, element)
-        elif isinstance(element, tree.Function):
-            if isinstance(element, tree.Lambda):
-                return er.LambdaWrapper(self, element)
-            else:
-                return er.Function(self, element)
-        elif isinstance(element, (tree.Module)) \
-                and not isinstance(element, er.ModuleWrapper):
+        elif element.type == 'funcdef':
+            return er.Function(self, element)
+        elif element.type == 'lambda':
+            return er.LambdaWrapper(self, element)
+        elif element.type == 'file_input':
             return er.ModuleWrapper(self, element)
         else:
             return element
@@ -125,10 +140,10 @@ class Evaluator(object):
         scopes = f.scopes(search_global)
         if is_goto:
             return f.filter_name(scopes)
-        return f.find(scopes, search_global)
+        return f.find(scopes, attribute_lookup=not search_global)
 
-    @memoize_default(default=[], evaluator_is_first_arg=True)
-    @recursion.recursion_decorator
+    #@memoize_default(default=[], evaluator_is_first_arg=True)
+    #@recursion.recursion_decorator
     @debug.increase_indent
     def eval_statement(self, stmt, seek_name=None):
         """
@@ -140,10 +155,11 @@ class Evaluator(object):
         :param stmt: A `tree.ExprStmt`.
         """
         debug.dbg('eval_statement %s (%s)', stmt, seek_name)
-        types = self.eval_element(stmt.get_rhs())
+        rhs = stmt.get_rhs()
+        types = self.eval_element(rhs)
 
         if seek_name:
-            types = finder.check_tuple_assignments(types, seek_name)
+            types = finder.check_tuple_assignments(self, types, seek_name)
 
         first_operation = stmt.first_operation()
         if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement):  # TODO don't check for this.
@@ -153,71 +169,168 @@ class Evaluator(object):
             name = str(stmt.get_defined_names()[0])
             parent = self.wrap(stmt.get_parent_scope())
             left = self.find_types(parent, name, stmt.start_pos, search_global=True)
-            if isinstance(stmt.get_parent_until(tree.ForStmt), tree.ForStmt):
+
+            for_stmt = stmt.get_parent_until(tree.ForStmt)
+            if isinstance(for_stmt, tree.ForStmt) and types \
+                    and for_stmt.defines_one_name():
                 # Iterate through result and add the values, that's possible
                 # only in for loops without clutter, because they are
-                # predictable.
-                for r in types:
-                    left = precedence.calculate(self, left, operator, [r])
+                # predictable. Also only do it, if the variable is not a tuple.
+                node = for_stmt.get_input_node()
+                for_iterables = self.eval_element(node)
+                ordered = list(iterable.py__iter__(self, for_iterables, node))
+
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    self.predefined_if_name_dict_dict[for_stmt] = dct
+                    t = self.eval_element(rhs)
+                    left = precedence.calculate(self, left, operator, t)
                 types = left
+                if ordered:
+                    # If there are no for entries, we cannot iterate and the
+                    # types are defined by += entries. Therefore the for loop
+                    # is never called.
+                    del self.predefined_if_name_dict_dict[for_stmt]
             else:
                 types = precedence.calculate(self, left, operator, types)
         debug.dbg('eval_statement result %s', types)
         return types
 
-    @memoize_default(evaluator_is_first_arg=True)
     def eval_element(self, element):
         if isinstance(element, iterable.AlreadyEvaluated):
-            return list(element)
+            return set(element)
         elif isinstance(element, iterable.MergedNodes):
             return iterable.unite(self.eval_element(e) for e in element)
 
+        if_stmt = element.get_parent_until((tree.IfStmt, tree.ForStmt, tree.IsScope))
+        predefined_if_name_dict = self.predefined_if_name_dict_dict.get(if_stmt)
+        if predefined_if_name_dict is None and isinstance(if_stmt, tree.IfStmt):
+            if_stmt_test = if_stmt.children[1]
+            name_dicts = [{}]
+            # If we already did a check, we don't want to do it again -> If
+            # predefined_if_name_dict_dict is filled, we stop.
+            # We don't want to check the if stmt itself, it's just about
+            # the content.
+            if element.start_pos > if_stmt_test.end_pos:
+                # Now we need to check if the names in the if_stmt match the
+                # names in the suite.
+                if_names = helpers.get_names_of_node(if_stmt_test)
+                element_names = helpers.get_names_of_node(element)
+                str_element_names = [str(e) for e in element_names]
+                if any(str(i) in str_element_names for i in if_names):
+                    for if_name in if_names:
+                        definitions = self.goto_definitions(if_name)
+                        # Every name that has multiple different definitions
+                        # causes the complexity to rise. The complexity should
+                        # never fall below 1.
+                        if len(definitions) > 1:
+                            if len(name_dicts) * len(definitions) > 16:
+                                debug.dbg('Too many options for if branch evaluation %s.', if_stmt)
+                                # There's only a certain amount of branches
+                                # Jedi can evaluate, otherwise it will take to
+                                # long.
+                                name_dicts = [{}]
+                                break
+
+                            original_name_dicts = list(name_dicts)
+                            name_dicts = []
+                            for definition in definitions:
+                                new_name_dicts = list(original_name_dicts)
+                                for i, name_dict in enumerate(new_name_dicts):
+                                    new_name_dicts[i] = name_dict.copy()
+                                    new_name_dicts[i][str(if_name)] = [definition]
+
+                                name_dicts += new_name_dicts
+                        else:
+                            for name_dict in name_dicts:
+                                name_dict[str(if_name)] = definitions
+            if len(name_dicts) > 1:
+                result = set()
+                for name_dict in name_dicts:
+                    self.predefined_if_name_dict_dict[if_stmt] = name_dict
+                    try:
+                        result |= self._eval_element_not_cached(element)
+                    finally:
+                        del self.predefined_if_name_dict_dict[if_stmt]
+                return result
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+        else:
+            if predefined_if_name_dict:
+                return self._eval_element_not_cached(element)
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+
+    def _eval_element_if_evaluated(self, element):
+        """
+        TODO This function is temporary: Merge with eval_element.
+        """
+        parent = element
+        while parent is not None:
+            parent = parent.parent
+            predefined_if_name_dict = self.predefined_if_name_dict_dict.get(parent)
+            if predefined_if_name_dict is not None:
+                return self._eval_element_not_cached(element)
+        return self._eval_element_cached(element)
+
+    @memoize_default(evaluator_is_first_arg=True)
+    def _eval_element_cached(self, element):
+        return self._eval_element_not_cached(element)
+
+    @debug.increase_indent
+    def _eval_element_not_cached(self, element):
         debug.dbg('eval_element %s@%s', element, element.start_pos)
+        types = set()
         if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
-            return self._eval_atom(element)
+            types = self._eval_atom(element)
         elif isinstance(element, tree.Keyword):
             # For False/True/None
             if element.value in ('False', 'True', 'None'):
-                return [compiled.builtin.get_by_name(element.value)]
-            else:
-                return []
+                types.add(compiled.builtin_from_name(self, element.value))
+            # else: print e.g. could be evaluated like this in Python 2.7
         elif element.isinstance(tree.Lambda):
-            return [er.LambdaWrapper(self, element)]
+            types = set([er.LambdaWrapper(self, element)])
         elif element.isinstance(er.LambdaWrapper):
-            return [element]  # TODO this is no real evaluation.
+            types = set([element])  # TODO this is no real evaluation.
         elif element.type == 'expr_stmt':
-            return self.eval_statement(element)
-        elif element.type == 'power':
+            types = self.eval_statement(element)
+        elif element.type in ('power', 'atom_expr'):
             types = self._eval_atom(element.children[0])
             for trailer in element.children[1:]:
                 if trailer == '**':  # has a power operation.
-                    raise NotImplementedError
+                    right = self.eval_element(element.children[2])
+                    types = set(precedence.calculate(self, types, trailer, right))
+                    break
                 types = self.eval_trailer(types, trailer)
-
-            return types
         elif element.type in ('testlist_star_expr', 'testlist',):
             # The implicit tuple in statements.
-            return [iterable.ImplicitTuple(self, element)]
+            types = set([iterable.ImplicitTuple(self, element)])
         elif element.type in ('not_test', 'factor'):
             types = self.eval_element(element.children[-1])
             for operator in element.children[:-1]:
-                types = list(precedence.factor_calculate(self, types, operator))
-            return types
+                types = set(precedence.factor_calculate(self, types, operator))
         elif element.type == 'test':
             # `x if foo else y` case.
-            return (self.eval_element(element.children[0]) +
-                    self.eval_element(element.children[-1]))
+            types = (self.eval_element(element.children[0]) |
+                     self.eval_element(element.children[-1]))
         elif element.type == 'operator':
             # Must be an ellipsis, other operators are not evaluated.
-            return []  # Ignore for now.
+            assert element.value == '...'
+            types = set([compiled.create(self, Ellipsis)])
         elif element.type == 'dotted_name':
             types = self._eval_atom(element.children[0])
             for next_name in element.children[2::2]:
-                types = list(chain.from_iterable(self.find_types(typ, next_name)
-                                                 for typ in types))
-            return types
+                types = set(chain.from_iterable(self.find_types(typ, next_name)
+                                                for typ in types))
+            types = types
+        elif element.type == 'eval_input':
+            types = self._eval_element_not_cached(element.children[0])
         else:
-            return precedence.calculate_children(self, element.children)
+            types = precedence.calculate_children(self, element.children)
+        debug.dbg('eval_element result %s', types)
+        return types
 
     def _eval_atom(self, atom):
         """
@@ -229,6 +342,13 @@ class Evaluator(object):
             # This is the first global lookup.
             stmt = atom.get_definition()
             scope = stmt.get_parent_until(tree.IsScope, include_current=True)
+            if isinstance(scope, (tree.Function, er.FunctionExecution)):
+                # Adjust scope: If the name is not in the suite, it's a param
+                # default or annotation and will be resolved as part of the
+                # parent scope.
+                colon = scope.children.index(':')
+                if atom.start_pos < scope.children[colon + 1].start_pos:
+                    scope = scope.get_parent_scope()
             if isinstance(stmt, tree.CompFor):
                 stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
             if stmt.type != 'expr_stmt':
@@ -237,42 +357,53 @@ class Evaluator(object):
                 stmt = atom
             return self.find_types(scope, atom, stmt.start_pos, search_global=True)
         elif isinstance(atom, tree.Literal):
-            return [compiled.create(self, atom.eval())]
+            return set([compiled.create(self, atom.eval())])
         else:
             c = atom.children
+            if c[0].type == 'string':
+                # Will be one string.
+                types = self._eval_atom(c[0])
+                for string in c[1:]:
+                    right = self._eval_atom(string)
+                    types = precedence.calculate(self, types, '+', right)
+                return types
             # Parentheses without commas are not tuples.
-            if c[0] == '(' and not len(c) == 2 \
+            elif c[0] == '(' and not len(c) == 2 \
                     and not(tree.is_node(c[1], 'testlist_comp')
                             and len(c[1].children) > 1):
                 return self.eval_element(c[1])
+
             try:
                 comp_for = c[1].children[1]
             except (IndexError, AttributeError):
                 pass
             else:
-                if isinstance(comp_for, tree.CompFor):
-                    return [iterable.Comprehension.from_atom(self, atom)]
-            return [iterable.Array(self, atom)]
+                if comp_for == ':':
+                    # Dict comprehensions have a colon at the 3rd index.
+                    try:
+                        comp_for = c[1].children[3]
+                    except IndexError:
+                        pass
+
+                if comp_for.type == 'comp_for':
+                    return set([iterable.Comprehension.from_atom(self, atom)])
+            return set([iterable.Array(self, atom)])
 
     def eval_trailer(self, types, trailer):
         trailer_op, node = trailer.children[:2]
         if node == ')':  # `arglist` is optional.
             node = ()
-        new_types = []
-        for typ in types:
-            debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
-            if trailer_op == '.':
-                new_types += self.find_types(typ, node)
-            elif trailer_op == '(':
-                new_types += self.execute(typ, node, trailer)
-            elif trailer_op == '[':
-                try:
-                    get = typ.get_index_types
-                except AttributeError:
-                    debug.warning("TypeError: '%s' object is not subscriptable"
-                                  % typ)
-                else:
-                    new_types += get(self, node)
+
+        new_types = set()
+        if trailer_op == '[':
+            new_types |= iterable.py__getitem__(self, types, trailer)
+        else:
+            for typ in types:
+                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
+                if trailer_op == '.':
+                    new_types |= self.find_types(typ, node)
+                elif trailer_op == '(':
+                    new_types |= self.execute(typ, node, trailer)
         return new_types
 
     def execute_evaluated(self, obj, *args):
@@ -287,6 +418,9 @@ class Evaluator(object):
         if not isinstance(arguments, param.Arguments):
             arguments = param.Arguments(self, arguments, trailer)
 
+        if self.is_analysis:
+            arguments.eval_all()
+
         if obj.isinstance(er.Function):
             obj = obj.get_decorated_func()
 
@@ -302,17 +436,28 @@ class Evaluator(object):
             func = obj.py__call__
         except AttributeError:
             debug.warning("no execution possible %s", obj)
-            return []
+            return set()
         else:
-            types = func(self, arguments)
+            types = func(arguments)
             debug.dbg('execute result: %s in %s', types, obj)
             return types
 
-    def goto_definition(self, name):
+    def goto_definitions(self, name):
         def_ = name.get_definition()
-        if def_.type == 'expr_stmt' and name in def_.get_defined_names():
-            return self.eval_statement(def_, name)
-        call = helpers.call_of_name(name)
+        is_simple_name = name.parent.type not in ('power', 'trailer')
+        if is_simple_name:
+            if name.parent.type in ('file_input', 'classdef', 'funcdef'):
+                return [self.wrap(name.parent)]
+            if def_.type == 'expr_stmt' and name in def_.get_defined_names():
+                return self.eval_statement(def_, name)
+            elif def_.type == 'for_stmt':
+                container_types = self.eval_element(def_.children[3])
+                for_types = iterable.py__iter__types(self, container_types, def_.children[3])
+                return finder.check_tuple_assignments(self, for_types, name)
+            elif def_.type in ('import_from', 'import_name'):
+                return imports.ImportWrapper(self, name).follow()
+
+        call = helpers.call_of_leaf(name)
         return self.eval_element(call)
 
     def goto(self, name):
@@ -372,8 +517,8 @@ class Evaluator(object):
                 ))
 
         scope = name.get_parent_scope()
-        if tree.is_node(name.parent, 'trailer'):
-            call = helpers.call_of_name(name, cut_own_trailer=True)
+        if tree.is_node(par, 'trailer') and par.children[0] == '.':
+            call = helpers.call_of_leaf(name, cut_own_trailer=True)
             types = self.eval_element(call)
             return resolve_implicit_imports(iterable.unite(
                 self.find_types(typ, name, is_goto=True) for typ in types

jedi/evaluate/analysis.py

@@ -5,19 +5,25 @@ from jedi import debug
 from jedi.parser import tree
 from jedi.evaluate.compiled import CompiledObject
 
+from jedi.common import unite
+
 
 CODES = {
     'attribute-error': (1, AttributeError, 'Potential AttributeError.'),
     'name-error': (2, NameError, 'Potential NameError.'),
     'import-error': (3, ImportError, 'Potential ImportError.'),
-    'type-error-generator': (4, TypeError, "TypeError: 'generator' object is not subscriptable."),
-    'type-error-too-many-arguments': (5, TypeError, None),
-    'type-error-too-few-arguments': (6, TypeError, None),
-    'type-error-keyword-argument': (7, TypeError, None),
-    'type-error-multiple-values': (8, TypeError, None),
-    'type-error-star-star': (9, TypeError, None),
-    'type-error-star': (10, TypeError, None),
-    'type-error-operation': (11, TypeError, None),
+    'type-error-too-many-arguments': (4, TypeError, None),
+    'type-error-too-few-arguments': (5, TypeError, None),
+    'type-error-keyword-argument': (6, TypeError, None),
+    'type-error-multiple-values': (7, TypeError, None),
+    'type-error-star-star': (8, TypeError, None),
+    'type-error-star': (9, TypeError, None),
+    'type-error-operation': (10, TypeError, None),
+    'type-error-not-iterable': (11, TypeError, None),
+    'type-error-isinstance': (12, TypeError, None),
+    'type-error-not-subscriptable': (13, TypeError, None),
+    'value-error-too-many-values': (14, ValueError, None),
+    'value-error-too-few-values': (15, ValueError, None),
 }
 
 
@@ -158,8 +164,8 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
                     from jedi.evaluate import iterable
                     if isinstance(cls, iterable.Array) and cls.type == 'tuple':
                         # multiple exceptions
-                        for c in cls.values():
-                            if check_match(c, exception):
+                        for typ in unite(cls.py__iter__()):
+                            if check_match(typ, exception):
                                 return True
                     else:
                         if check_match(cls, exception):
@@ -168,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
     def check_hasattr(node, suite):
         try:
             assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos
-            assert node.type == 'power'
+            assert node.type in ('power', 'atom_expr')
             base = node.children[0]
             assert base.type == 'name' and base.value == 'hasattr'
             trailer = node.children[1]
@@ -183,7 +189,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
             # Check name
             key, values = args[1]
             assert len(values) == 1
-            names = evaluator.eval_element(values[0])
+            names = list(evaluator.eval_element(values[0]))
             assert len(names) == 1 and isinstance(names[0], CompiledObject)
             assert names[0].obj == str(payload[1])
 
@@ -208,95 +214,3 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
         obj = obj.parent
 
     return False
-
-
-def get_module_statements(module):
-    """
-    Returns the statements used in a module. All these statements should be
-    evaluated to check for potential exceptions.
-    """
-    def check_children(node):
-        try:
-            children = node.children
-        except AttributeError:
-            return []
-        else:
-            nodes = []
-            for child in children:
-                nodes += check_children(child)
-                if child.type == 'trailer':
-                    c = child.children
-                    if c[0] == '(' and c[1] != ')':
-                        if c[1].type != 'arglist':
-                            if c[1].type == 'argument':
-                                nodes.append(c[1].children[-1])
-                            else:
-                                nodes.append(c[1])
-                        else:
-                            for argument in c[1].children:
-                                if argument.type == 'argument':
-                                    nodes.append(argument.children[-1])
-                                elif argument.type != 'operator':
-                                    nodes.append(argument)
-            return nodes
-
-    def add_nodes(nodes):
-        new = set()
-        for node in nodes:
-            if isinstance(node, tree.Flow):
-                children = node.children
-                if node.type == 'for_stmt':
-                    children = children[2:]  # Don't want to include the names.
-                # Pick the suite/simple_stmt.
-                new |= add_nodes(children)
-            elif node.type in ('simple_stmt', 'suite'):
-                new |= add_nodes(node.children)
-            elif node.type in ('return_stmt', 'yield_expr'):
-                try:
-                    new.add(node.children[1])
-                except IndexError:
-                    pass
-            elif node.type not in ('whitespace', 'operator', 'keyword',
-                                   'parameters', 'decorated', 'except_clause') \
-                    and not isinstance(node, (tree.ClassOrFunc, tree.Import)):
-                new.add(node)
-
-                try:
-                    children = node.children
-                except AttributeError:
-                    pass
-                else:
-                    for next_node in children:
-                        new.update(check_children(node))
-                        if next_node.type != 'keyword' and node.type != 'expr_stmt':
-                            new.add(node)
-        return new
-
-    nodes = set()
-    import_names = set()
-    decorated_funcs = []
-    for scope in module.walk():
-        for imp in set(scope.imports):
-            import_names |= set(imp.get_defined_names())
-            if imp.is_nested():
-                import_names |= set(path[-1] for path in imp.paths())
-
-        children = scope.children
-        if isinstance(scope, tree.ClassOrFunc):
-            children = children[2:]  # We don't want to include the class name.
-        nodes |= add_nodes(children)
-
-        for flow in scope.flows:
-            if flow.type == 'for_stmt':
-                nodes.add(flow.children[3])
-            elif flow.type == 'try_stmt':
-                nodes.update(e for e in flow.except_clauses() if e is not None)
-
-        try:
-            decorators = scope.get_decorators()
-        except AttributeError:
-            pass
-        else:
-            if decorators:
-                decorated_funcs.append(scope)
-    return nodes, import_names, decorated_funcs

jedi/evaluate/compiled/__init__.py

@@ -41,34 +41,30 @@ class CompiledObject(Base):
     path = None  # modules have this attribute - set it to None.
     used_names = {}  # To be consistent with modules.
 
-    def __init__(self, obj, parent=None):
+    def __init__(self, evaluator, obj, parent=None):
+        self._evaluator = evaluator
         self.obj = obj
         self.parent = parent
 
-    @property
-    def py__call__(self):
-        def actual(evaluator, params):
-            if inspect.isclass(self.obj):
-                from jedi.evaluate.representation import Instance
-                return [Instance(evaluator, self, params)]
-            else:
-                return list(self._execute_function(evaluator, params))
-
-        # Might raise an AttributeError, which is intentional.
-        self.obj.__call__
-        return actual
+    @CheckAttribute
+    def py__call__(self, params):
+        if inspect.isclass(self.obj):
+            from jedi.evaluate.representation import Instance
+            return set([Instance(self._evaluator, self, params)])
+        else:
+            return set(self._execute_function(params))
 
     @CheckAttribute
-    def py__class__(self, evaluator):
-        return CompiledObject(self.obj.__class__, parent=self.parent)
+    def py__class__(self):
+        return create(self._evaluator, self.obj.__class__)
 
     @CheckAttribute
-    def py__mro__(self, evaluator):
-        return tuple(create(evaluator, cls, self.parent) for cls in self.obj.__mro__)
+    def py__mro__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__)
 
     @CheckAttribute
-    def py__bases__(self, evaluator):
-        return tuple(create(evaluator, cls) for cls in self.obj.__bases__)
+    def py__bases__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__)
 
     def py__bool__(self):
         return bool(self.obj)
@@ -87,7 +83,7 @@ class CompiledObject(Base):
     def params(self):
         params_str, ret = self._parse_function_doc()
         tokens = params_str.split(',')
-        if inspect.ismethoddescriptor(self._cls().obj):
+        if inspect.ismethoddescriptor(self.obj):
             tokens.insert(0, 'self')
         params = []
         for p in tokens:
@@ -108,22 +104,21 @@ class CompiledObject(Base):
         return _parse_function_doc(self.doc)
 
     def api_type(self):
-        if fake.is_class_instance(self.obj):
-            return 'instance'
-
-        cls = self._cls().obj
-        if inspect.isclass(cls):
+        obj = self.obj
+        if inspect.isclass(obj):
             return 'class'
-        elif inspect.ismodule(cls):
+        elif inspect.ismodule(obj):
             return 'module'
-        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \
-                or inspect.ismethoddescriptor(cls):
+        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
             return 'function'
+        # Everything else...
+        return 'instance'
 
     @property
     def type(self):
         """Imitate the tree.Node.type values."""
-        cls = self._cls().obj
+        cls = self._get_class()
         if inspect.isclass(cls):
             return 'classdef'
         elif inspect.ismodule(cls):
@@ -134,17 +129,24 @@ class CompiledObject(Base):
 
     @underscore_memoization
     def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
         # Ensures that a CompiledObject is returned that is not an instance (like list)
-        if fake.is_class_instance(self.obj):
-            try:
-                c = self.obj.__class__
-            except AttributeError:
-                # happens with numpy.core.umath._UFUNC_API (you get it
-                # automatically by doing `import numpy`.
-                c = type(None)
-            return CompiledObject(c, self.parent)
         return self
 
+    def _get_class(self):
+        if not fake.is_class_instance(self.obj):
+            return self.obj
+
+        try:
+            return self.obj.__class__
+        except AttributeError:
+            # happens with numpy.core.umath._UFUNC_API (you get it
+            # automatically by doing `import numpy`.
+            return type
+
     @property
     def names_dict(self):
         # For compatibility with `representation.Class`.
@@ -159,64 +161,55 @@ class CompiledObject(Base):
         search_global shouldn't change the fact that there's one dict, this way
         there's only one `object`.
         """
-        return [LazyNamesDict(self._cls(), is_instance)]
+        return [LazyNamesDict(self._evaluator, self, is_instance)]
 
     def get_subscope_by_name(self, name):
-        if name in dir(self._cls().obj):
-            return CompiledName(self._cls(), name).parent
+        if name in dir(self.obj):
+            return CompiledName(self._evaluator, self, name).parent
         else:
             raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
 
-    def get_index_types(self, evaluator, index_array=()):
-        # If the object doesn't have `__getitem__`, just raise the
-        # AttributeError.
-        if not hasattr(self.obj, '__getitem__'):
-            debug.warning('Tried to call __getitem__ on non-iterable.')
-            return []
+    @CheckAttribute
+    def py__getitem__(self, index):
         if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
             # Get rid of side effects, we won't call custom `__getitem__`s.
-            return []
+            return set()
 
-        result = []
-        from jedi.evaluate.iterable import create_indexes_or_slices
-        for typ in create_indexes_or_slices(evaluator, index_array):
-            index = None
-            try:
-                index = typ.obj
-                new = self.obj[index]
-            except (KeyError, IndexError, TypeError, AttributeError):
-                # Just try, we don't care if it fails, except for slices.
-                if isinstance(index, slice):
-                    result.append(self)
-            else:
-                result.append(CompiledObject(new))
-        if not result:
-            try:
-                for obj in self.obj:
-                    result.append(CompiledObject(obj))
-            except TypeError:
-                pass  # self.obj maynot have an __iter__ method.
-        return result
+        return set([create(self._evaluator, self.obj[index])])
+
+    @CheckAttribute
+    def py__iter__(self):
+        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return
+
+        for part in self.obj:
+            yield set([create(self._evaluator, part)])
 
     @property
     def name(self):
-        # might not exist sometimes (raises AttributeError)
-        return FakeName(self._cls().obj.__name__, self)
+        try:
+            name = self._get_class().__name__
+        except AttributeError:
+            name = repr(self.obj)
+        return FakeName(name, self)
 
-    def _execute_function(self, evaluator, params):
+    def _execute_function(self, params):
         if self.type != 'funcdef':
             return
 
         for name in self._parse_function_doc()[1].split():
             try:
-                bltn_obj = _create_from_name(builtin, builtin, name)
+                bltn_obj = getattr(_builtins, name)
             except AttributeError:
                 continue
             else:
-                if isinstance(bltn_obj, CompiledObject) and bltn_obj.obj is None:
-                    # We want everything except None.
+                if bltn_obj is None:
+                    # We want to evaluate everything except None.
+                    # TODO do we?
                     continue
-                for result in evaluator.execute(bltn_obj, params):
+                bltn_obj = create(self._evaluator, bltn_obj)
+                for result in self._evaluator.execute(bltn_obj, params):
                     yield result
 
     @property
@@ -228,7 +221,7 @@ class CompiledObject(Base):
         """
         module = self.get_parent_until()
         faked_subscopes = []
-        for name in dir(self._cls().obj):
+        for name in dir(self.obj):
             f = fake.get_faked(module.obj, self.obj, name)
             if f:
                 f.parent = self
@@ -245,11 +238,42 @@ class CompiledObject(Base):
         return []  # Builtins don't have imports
 
 
+class CompiledName(FakeName):
+    def __init__(self, evaluator, compiled_obj, name):
+        super(CompiledName, self).__init__(name)
+        self._evaluator = evaluator
+        self._compiled_obj = compiled_obj
+        self.name = name
+
+    def __repr__(self):
+        try:
+            name = self._compiled_obj.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
+
+    def is_definition(self):
+        return True
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        module = self._compiled_obj.get_parent_until()
+        return _create_from_name(self._evaluator, module, self._compiled_obj, self.name)
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
+
+
 class LazyNamesDict(object):
     """
     A names_dict instance for compiled objects, resembles the parser.tree.
     """
-    def __init__(self, compiled_obj, is_instance):
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_obj, is_instance=False):
+        self._evaluator = evaluator
         self._compiled_obj = compiled_obj
         self._is_instance = is_instance
 
@@ -262,7 +286,12 @@ class LazyNamesDict(object):
             getattr(self._compiled_obj.obj, name)
         except AttributeError:
             raise KeyError('%s in %s not found.' % (name, self._compiled_obj))
-        return [CompiledName(self._compiled_obj, name)]
+        except Exception:
+            # This is a bit ugly. We're basically returning this to make
+            # lookups possible without having the actual attribute. However
+            # this makes proper completion possible.
+            return [FakeName(name, create(self._evaluator, None), is_definition=True)]
+        return [self.name_class(self._evaluator, self._compiled_obj, name)]
 
     def values(self):
         obj = self._compiled_obj.obj
@@ -275,39 +304,13 @@ class LazyNamesDict(object):
                 # The dir function can be wrong.
                 pass
 
-        # dir doesn't include the type names.
-        if not inspect.ismodule(obj) and obj != type and not self._is_instance:
-            values += _type_names_dict.values()
+        is_instance = self._is_instance or fake.is_class_instance(obj)
+        # ``dir`` doesn't include the type names.
+        if not inspect.ismodule(obj) and obj != type and not is_instance:
+            values += create(self._evaluator, type).names_dict.values()
         return values
 
 
-class CompiledName(FakeName):
-    def __init__(self, obj, name):
-        super(CompiledName, self).__init__(name)
-        self._obj = obj
-        self.name = name
-
-    def __repr__(self):
-        try:
-            name = self._obj.name  # __name__ is not defined all the time
-        except AttributeError:
-            name = None
-        return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
-
-    def is_definition(self):
-        return True
-
-    @property
-    @underscore_memoization
-    def parent(self):
-        module = self._obj.get_parent_until()
-        return _create_from_name(module, self._obj, self.name)
-
-    @parent.setter
-    def parent(self, value):
-        pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
-
-
 def dotted_from_fs_path(fs_path, sys_path):
     """
     Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
@@ -369,7 +372,7 @@ def load_module(evaluator, path=None, name=None):
     # complicated import structure of Python.
     module = sys.modules[dotted_path]
 
-    return CompiledObject(module)
+    return create(evaluator, module)
 
 
 docstr_defaults = {
@@ -441,19 +444,7 @@ def _parse_function_doc(doc):
     return param_str, ret
 
 
-class Builtin(CompiledObject):
-    @memoize_method
-    def get_by_name(self, name):
-        return self.names_dict[name][0].parent
-
-
-def _a_generator(foo):
-    """Used to have an object to return for generators."""
-    yield 42
-    yield foo
-
-
-def _create_from_name(module, parent, name):
+def _create_from_name(evaluator, module, parent, name):
     faked = fake.get_faked(module.obj, parent.obj, name)
     # only functions are necessary.
     if faked is not None:
@@ -467,61 +458,78 @@ def _create_from_name(module, parent, name):
         # PyQt4.QtGui.QStyleOptionComboBox.currentText
         # -> just set it to None
         obj = None
-    return CompiledObject(obj, parent)
+    return create(evaluator, obj, parent)
 
 
-builtin = Builtin(_builtins)
-magic_function_class = CompiledObject(type(load_module), parent=builtin)
-generator_obj = CompiledObject(_a_generator(1.0))
-_type_names_dict = builtin.get_by_name('type').names_dict
-none_obj = builtin.get_by_name('None')
-false_obj = builtin.get_by_name('False')
-true_obj = builtin.get_by_name('True')
-object_obj = builtin.get_by_name('object')
+def builtin_from_name(evaluator, string):
+    bltn_obj = getattr(_builtins, string)
+    return create(evaluator, bltn_obj)
 
 
-def keyword_from_value(obj):
-    if obj is None:
-        return none_obj
-    elif obj is False:
-        return false_obj
-    elif obj is True:
-        return true_obj
-    else:
-        raise NotImplementedError
+def _a_generator(foo):
+    """Used to have an object to return for generators."""
+    yield 42
+    yield foo
 
 
-def compiled_objects_cache(func):
-    def wrapper(evaluator, obj, parent=builtin, module=None):
-        # Do a very cheap form of caching here.
-        key = id(obj), id(parent), id(module)
-        try:
-            return evaluator.compiled_cache[key][0]
-        except KeyError:
-            result = func(evaluator, obj, parent, module)
-            # Need to cache all of them, otherwise the id could be overwritten.
-            evaluator.compiled_cache[key] = result, obj, parent, module
-            return result
-    return wrapper
+_SPECIAL_OBJECTS = {
+    'FUNCTION_CLASS': type(load_module),
+    'MODULE_CLASS': type(os),
+    'GENERATOR_OBJECT': _a_generator(1.0),
+    'BUILTINS': _builtins,
+}
 
 
-@compiled_objects_cache
-def create(evaluator, obj, parent=builtin, module=None):
+def get_special_object(evaluator, identifier):
+    obj = _SPECIAL_OBJECTS[identifier]
+    return create(evaluator, obj, parent=create(evaluator, _builtins))
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent=None, module=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj), id(parent)
+            try:
+                return cache[key][0]
+            except KeyError:
+                # TODO this whole decorator looks way too ugly and this if
+                # doesn't make it better. Find a more generic solution.
+                if parent or module:
+                    result = func(evaluator, obj, parent, module)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent, module
+                return result
+        return wrapper
+
+    return decorator
+
+
+@compiled_objects_cache('compiled_cache')
+def create(evaluator, obj, parent=None, module=None):
     """
     A very weird interface class to this module. The more options provided the
     more acurate loading compiled objects is.
     """
+    if inspect.ismodule(obj):
+        if parent is not None:
+            # Modules don't have parents, be careful with caching: recurse.
+            return create(evaluator, obj)
+    else:
+        if parent is None and obj != _builtins:
+            return create(evaluator, obj, create(evaluator, _builtins))
 
-    if not inspect.ismodule(obj):
         faked = fake.get_faked(module and module.obj, obj)
         if faked is not None:
             faked.parent = parent
             return faked
 
-    try:
-        if parent == builtin and obj.__module__ in ('builtins', '__builtin__'):
-            return builtin.get_by_name(obj.__name__)
-    except AttributeError:
-        pass
-
-    return CompiledObject(obj, parent)
+    return CompiledObject(evaluator, obj, parent)

jedi/evaluate/compiled/fake.py

@@ -6,16 +6,44 @@ mixing in Python code, the autocompletion should work much better for builtins.
 
 import os
 import inspect
+import types
 
-from jedi._compatibility import is_py3, builtins, unicode
+from jedi._compatibility import is_py3, builtins, unicode, is_py34
 from jedi.cache import memoize_function
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.parser import tree as pt
 from jedi.evaluate.helpers import FakeName
 
 modules = {}
 
 
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace
+    )
+    if is_py34:
+        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
+
+
 def _load_faked_module(module):
     module_name = module.__name__
     if module_name == '__builtin__' and not is_py3:
@@ -31,8 +59,8 @@ def _load_faked_module(module):
         except IOError:
             modules[module_name] = None
             return
-        grammar = load_grammar('grammar3.4')
-        module = Parser(grammar, unicode(source), module_name).module
+        grammar = load_grammar(version='3.4')
+        module = ParserWithRecovery(grammar, unicode(source), module_name).module
         modules[module_name] = module
 
         if module_name == 'builtins' and not is_py3:
@@ -65,7 +93,15 @@ def get_module(obj):
         # Unfortunately in some cases like `int` there's no __module__
         return builtins
     else:
-        return __import__(imp_plz)
+        if imp_plz is None:
+            # Happens for example in `(_ for _ in []).send.__module__`.
+            return builtins
+        else:
+            try:
+                return __import__(imp_plz)
+            except ImportError:
+                # __module__ can be something arbitrary that doesn't exist.
+                return builtins
 
 
 def _faked(module, obj, name):
@@ -75,7 +111,7 @@ def _faked(module, obj, name):
 
     faked_mod = _load_faked_module(module)
     if faked_mod is None:
-        return
+        return None
 
     # Having the module as a `parser.representation.module`, we need to scan
     # for methods.
@@ -84,23 +120,32 @@ def _faked(module, obj, name):
             return search_scope(faked_mod, obj.__name__)
         elif not inspect.isclass(obj):
             # object is a method or descriptor
-            cls = search_scope(faked_mod, obj.__objclass__.__name__)
-            if cls is None:
-                return
-            return search_scope(cls, obj.__name__)
+            try:
+                objclass = obj.__objclass__
+            except AttributeError:
+                return None
+            else:
+                cls = search_scope(faked_mod, objclass.__name__)
+                if cls is None:
+                    return None
+                return search_scope(cls, obj.__name__)
     else:
         if obj == module:
             return search_scope(faked_mod, name)
         else:
-            cls = search_scope(faked_mod, obj.__name__)
+            try:
+                cls_name = obj.__name__
+            except AttributeError:
+                return None
+            cls = search_scope(faked_mod, cls_name)
             if cls is None:
-                return
+                return None
             return search_scope(cls, name)
 
 
 @memoize_function
 def get_faked(module, obj, name=None):
-    obj = obj.__class__ if is_class_instance(obj) else obj
+    obj = type(obj) if is_class_instance(obj) else obj
     result = _faked(module, obj, name)
     if result is None or isinstance(result, pt.Class):
         # We're not interested in classes. What we want is functions.
@@ -119,7 +164,9 @@ def get_faked(module, obj, name=None):
 
 def is_class_instance(obj):
     """Like inspect.* methods."""
-    return not (inspect.isclass(obj) or inspect.ismodule(obj)
-                or inspect.isbuiltin(obj) or inspect.ismethod(obj)
-                or inspect.ismethoddescriptor(obj) or inspect.iscode(obj)
-                or inspect.isgenerator(obj))
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

jedi/evaluate/compiled/fake/builtins.pym

@@ -124,7 +124,7 @@ class list():
         return self.__iterable[y]
 
     def pop(self):
-        return self.__iterable[-1]
+        return self.__iterable[int()]
 
 
 class tuple():
@@ -202,11 +202,29 @@ class dict():
         except KeyError:
             return d
 
+    def values(self):
+        return self.__elements.values()
+
     def setdefault(self, k, d):
         # TODO maybe also return the content
         return d
 
 
+class enumerate():
+    def __init__(self, sequence, start=0):
+        self.__sequence = sequence
+
+    def __iter__(self):
+        for i in self.__sequence:
+            yield 1, i
+
+    def __next__(self):
+        return next(self.__iter__())
+
+    def next(self):
+        return next(self.__iter__())
+
+
 class reversed():
     def __init__(self, sequence):
         self.__sequence = sequence

jedi/evaluate/compiled/mixed.py

@@ -0,0 +1,158 @@
+"""
+Used only for REPL Completion.
+"""
+
+import inspect
+import os
+
+from jedi import common
+from jedi.parser.fast import FastParser
+from jedi.evaluate import compiled
+from jedi.cache import underscore_memoization
+
+
+class MixedObject(object):
+    """
+    A ``MixedObject`` is used in two ways:
+
+    1. It uses the default logic of ``parser.tree`` objects,
+    2. except for getattr calls. The names dicts are generated in a fashion
+       like ``CompiledObject``.
+
+    This combined logic makes it possible to provide more powerful REPL
+    completion. It allows side effects that are not noticable with the default
+    parser structure to still be completeable.
+
+    The biggest difference from CompiledObject to MixedObject is that we are
+    generally dealing with Python code and not with C code. This will generate
+    fewer special cases, because we in Python you don't have the same freedoms
+    to modify the runtime.
+    """
+    def __init__(self, evaluator, obj, node_name):
+        self._evaluator = evaluator
+        self.obj = obj
+        self.node_name = node_name
+        self.definition = node_name.get_definition()
+
+    @property
+    def names_dict(self):
+        return LazyMixedNamesDict(self._evaluator, self)
+
+    def names_dicts(self, search_global):
+        # TODO is this needed?
+        assert search_global is False
+        return [self.names_dict]
+
+    def api_type(self):
+        mappings = {
+            'expr_stmt': 'statement',
+            'classdef': 'class',
+            'funcdef': 'function',
+            'file_input': 'module',
+        }
+        return mappings[self.definition.type]
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+
+    def __getattr__(self, name):
+        return getattr(self.definition, name)
+
+
+class MixedName(compiled.CompiledName):
+    """
+    The ``CompiledName._compiled_object`` is our MixedObject.
+    """
+    @property
+    @underscore_memoization
+    def parent(self):
+        return create(self._evaluator, getattr(self._compiled_obj.obj, self.name))
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, Name tries to overwrite the parent attribute.
+
+    @property
+    def start_pos(self):
+        if isinstance(self.parent, MixedObject):
+            return self.parent.node_name.start_pos
+
+        # This means a start_pos that doesn't exist (compiled objects).
+        return (0, 0)
+
+
+class LazyMixedNamesDict(compiled.LazyNamesDict):
+    name_class = MixedName
+
+
+def parse(grammar, path):
+    with open(path) as f:
+        source = f.read()
+    source = common.source_to_unicode(source)
+    return FastParser(grammar, source, path)
+
+
+def _load_module(evaluator, path, python_object):
+    module = parse(evaluator.grammar, path).module
+    python_module = inspect.getmodule(python_object)
+
+    evaluator.modules[python_module.__name__] = module
+    return module
+
+
+def find_syntax_node_name(evaluator, python_object):
+    try:
+        path = inspect.getsourcefile(python_object)
+    except TypeError:
+        # The type might not be known (e.g. class_with_dict.__weakref__)
+        return None
+    if path is None or not os.path.exists(path):
+        # The path might not exist or be e.g. <stdin>.
+        return None
+
+    module = _load_module(evaluator, path, python_object)
+
+    if inspect.ismodule(python_object):
+        # We don't need to check names for modules, because there's not really
+        # a way to write a module in a module in Python (and also __name__ can
+        # be something like ``email.utils``).
+        return module
+
+    try:
+        names = module.used_names[python_object.__name__]
+    except NameError:
+        return None
+
+    names = [n for n in names if n.is_definition()]
+
+    try:
+        code = python_object.__code__
+        # By using the line number of a code object we make the lookup in a
+        # file pretty easy. There's still a possibility of people defining
+        # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
+        # do so we just don't care.
+        line_nr = code.co_firstlineno
+    except AttributeError:
+        pass
+    else:
+        line_names = [name for name in names if name.start_pos[0] == line_nr]
+        # There's a chance that the object is not available anymore, because
+        # the code has changed in the background.
+        if line_names:
+            return line_names[-1]
+
+    # It's really hard to actually get the right definition, here as a last
+    # resort we just return the last one. This chance might lead to odd
+    # completions at some points but will lead to mostly correct type
+    # inference, because people tend to define a public name in a module only
+    # once.
+    return names[-1]
+
+
+@compiled.compiled_objects_cache('mixed_cache')
+def create(evaluator, obj):
+    name = find_syntax_node_name(evaluator, obj)
+    if name is None:
+        return compiled.create(evaluator, obj)
+    else:
+        return MixedObject(evaluator, obj, name)

jedi/evaluate/docstrings.py

@@ -20,7 +20,7 @@ from itertools import chain
 from textwrap import dedent
 
 from jedi.evaluate.cache import memoize_default
-from jedi.parser import Parser, load_grammar
+from jedi.parser import ParserWithRecovery, load_grammar
 from jedi.common import indent_block
 from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated
 
@@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module):
     # Take the default grammar here, if we load the Python 2.7 grammar here, it
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
-    p = Parser(load_grammar(), code % indent_block(string))
+    p = ParserWithRecovery(load_grammar(), code % indent_block(string))
     try:
         pseudo_cls = p.module.subscopes[0]
         # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
@@ -164,8 +164,8 @@ def _execute_array_values(evaluator, array):
     """
     if isinstance(array, Array):
         values = []
-        for typ in array.values():
-            objects = _execute_array_values(evaluator, typ)
+        for types in array.py__iter__():
+            objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types))
             values.append(AlreadyEvaluated(objects))
         return [FakeSequence(evaluator, values, array.type)]
     else:
@@ -176,11 +176,11 @@ def _execute_array_values(evaluator, array):
 def follow_param(evaluator, param):
     func = param.parent_function
 
-    return [p
-            for param_str in _search_param_in_docstr(func.raw_doc,
-                                                     str(param.name))
+    return set(
+        [p for param_str in _search_param_in_docstr(func.raw_doc,
+                                                    str(param.name))
             for p in _evaluate_for_statement_string(evaluator, param_str,
-                                                    param.get_parent_until())]
+                                                    param.get_parent_until())])
 
 
 @memoize_default(None, evaluator_is_first_arg=True)

jedi/evaluate/dynamic.py

@@ -26,6 +26,9 @@ from jedi.evaluate.cache import memoize_default
 from jedi.evaluate import imports
 
 
+MAX_PARAM_SEARCHES = 20
+
+
 class ParamListener(object):
     """
     This listener is used to get the params for a function.
@@ -52,17 +55,21 @@ def search_params(evaluator, param):
     is.
     """
     if not settings.dynamic_params:
-        return []
+        return set()
 
-    func = param.get_parent_until(tree.Function)
-    debug.dbg('Dynamic param search for %s in %s.', param, str(func.name))
-    # Compare the param names.
-    names = [n for n in search_function_call(evaluator, func)
-             if n.value == param.name.value]
-    # Evaluate the ExecutedParams to types.
-    result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names))
-    debug.dbg('Dynamic param result %s', result)
-    return result
+    evaluator.dynamic_params_depth += 1
+    try:
+        func = param.get_parent_until(tree.Function)
+        debug.dbg('Dynamic param search for %s in %s.', param, str(func.name), color='MAGENTA')
+        # Compare the param names.
+        names = [n for n in search_function_call(evaluator, func)
+                 if n.value == param.name.value]
+        # Evaluate the ExecutedParams to types.
+        result = set(chain.from_iterable(n.parent.eval(evaluator) for n in names))
+        debug.dbg('Dynamic param result %s', result, color='MAGENTA')
+        return result
+    finally:
+        evaluator.dynamic_params_depth -= 1
 
 
 @memoize_default([], evaluator_is_first_arg=True)
@@ -72,52 +79,29 @@ def search_function_call(evaluator, func):
     """
     from jedi.evaluate import representation as er
 
-    def get_params_for_module(module):
-        """
-        Returns the values of a param, or an empty array.
-        """
-        @memoize_default([], evaluator_is_first_arg=True)
-        def get_posibilities(evaluator, module, func_name):
+    def get_possible_nodes(module, func_name):
             try:
                 names = module.used_names[func_name]
             except KeyError:
-                return []
+                return
 
             for name in names:
-                parent = name.parent
-                if tree.is_node(parent, 'trailer'):
-                    parent = parent.parent
+                bracket = name.get_next_leaf()
+                trailer = bracket.parent
+                if trailer.type == 'trailer' and bracket == '(':
+                    yield name, trailer
 
-                trailer = None
-                if tree.is_node(parent, 'power'):
-                    for t in parent.children[1:]:
-                        if t == '**':
-                            break
-                        if t.start_pos > name.start_pos and t.children[0] == '(':
-                            trailer = t
-                            break
-                if trailer is not None:
-                    types = evaluator.goto_definition(name)
-
-                    # We have to remove decorators, because they are not the
-                    # "original" functions, this way we can easily compare.
-                    # At the same time we also have to remove InstanceElements.
-                    undec = []
-                    for escope in types:
-                        if escope.isinstance(er.Function, er.Instance) \
-                                and escope.decorates is not None:
-                            undec.append(escope.decorates)
-                        elif isinstance(escope, er.InstanceElement):
-                            undec.append(escope.var)
-                        else:
-                            undec.append(escope)
-
-                    if evaluator.wrap(compare) in undec:
-                        # Only if we have the correct function we execute
-                        # it, otherwise just ignore it.
-                        evaluator.eval_trailer(types, trailer)
-            return listener.param_possibilities
-        return get_posibilities(evaluator, module, func_name)
+    def undecorate(typ):
+        # We have to remove decorators, because they are not the
+        # "original" functions, this way we can easily compare.
+        # At the same time we also have to remove InstanceElements.
+        if typ.isinstance(er.Function, er.Instance) \
+                and typ.decorates is not None:
+            return typ.decorates
+        elif isinstance(typ, er.InstanceElement):
+            return typ.var
+        else:
+            return typ
 
     current_module = func.get_parent_until()
     func_name = unicode(func.name)
@@ -134,13 +118,32 @@ def search_function_call(evaluator, func):
 
     try:
         result = []
-        # This is like backtracking: Get the first possible result.
+        i = 0
         for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name):
-            result = get_params_for_module(mod)
+            for name, trailer in get_possible_nodes(mod, func_name):
+                i += 1
+
+                # This is a simple way to stop Jedi's dynamic param recursion
+                # from going wild: The deeper Jedi's in the recursin, the less
+                # code should be evaluated.
+                if i * evaluator.dynamic_params_depth > MAX_PARAM_SEARCHES:
+                    return listener.param_possibilities
+
+                for typ in evaluator.goto_definitions(name):
+                    undecorated = undecorate(typ)
+                    if evaluator.wrap(compare) == undecorated:
+                        # Only if we have the correct function we execute
+                        # it, otherwise just ignore it.
+                        evaluator.eval_trailer([typ], trailer)
+
+            result = listener.param_possibilities
+
+            # If there are results after processing a module, we're probably
+            # good to process.
             if result:
-                break
+                return result
     finally:
         # cleanup: remove the listener; important: should not stick.
         func.listeners.remove(listener)
 
-    return result
+    return set()

jedi/evaluate/finder.py

@@ -3,6 +3,9 @@ Searching for names with given scope and name. This is very central in Jedi and
 Python. The name resolution is quite complicated with descripter,
 ``__getattribute__``, ``__getattr__``, ``global``, etc.
 
+If you want to understand name resolution, please read the first few chapters
+in http://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/.
+
 Flow checks
 +++++++++++
 
@@ -13,15 +16,17 @@ check for -> a is a string). There's big potential in these checks.
 """
 from itertools import chain
 
-from jedi._compatibility import unicode, u
+from jedi._compatibility import unicode
 from jedi.parser import tree
 from jedi import debug
 from jedi import common
+from jedi.common import unite
 from jedi import settings
 from jedi.evaluate import representation as er
 from jedi.evaluate import dynamic
 from jedi.evaluate import compiled
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import iterable
 from jedi.evaluate import imports
 from jedi.evaluate import analysis
@@ -53,12 +58,15 @@ def filter_definition_names(names, origin, position=None):
     Filter names that are actual definitions in a scope. Names that are just
     used will be ignored.
     """
+    if not names:
+        return []
+
     # Just calculate the scope from the first
     stmt = names[0].get_definition()
     scope = stmt.get_parent_scope()
 
-    if not (isinstance(scope, er.FunctionExecution)
-            and isinstance(scope.base, er.LambdaWrapper)):
+    if not (isinstance(scope, er.FunctionExecution) and
+            isinstance(scope.base, er.LambdaWrapper)):
         names = filter_after_position(names, position)
     names = [name for name in names if name.is_definition()]
 
@@ -79,25 +87,34 @@ class NameFinder(object):
         self.scope = evaluator.wrap(scope)
         self.name_str = name_str
         self.position = position
+        self._found_predefined_if_name = None
 
     @debug.increase_indent
-    def find(self, scopes, search_global=False):
+    def find(self, scopes, attribute_lookup):
+        """
+        :params bool attribute_lookup: Tell to logic if we're accessing the
+            attribute or the contents of e.g. a function.
+        """
         # TODO rename scopes to names_dicts
+
         names = self.filter_name(scopes)
-        types = self._names_to_types(names, search_global)
+        if self._found_predefined_if_name is not None:
+            return self._found_predefined_if_name
+
+        types = self._names_to_types(names, attribute_lookup)
 
         if not names and not types \
-                and not (isinstance(self.name_str, tree.Name)
-                         and isinstance(self.name_str.parent.parent, tree.Param)):
+                and not (isinstance(self.name_str, tree.Name) and
+                         isinstance(self.name_str.parent.parent, tree.Param)):
             if not isinstance(self.name_str, (str, unicode)):  # TODO Remove?
-                if search_global:
+                if attribute_lookup:
+                    analysis.add_attribute_error(self._evaluator,
+                                                 self.scope, self.name_str)
+                else:
                     message = ("NameError: name '%s' is not defined."
                                % self.name_str)
                     analysis.add(self._evaluator, 'name-error', self.name_str,
                                  message)
-                else:
-                    analysis.add_attribute_error(self._evaluator,
-                                                 self.scope, self.name_str)
 
         debug.dbg('finder._names_to_types: %s -> %s', names, types)
         return types
@@ -143,6 +160,13 @@ class NameFinder(object):
                 last_names.append(name)
                 continue
 
+            if isinstance(stmt, er.ModuleWrapper):
+                # In case of REPL completion, we can infer modules names that
+                # don't really have a definition (because they are really just
+                # namespaces). In this case we can just add it.
+                last_names.append(name)
+                continue
+
             if isinstance(name, compiled.CompiledName) \
                     or isinstance(name, er.InstanceName) and isinstance(name._origin_name, compiled.CompiledName):
                 last_names.append(name)
@@ -150,8 +174,35 @@ class NameFinder(object):
 
             if isinstance(self.name_str, tree.Name):
                 origin_scope = self.name_str.get_parent_until(tree.Scope, reverse=True)
+                scope = self.name_str
+                check = None
+                while True:
+                    scope = scope.parent
+                    if scope.type in ("if_stmt", "for_stmt", "comp_for"):
+                        try:
+                            name_dict = self._evaluator.predefined_if_name_dict_dict[scope]
+                            types = set(name_dict[str(self.name_str)])
+                        except KeyError:
+                            continue
+                        else:
+                            if self.name_str.start_pos < scope.children[1].end_pos:
+                                # It doesn't make any sense to check if
+                                # statements in the if statement itself, just
+                                # deliver types.
+                                self._found_predefined_if_name = types
+                            else:
+                                check = flow_analysis.break_check(self._evaluator, self.scope,
+                                                                  origin_scope)
+                                if check is flow_analysis.UNREACHABLE:
+                                    self._found_predefined_if_name = set()
+                                else:
+                                    self._found_predefined_if_name = types
+                            break
+                    if isinstance(scope, tree.IsScope) or scope is None:
+                        break
             else:
                 origin_scope = None
+
             if isinstance(stmt.parent, compiled.CompiledObject):
                 # TODO seriously? this is stupid.
                 continue
@@ -159,6 +210,7 @@ class NameFinder(object):
                                               stmt, origin_scope)
             if check is not flow_analysis.UNREACHABLE:
                 last_names.append(name)
+
             if check is flow_analysis.REACHABLE:
                 break
 
@@ -179,7 +231,7 @@ class NameFinder(object):
                 break
 
         debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self.name_str,
-                  self.scope, u(names), self.position)
+                  self.scope, names, self.position)
         return list(self._clean_names(names))
 
     def _clean_names(self, names):
@@ -190,29 +242,37 @@ class NameFinder(object):
         """
         for n in names:
             definition = n.parent
-            if isinstance(definition, (tree.Function, tree.Class, tree.Module)):
+            if isinstance(definition, (compiled.CompiledObject,
+                iterable.BuiltinMethod)):
+                # TODO this if should really be removed by changing the type of
+                #      those classes.
+                yield n
+            elif definition.type in ('funcdef', 'classdef', 'file_input'):
                 yield self._evaluator.wrap(definition).name
             else:
                 yield n
 
     def _check_getattr(self, inst):
         """Checks for both __getattr__ and __getattribute__ methods"""
-        result = []
+        result = set()
         # str is important, because it shouldn't be `Name`!
         name = compiled.create(self._evaluator, str(self.name_str))
         with common.ignored(KeyError):
             result = inst.execute_subscope_by_name('__getattr__', name)
         if not result:
-            # this is a little bit special. `__getattribute__` is executed
-            # before anything else. But: I know no use case, where this
-            # could be practical and the jedi would return wrong types. If
-            # you ever have something, let me know!
+            # This is a little bit special. `__getattribute__` is in Python
+            # executed before `__getattr__`. But: I know no use case, where
+            # this could be practical and where jedi would return wrong types.
+            # If you ever find something, let me know!
+            # We are inversing this, because a hand-crafted `__getattribute__`
+            # could still call another hand-crafted `__getattr__`, but not the
+            # other way around.
             with common.ignored(KeyError):
                 result = inst.execute_subscope_by_name('__getattribute__', name)
         return result
 
-    def _names_to_types(self, names, search_global):
-        types = []
+    def _names_to_types(self, names, attribute_lookup):
+        types = set()
 
         # Add isinstance and other if/assert knowledge.
         if isinstance(self.name_str, tree.Name):
@@ -231,13 +291,13 @@ class NameFinder(object):
 
         for name in names:
             new_types = _name_to_types(self._evaluator, name, self.scope)
-            if isinstance(self.scope, (er.Class, er.Instance)) and not search_global:
-                types += self._resolve_descriptors(name, new_types)
+            if isinstance(self.scope, (er.Class, er.Instance)) and attribute_lookup:
+                types |= set(self._resolve_descriptors(name, new_types))
             else:
-                types += new_types
+                types |= set(new_types)
         if not names and isinstance(self.scope, er.Instance):
             # handling __getattr__ / __getattribute__
-            types = self._check_getattr(self.scope)
+            return self._check_getattr(self.scope)
 
         return types
 
@@ -249,56 +309,67 @@ class NameFinder(object):
         if not isinstance(name_scope, (er.Instance, tree.Class)):
             return types
 
-        result = []
+        result = set()
         for r in types:
             try:
                 desc_return = r.get_descriptor_returns
             except AttributeError:
-                result.append(r)
+                result.add(r)
             else:
-                result += desc_return(self.scope)
+                result |= desc_return(self.scope)
         return result
 
 
-@memoize_default([], evaluator_is_first_arg=True)
+def _get_global_stmt_scopes(evaluator, global_stmt, name):
+    global_stmt_scope = global_stmt.get_parent_scope()
+    module = global_stmt_scope.get_parent_until()
+    for used_name in module.used_names[str(name)]:
+        if used_name.parent.type == 'global_stmt':
+            yield evaluator.wrap(used_name.get_parent_scope())
+
+
+@memoize_default(set(), evaluator_is_first_arg=True)
 def _name_to_types(evaluator, name, scope):
     types = []
     typ = name.get_definition()
     if typ.isinstance(tree.ForStmt):
-        for_types = evaluator.eval_element(typ.children[3])
-        for_types = iterable.get_iterator_types(for_types)
-        types += check_tuple_assignments(for_types, name)
-    elif typ.isinstance(tree.CompFor):
-        for_types = evaluator.eval_element(typ.children[3])
-        for_types = iterable.get_iterator_types(for_types)
-        types += check_tuple_assignments(for_types, name)
+        types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.WithStmt):
+        types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.ForStmt, tree.CompFor):
+        container_types = evaluator.eval_element(typ.children[3])
+        for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3])
+        types = check_tuple_assignments(evaluator, for_types, name)
     elif isinstance(typ, tree.Param):
-        types += _eval_param(evaluator, typ, scope)
+        types = _eval_param(evaluator, typ, scope)
     elif typ.isinstance(tree.ExprStmt):
-        types += _remove_statements(evaluator, typ, name)
+        types = _remove_statements(evaluator, typ, name)
     elif typ.isinstance(tree.WithStmt):
-        types += evaluator.eval_element(typ.node_from_name(name))
+        types = evaluator.eval_element(typ.node_from_name(name))
     elif isinstance(typ, tree.Import):
-        types += imports.ImportWrapper(evaluator, name).follow()
-    elif isinstance(typ, tree.GlobalStmt):
-        # TODO theoretically we shouldn't be using search_global here, it
-        # doesn't make sense, because it's a local search (for that name)!
-        # However, globals are not that important and resolving them doesn't
-        # guarantee correctness in any way, because we don't check for when
-        # something is executed.
-        types += evaluator.find_types(typ.get_parent_scope(), str(name),
-                                      search_global=True)
+        types = imports.ImportWrapper(evaluator, name).follow()
+    elif typ.type == 'global_stmt':
+        for s in _get_global_stmt_scopes(evaluator, typ, name):
+            finder = NameFinder(evaluator, s, str(name))
+            names_dicts = finder.scopes(search_global=True)
+            # For global_stmt lookups, we only need the first possible scope,
+            # which means the function itself.
+            names_dicts = [next(names_dicts)]
+            types += finder.find(names_dicts, attribute_lookup=False)
     elif isinstance(typ, tree.TryStmt):
         # TODO an exception can also be a tuple. Check for those.
         # TODO check for types that are not classes and add it to
         # the static analysis report.
-        exceptions = evaluator.eval_element(name.prev_sibling().prev_sibling())
-        types = list(chain.from_iterable(
-                     evaluator.execute(t) for t in exceptions))
+        exceptions = evaluator.eval_element(name.get_previous_sibling().get_previous_sibling())
+        types = set(chain.from_iterable(evaluator.execute(t) for t in exceptions))
     else:
         if typ.isinstance(er.Function):
             typ = typ.get_decorated_func()
-        types.append(typ)
+        types = set([typ])
     return types
 
 
@@ -309,7 +380,7 @@ def _remove_statements(evaluator, stmt, name):
     Due to lazy evaluation, statements like a = func; b = a; b() have to be
     evaluated.
     """
-    types = []
+    types = set()
     # Remove the statement docstr stuff for now, that has to be
     # implemented with the evaluator class.
     #if stmt.docstr:
@@ -320,18 +391,22 @@ def _remove_statements(evaluator, stmt, name):
         check_instance = stmt.instance
         stmt = stmt.var
 
-    types += evaluator.eval_statement(stmt, seek_name=name)
+    pep0484types = \
+        pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name)
+    if pep0484types:
+        return pep0484types
+    types |= evaluator.eval_statement(stmt, seek_name=name)
 
     if check_instance is not None:
         # class renames
-        types = [er.get_instance_el(evaluator, check_instance, a, True)
-                 if isinstance(a, (er.Function, tree.Function))
-                 else a for a in types]
+        types = set([er.get_instance_el(evaluator, check_instance, a, True)
+                     if isinstance(a, (er.Function, tree.Function))
+                     else a for a in types])
     return types
 
 
 def _eval_param(evaluator, param, scope):
-    res_new = []
+    res_new = set()
     func = param.get_parent_scope()
 
     cls = func.parent.get_parent_until((tree.Class, tree.Function))
@@ -342,11 +417,11 @@ def _eval_param(evaluator, param, scope):
         # This is where we add self - if it has never been
         # instantiated.
         if isinstance(scope, er.InstanceElement):
-            res_new.append(scope.instance)
+            res_new.add(scope.instance)
         else:
             inst = er.Instance(evaluator, evaluator.wrap(cls),
                                Arguments(evaluator, ()), is_generated=True)
-            res_new.append(inst)
+            res_new.add(inst)
         return res_new
 
     # Instances are typically faked, if the instance is not called from
@@ -355,23 +430,24 @@ def _eval_param(evaluator, param, scope):
             and func.instance.is_generated and str(func.name) == '__init__':
         param = func.var.params[param.position_nr]
 
-    # Add docstring knowledge.
+    # Add pep0484 and docstring knowledge.
+    pep0484_hints = pep0484.follow_param(evaluator, param)
     doc_params = docstrings.follow_param(evaluator, param)
-    if doc_params:
-        return doc_params
+    if pep0484_hints or doc_params:
+        return list(set(pep0484_hints) | set(doc_params))
 
     if isinstance(param, ExecutedParam):
-        return res_new + param.eval(evaluator)
+        return res_new | param.eval(evaluator)
     else:
         # Param owns no information itself.
-        res_new += dynamic.search_params(evaluator, param)
+        res_new |= dynamic.search_params(evaluator, param)
         if not res_new:
             if param.stars:
                 t = 'tuple' if param.stars == 1 else 'dict'
-                typ = evaluator.find_types(compiled.builtin, t)[0]
+                typ = list(evaluator.find_types(evaluator.BUILTINS, t))[0]
                 res_new = evaluator.execute(typ)
         if param.default:
-            res_new += evaluator.eval_element(param.default)
+            res_new |= evaluator.eval_element(param.default)
         return res_new
 
 
@@ -387,7 +463,7 @@ def check_flow_information(evaluator, flow, search_name, pos):
     if not settings.dynamic_flow_information:
         return None
 
-    result = []
+    result = set()
     if flow.is_scope():
         # Check for asserts.
         try:
@@ -403,14 +479,16 @@ def check_flow_information(evaluator, flow, search_name, pos):
                     break
 
     if isinstance(flow, (tree.IfStmt, tree.WhileStmt)):
-        element = flow.children[1]
-        result = _check_isinstance_type(evaluator, element, search_name)
+        potential_ifs = [c for c in flow.children[1::4] if c != ':']
+        for if_test in reversed(potential_ifs):
+            if search_name.start_pos > if_test.end_pos:
+                return _check_isinstance_type(evaluator, if_test, search_name)
     return result
 
 
 def _check_isinstance_type(evaluator, element, search_name):
     try:
-        assert element.type == 'power'
+        assert element.type in ('power', 'atom_expr')
         # this might be removed if we analyze and, etc
         assert len(element.children) == 2
         first, trailer = element.children
@@ -428,15 +506,18 @@ def _check_isinstance_type(evaluator, element, search_name):
         # Do a simple get_code comparison. They should just have the same code,
         # and everything will be all right.
         classes = lst[1][1][0]
-        call = helpers.call_of_name(search_name)
-        assert name.get_code() == call.get_code()
+        call = helpers.call_of_leaf(search_name)
+        assert name.get_code(normalized=True) == call.get_code(normalized=True)
     except AssertionError:
-        return []
+        return set()
 
-    result = []
-    for typ in evaluator.eval_element(classes):
-        for typ in (typ.values() if isinstance(typ, iterable.Array) else [typ]):
-            result += evaluator.execute(typ)
+    result = set()
+    for cls_or_tup in evaluator.eval_element(classes):
+        if isinstance(cls_or_tup, iterable.Array) and cls_or_tup.type == 'tuple':
+            for typ in unite(cls_or_tup.py__iter__()):
+                result |= evaluator.execute(typ)
+        else:
+            result |= evaluator.execute(cls_or_tup)
     return result
 
 
@@ -449,8 +530,8 @@ def global_names_dict_generator(evaluator, scope, position):
     the current scope is function:
 
     >>> from jedi._compatibility import u, no_unicode_pprint
-    >>> from jedi.parser import Parser, load_grammar
-    >>> parser = Parser(load_grammar(), u('''
+    >>> from jedi.parser import ParserWithRecovery, load_grammar
+    >>> parser = ParserWithRecovery(load_grammar(), u('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None
@@ -499,6 +580,12 @@ def global_names_dict_generator(evaluator, scope, position):
 
             for names_dict in scope.names_dicts(True):
                 yield names_dict, position
+                if hasattr(scope, 'resets_positions'):
+                    # TODO This is so ugly, seriously. However there's
+                    #      currently no good way of influencing
+                    #      global_names_dict_generator when it comes to certain
+                    #      objects.
+                    position = None
             if scope.type == 'funcdef':
                 # The position should be reset if the current scope is a function.
                 in_func = True
@@ -506,28 +593,26 @@ def global_names_dict_generator(evaluator, scope, position):
         scope = evaluator.wrap(scope.get_parent_scope())
 
     # Add builtins to the global scope.
-    for names_dict in compiled.builtin.names_dicts(True):
+    for names_dict in evaluator.BUILTINS.names_dicts(True):
         yield names_dict, None
 
 
-def check_tuple_assignments(types, name):
+def check_tuple_assignments(evaluator, types, name):
     """
     Checks if tuples are assigned.
     """
-    for index in name.assignment_indexes():
-        new_types = []
-        for r in types:
+    for index, node in name.assignment_indexes():
+        iterated = iterable.py__iter__(evaluator, types, node)
+        for _ in range(index + 1):
             try:
-                func = r.get_exact_index_types
-            except AttributeError:
-                debug.warning("Invalid tuple lookup #%s of result %s in %s",
-                              index, types, name)
-            else:
-                try:
-                    new_types += func(index)
-                except IndexError:
-                    pass
-        types = new_types
+                types = next(iterated)
+            except StopIteration:
+                # We could do this with the default param in next. But this
+                # would allow this loop to run for a very long time if the
+                # index number is high. Therefore break if the loop is
+                # finished.
+                types = set()
+                break
     return types
 
 

jedi/evaluate/flow_analysis.py

@@ -45,7 +45,8 @@ def break_check(evaluator, base_scope, stmt, origin_scope=None):
         if element_scope == origin_scope:
             return REACHABLE
         origin_scope = origin_scope.parent
-    return _break_check(evaluator, stmt, base_scope, element_scope)
+    x = _break_check(evaluator, stmt, base_scope, element_scope)
+    return x
 
 
 def _break_check(evaluator, stmt, base_scope, element_scope):
@@ -62,7 +63,8 @@ def _break_check(evaluator, stmt, base_scope, element_scope):
             reachable = reachable.invert()
         else:
             node = element_scope.node_in_which_check_node(stmt)
-            reachable = _check_if(evaluator, node)
+            if node is not None:
+                reachable = _check_if(evaluator, node)
     elif isinstance(element_scope, (tree.TryStmt, tree.WhileStmt)):
         return UNSURE
 
@@ -70,9 +72,14 @@ def _break_check(evaluator, stmt, base_scope, element_scope):
     if reachable in (UNREACHABLE, UNSURE):
         return reachable
 
+    if element_scope.type == 'file_input':
+        # The definition is in another module and therefore just return what we
+        # have generated.
+        return reachable
     if base_scope != element_scope and base_scope != element_scope.parent:
         return reachable & _break_check(evaluator, stmt, base_scope, element_scope.parent)
-    return reachable
+    else:
+        return reachable
 
 
 def _check_if(evaluator, node):

jedi/evaluate/helpers.py

@@ -26,7 +26,8 @@ def deep_ast_copy(obj, parent=None, new_elements=None):
         new_children = []
         for child in obj.children:
             typ = child.type
-            if typ in ('whitespace', 'operator', 'keyword', 'number', 'string'):
+            if typ in ('whitespace', 'operator', 'keyword', 'number', 'string',
+                       'indent', 'dedent', 'error_leaf'):
                 # At the moment we're not actually copying those primitive
                 # elements, because there's really no need to. The parents are
                 # obviously wrong, but that's not an issue.
@@ -55,23 +56,18 @@ def deep_ast_copy(obj, parent=None, new_elements=None):
                     new_names_dict[string] = [new_elements[n] for n in names]
         return new_obj
 
-    if obj.type == 'name':
+    if isinstance(obj, tree.BaseNode):
+        new_obj = copy_node(obj)
+    else:
         # Special case of a Name object.
         new_elements[obj] = new_obj = copy.copy(obj)
-        if parent is not None:
-            new_obj.parent = parent
-    elif isinstance(obj, tree.BaseNode):
-        new_obj = copy_node(obj)
-        if parent is not None:
-            for child in new_obj.children:
-                if isinstance(child, (tree.Name, tree.BaseNode)):
-                    child.parent = parent
-    else:  # String literals and so on.
-        new_obj = obj  # Good enough, don't need to copy anything.
+
+    if parent is not None:
+        new_obj.parent = parent
     return new_obj
 
 
-def call_of_name(name, cut_own_trailer=False):
+def call_of_leaf(leaf, cut_own_trailer=False):
     """
     Creates a "call" node that consist of all ``trailer`` and ``power``
     objects.  E.g. if you call it with ``append``::
@@ -81,28 +77,53 @@ def call_of_name(name, cut_own_trailer=False):
     You would get a node with the content ``list([]).append`` back.
 
     This generates a copy of the original ast node.
+
+    If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
+
+    # TODO remove cut_own_trailer option, since its always used with it. Just
+    #      ignore it, It's not what we want anyway. Or document it better?
     """
-    par = name
-    if tree.is_node(par.parent, 'trailer'):
-        par = par.parent
+    trailer = leaf.parent
+    # The leaf may not be the last or first child, because there exist three
+    # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
+    # we should not match anything more than x.
+    if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]):
+        if trailer.type == 'atom':
+            return trailer
+        return leaf
 
-    power = par.parent
-    if tree.is_node(power, 'power') and power.children[0] != name \
-            and not (power.children[-2] == '**' and
-                     name.start_pos > power.children[-1].start_pos):
-        par = power
-        # Now the name must be part of a trailer
-        index = par.children.index(name.parent)
-        if index != len(par.children) - 1 or cut_own_trailer:
-            # Now we have to cut the other trailers away.
-            par = deep_ast_copy(par)
-            if not cut_own_trailer:
-                # Normally we would remove just the stuff after the index, but
-                # if the option is set remove the index as well. (for goto)
-                index = index + 1
-            par.children[index:] = []
+    power = trailer.parent
+    index = power.children.index(trailer)
+    power = deep_ast_copy(power)
+    if cut_own_trailer:
+        cut = index
+    else:
+        cut = index + 1
+    power.children[cut:] = []
 
-    return par
+    if power.type == 'error_node':
+        start = index
+        while True:
+            start -= 1
+            if power.children[start].type != 'trailer':
+                break
+        transformed = tree.Node('power', power.children[start:])
+        transformed.parent = power.parent
+        return transformed
+
+    return power
+
+
+def get_names_of_node(node):
+    try:
+        children = node.children
+    except AttributeError:
+        if node.type == 'name':
+            return [node]
+        else:
+            return []
+    else:
+        return list(chain.from_iterable(get_names_of_node(c) for c in children))
 
 
 def get_module_names(module, all_scopes):

jedi/evaluate/imports.py

@@ -20,9 +20,9 @@ from itertools import chain
 from jedi._compatibility import find_module, unicode
 from jedi import common
 from jedi import debug
-from jedi import cache
 from jedi.parser import fast
 from jedi.parser import tree
+from jedi.parser.utils import save_parser, load_parser, parser_cache
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi import settings
@@ -70,7 +70,7 @@ class ImportWrapper(tree.Base):
     def follow(self, is_goto=False):
         if self._evaluator.recursion_detector.push_stmt(self._import):
             # check recursion
-            return []
+            return set()
 
         try:
             module = self._evaluator.wrap(self._import.get_parent_until())
@@ -97,7 +97,7 @@ class ImportWrapper(tree.Base):
             #    scopes = [NestedImportModule(module, self._import)]
 
             if from_import_name is not None:
-                types = list(chain.from_iterable(
+                types = set(chain.from_iterable(
                     self._evaluator.find_types(t, unicode(from_import_name),
                                                is_goto=is_goto)
                     for t in types))
@@ -109,11 +109,11 @@ class ImportWrapper(tree.Base):
                     types = importer.follow()
                     # goto only accepts `Name`
                     if is_goto:
-                        types = [s.name for s in types]
+                        types = set(s.name for s in types)
             else:
                 # goto only accepts `Name`
                 if is_goto:
-                    types = [s.name for s in types]
+                    types = set(s.name for s in types)
 
             debug.dbg('after import: %s', types)
         finally:
@@ -201,23 +201,24 @@ class Importer(object):
                 base = []
             if level > len(base):
                 path = module.py__file__()
-                import_path = list(import_path)
-                for i in range(level):
-                    path = os.path.dirname(path)
-                dir_name = os.path.basename(path)
-                # This is not the proper way to do relative imports. However, since
-                # Jedi cannot be sure about the entry point, we just calculate an
-                # absolute path here.
-                if dir_name:
-                    import_path.insert(0, dir_name)
-                else:
-                    _add_error(self._evaluator, import_path[-1])
-                    import_path = []
-                    # TODO add import error.
-                    debug.warning('Attempted relative import beyond top-level package.')
+                if path is not None:
+                    import_path = list(import_path)
+                    for i in range(level):
+                        path = os.path.dirname(path)
+                    dir_name = os.path.basename(path)
+                    # This is not the proper way to do relative imports. However, since
+                    # Jedi cannot be sure about the entry point, we just calculate an
+                    # absolute path here.
+                    if dir_name:
+                        import_path.insert(0, dir_name)
+                    else:
+                        _add_error(self._evaluator, import_path[-1])
+                        import_path = []
+                        # TODO add import error.
+                        debug.warning('Attempted relative import beyond top-level package.')
             else:
                 # Here we basically rewrite the level to 0.
-                import_path = tuple(base) + import_path
+                import_path = tuple(base) + tuple(import_path)
         self.import_path = import_path
 
     @property
@@ -248,7 +249,7 @@ class Importer(object):
     @memoize_default(NO_DEFAULT)
     def follow(self):
         if not self.import_path:
-            return []
+            return set()
         return self._do_import(self.import_path, self.sys_path_with_modifications())
 
     def _do_import(self, import_path, sys_path):
@@ -271,7 +272,7 @@ class Importer(object):
 
         module_name = '.'.join(import_parts)
         try:
-            return [self._evaluator.modules[module_name]]
+            return set([self._evaluator.modules[module_name]])
         except KeyError:
             pass
 
@@ -280,11 +281,11 @@ class Importer(object):
             # the module cache.
             bases = self._do_import(import_path[:-1], sys_path)
             if not bases:
-                return []
+                return set()
             # We can take the first element, because only the os special
             # case yields multiple modules, which is not important for
             # further imports.
-            base = bases[0]
+            base = list(bases)[0]
 
             # This is a huge exception, we follow a nested import
             # ``os.path``, because it's a very important one in Python
@@ -301,7 +302,7 @@ class Importer(object):
             except AttributeError:
                 # The module is not a package.
                 _add_error(self._evaluator, import_path[-1])
-                return []
+                return set()
             else:
                 debug.dbg('search_module %s in paths %s', module_name, paths)
                 for path in paths:
@@ -315,7 +316,7 @@ class Importer(object):
                         module_path = None
                 if module_path is None:
                     _add_error(self._evaluator, import_path[-1])
-                    return []
+                    return set()
         else:
             try:
                 debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
@@ -330,7 +331,7 @@ class Importer(object):
             except ImportError:
                 # The module is not a package.
                 _add_error(self._evaluator, import_path[-1])
-                return []
+                return set()
 
         source = None
         if is_pkg:
@@ -346,11 +347,20 @@ class Importer(object):
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path)
 
+        if module is None:
+            # The file might raise an ImportError e.g. and therefore not be
+            # importable.
+            return set()
+
         self._evaluator.modules[module_name] = module
-        return [module]
+        return set([module])
 
     def _generate_name(self, name):
-        return helpers.FakeName(name, parent=self.module)
+        # Create a pseudo import to be able to follow them.
+        name = helpers.FakeName(name)
+        imp = helpers.FakeImport(name, parent=self.module)
+        name.parent = imp
+        return name
 
     def _get_module_names(self, search_path=None):
         """
@@ -435,7 +445,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
     def load(source):
         dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
         if path is not None and path.endswith('.py') \
-                and not dotted_path in settings.auto_import_modules:
+                and dotted_path not in settings.auto_import_modules:
             if source is None:
                 with open(path, 'rb') as f:
                     source = f.read()
@@ -443,13 +453,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
             return compiled.load_module(evaluator, path)
         p = path
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
-        cache.save_parser(path, p)
+        save_parser(path, p)
         return p.module
 
     if sys_path is None:
         sys_path = evaluator.sys_path
 
-    cached = cache.load_parser(path)
+    cached = load_parser(path)
     module = load(source) if cached is None else cached.module
     module = evaluator.wrap(module)
     return module
@@ -470,7 +480,7 @@ def get_modules_containing_name(evaluator, mods, name):
     """
     def check_python_file(path):
         try:
-            return cache.parser_cache[path].parser.module
+            return parser_cache[path].parser.module
         except KeyError:
             try:
                 return check_fs(path)
@@ -497,7 +507,9 @@ def get_modules_containing_name(evaluator, mods, name):
         paths = set(settings.additional_dynamic_modules)
         for p in mod_paths:
             if p is not None:
-                d = os.path.dirname(p)
+                # We need abspath, because the seetings paths might not already
+                # have been converted to absolute paths.
+                d = os.path.dirname(os.path.abspath(p))
                 for entry in os.listdir(d):
                     if entry not in mod_paths:
                         if entry.endswith('.py'):

jedi/evaluate/iterable.py

@@ -20,74 +20,117 @@ It is important to note that:
 1. Array modfications work only in the current module.
 2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
 """
-from itertools import chain
-
-from jedi import common
+from jedi.common import unite, safe_property
 from jedi import debug
 from jedi import settings
-from jedi._compatibility import use_metaclass, is_py3, unicode
+from jedi._compatibility import use_metaclass, unicode, zip_longest
 from jedi.parser import tree
 from jedi.evaluate import compiled
 from jedi.evaluate import helpers
 from jedi.evaluate.cache import CachedMetaClass, memoize_default
 from jedi.evaluate import analysis
-
-
-def unite(iterable):
-    """Turns a two dimensional array into a one dimensional."""
-    return list(chain.from_iterable(iterable))
+from jedi.evaluate import pep0484
 
 
 class IterableWrapper(tree.Base):
     def is_class(self):
         return False
 
+    @memoize_default()
+    def _get_names_dict(self, names_dict):
+        builtin_methods = {}
+        for cls in reversed(type(self).mro()):
+            try:
+                builtin_methods.update(cls.builtin_methods)
+            except AttributeError:
+                pass
 
+        if not builtin_methods:
+            return names_dict
+
+        dct = {}
+        for names in names_dict.values():
+            for name in names:
+                name_str = name.value
+                try:
+                    method = builtin_methods[name_str, self.type]
+                except KeyError:
+                    dct[name_str] = [name]
+                else:
+                    parent = BuiltinMethod(self, method, name.parent)
+                    dct[name_str] = [helpers.FakeName(name_str, parent, is_definition=True)]
+        return dct
+
+
+class BuiltinMethod(IterableWrapper):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    def __init__(self, builtin, method, builtin_func):
+        self._builtin = builtin
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, params):
+        return self._method(self._builtin)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+def has_builtin_methods(cls):
+    cls.builtin_methods = {}
+    for func in cls.__dict__.values():
+        try:
+            cls.builtin_methods.update(func.registered_builtin_methods)
+        except AttributeError:
+            pass
+    return cls
+
+
+def register_builtin_method(method_name, type=None):
+    def wrapper(func):
+        dct = func.__dict__.setdefault('registered_builtin_methods', {})
+        dct[method_name, type] = func
+        return func
+    return wrapper
+
+
+@has_builtin_methods
 class GeneratorMixin(object):
+    type = None
+
+    @register_builtin_method('send')
+    @register_builtin_method('next')
+    @register_builtin_method('__next__')
+    def py__next__(self):
+        # TODO add TypeError if params are given.
+        return unite(self.py__iter__())
+
     @memoize_default()
     def names_dicts(self, search_global=False):  # is always False
-        dct = {}
-        executes_generator = '__next__', 'send', 'next'
-        for names in compiled.generator_obj.names_dict.values():
-            for name in names:
-                if name.value in executes_generator:
-                    parent = GeneratorMethod(self, name.parent)
-                    dct[name.value] = [helpers.FakeName(name.name, parent, is_definition=True)]
-                else:
-                    dct[name.value] = [name]
-        yield dct
-
-    def get_index_types(self, evaluator, index_array):
-        #debug.warning('Tried to get array access on a generator: %s', self)
-        analysis.add(self._evaluator, 'type-error-generator', index_array)
-        return []
-
-    def get_exact_index_types(self, index):
-        """
-        Exact lookups are used for tuple lookups, which are perfectly fine if
-        used with generators.
-        """
-        return [self.iter_content()[index]]
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        yield self._get_names_dict(gen_obj.names_dict)
 
     def py__bool__(self):
         return True
 
+    def py__class__(self):
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        return gen_obj.py__class__()
+
 
 class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)):
     """Handling of `yield` functions."""
+
     def __init__(self, evaluator, func, var_args):
         super(Generator, self).__init__()
         self._evaluator = evaluator
         self.func = func
         self.var_args = var_args
 
-    def iter_content(self):
-        """ returns the content of __iter__ """
-        # Directly execute it, because with a normal call to py__call__ a
-        # Generator will be returned.
+    def py__iter__(self):
         from jedi.evaluate.representation import FunctionExecution
         f = FunctionExecution(self._evaluator, self.func, self.var_args)
-        return f.get_return_types(check_yields=True)
+        return f.get_yield_types()
 
     def __getattr__(self, name):
         if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
@@ -101,90 +144,172 @@ class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin))
         return "<%s of %s>" % (type(self).__name__, self.func)
 
 
-class GeneratorMethod(IterableWrapper):
-    """``__next__`` and ``send`` methods."""
-    def __init__(self, generator, builtin_func):
-        self._builtin_func = builtin_func
-        self._generator = generator
-
-    def py__call__(self, evaluator, params):
-        # TODO add TypeError if params are given.
-        return self._generator.iter_content()
-
-    def __getattr__(self, name):
-        return getattr(self._builtin_func, name)
-
-
 class Comprehension(IterableWrapper):
     @staticmethod
     def from_atom(evaluator, atom):
-        mapping = {
-            '(': GeneratorComprehension,
-            '[': ListComprehension
-        }
-        return mapping[atom.children[0]](evaluator, atom)
+        bracket = atom.children[0]
+        if bracket == '{':
+            if atom.children[1].children[1] == ':':
+                cls = DictComprehension
+            else:
+                cls = SetComprehension
+        elif bracket == '(':
+            cls = GeneratorComprehension
+        elif bracket == '[':
+            cls = ListComprehension
+        return cls(evaluator, atom)
 
     def __init__(self, evaluator, atom):
         self._evaluator = evaluator
         self._atom = atom
 
+    def _get_comprehension(self):
+        # The atom contains a testlist_comp
+        return self._atom.children[1]
+
+    def _get_comp_for(self):
+        # The atom contains a testlist_comp
+        return self._get_comprehension().children[1]
+
     @memoize_default()
-    def eval_node(self):
+    def _eval_node(self, index=0):
         """
         The first part `x + 1` of the list comprehension:
 
             [x + 1 for x in foo]
         """
-        comprehension = self._atom.children[1]
+        comp_for = self._get_comp_for()
         # For nested comprehensions we need to search the last one.
-        last = comprehension.children[-1]
-        last_comp = comprehension.children[1]
-        while True:
-            if isinstance(last, tree.CompFor):
-                last_comp = last
-            elif not tree.is_node(last, 'comp_if'):
-                break
-            last = last.children[-1]
+        last_comp = list(comp_for.get_comp_fors())[-1]
+        return helpers.deep_ast_copy(self._get_comprehension().children[index], parent=last_comp)
 
-        return helpers.deep_ast_copy(comprehension.children[0], parent=last_comp)
+    @memoize_default()
+    def _iterate(self):
+        def nested(comp_fors):
+            comp_for = comp_fors[0]
+            input_node = comp_for.children[3]
+            input_types = evaluator.eval_element(input_node)
 
-    def get_exact_index_types(self, index):
-        return [self._evaluator.eval_element(self.eval_node())[index]]
+            iterated = py__iter__(evaluator, input_types, input_node)
+            exprlist = comp_for.children[1]
+            for types in iterated:
+                evaluator.predefined_if_name_dict_dict[comp_for] = \
+                    unpack_tuple_to_dict(evaluator, types, exprlist)
+                try:
+                    for result in nested(comp_fors[1:]):
+                        yield result
+                except IndexError:
+                    iterated = evaluator.eval_element(self._eval_node())
+                    if self.type == 'dict':
+                        yield iterated, evaluator.eval_element(self._eval_node(2))
+                    else:
+                        yield iterated
+                finally:
+                    del evaluator.predefined_if_name_dict_dict[comp_for]
+
+        evaluator = self._evaluator
+        comp_fors = list(self._get_comp_for().get_comp_fors())
+        for result in nested(comp_fors):
+            yield result
+
+    def py__iter__(self):
+        return self._iterate()
 
     def __repr__(self):
-        return "<e%s of %s>" % (type(self).__name__, self._atom)
+        return "<%s of %s>" % (type(self).__name__, self._atom)
 
 
+@has_builtin_methods
 class ArrayMixin(object):
     @memoize_default()
     def names_dicts(self, search_global=False):  # Always False.
         # `array.type` is a string with the type, e.g. 'list'.
-        scope = self._evaluator.find_types(compiled.builtin, self.type)[0]
+        scope = compiled.builtin_from_name(self._evaluator, self.type)
         # builtins only have one class -> [0]
-        scope = self._evaluator.execute(scope, (AlreadyEvaluated((self,)),))[0]
-        return scope.names_dicts(search_global)
+        scopes = self._evaluator.execute_evaluated(scope, self)
+        names_dicts = list(scopes)[0].names_dicts(search_global)
+        #yield names_dicts[0]
+        yield self._get_names_dict(names_dicts[1])
 
     def py__bool__(self):
         return None  # We don't know the length, because of appends.
 
+    def py__class__(self):
+        return compiled.builtin_from_name(self._evaluator, self.type)
+
+    @safe_property
+    def parent(self):
+        return self._evaluator.BUILTINS
+
+    @property
+    def name(self):
+        return FakeSequence(self._evaluator, [], self.type).name
+
+    @memoize_default()
+    def dict_values(self):
+        return unite(self._evaluator.eval_element(v) for k, v in self._items())
+
+    @register_builtin_method('values', type='dict')
+    def _imitate_values(self):
+        items = self.dict_values()
+        return create_evaluated_sequence_set(self._evaluator, items, type='list')
+        #return set([FakeSequence(self._evaluator, [AlreadyEvaluated(items)], 'tuple')])
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = [set([FakeSequence(self._evaluator, (k, v), 'tuple')])
+                 for k, v in self._items()]
+
+        return create_evaluated_sequence_set(self._evaluator, *items, type='list')
+
 
 class ListComprehension(Comprehension, ArrayMixin):
     type = 'list'
 
-    def get_index_types(self, evaluator, index):
-        return self.iter_content()
+    def py__getitem__(self, index):
+        all_types = list(self.py__iter__())
+        return all_types[index]
+
+
+class SetComprehension(Comprehension, ArrayMixin):
+    type = 'set'
+
+
+@has_builtin_methods
+class DictComprehension(Comprehension, ArrayMixin):
+    type = 'dict'
+
+    def _get_comp_for(self):
+        return self._get_comprehension().children[3]
+
+    def py__iter__(self):
+        for keys, values in self._iterate():
+            yield keys
+
+    def py__getitem__(self, index):
+        for keys, values in self._iterate():
+            for k in keys:
+                if isinstance(k, compiled.CompiledObject):
+                    if k.obj == index:
+                        return values
+        return self.dict_values()
+
+    def dict_values(self):
+        return unite(values for keys, values in self._iterate())
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = set(FakeSequence(self._evaluator,
+                    (AlreadyEvaluated(keys), AlreadyEvaluated(values)), 'tuple')
+                    for keys, values in self._iterate())
+
+        return create_evaluated_sequence_set(self._evaluator, items, type='list')
 
-    def iter_content(self):
-        return self._evaluator.eval_element(self.eval_node())
 
-    @property
-    def name(self):
-        return FakeSequence(self._evaluator, [], 'list').name
 
 
 class GeneratorComprehension(Comprehension, GeneratorMixin):
-    def iter_content(self):
-        return self._evaluator.eval_element(self.eval_node())
+    pass
 
 
 class Array(IterableWrapper, ArrayMixin):
@@ -209,60 +334,21 @@ class Array(IterableWrapper, ArrayMixin):
     def name(self):
         return helpers.FakeName(self.type, parent=self)
 
-    @memoize_default()
-    def get_index_types(self, evaluator, index=()):
-        """
-        Get the types of a specific index or all, if not given.
-
-        :param index: A subscriptlist node (or subnode).
-        """
-        indexes = create_indexes_or_slices(evaluator, index)
-        lookup_done = False
-        types = []
-        for index in indexes:
-            if isinstance(index, Slice):
-                types += [self]
-                lookup_done = True
-            elif isinstance(index, compiled.CompiledObject) \
-                    and isinstance(index.obj, (int, str, unicode)):
-                with common.ignored(KeyError, IndexError, TypeError):
-                    types += self.get_exact_index_types(index.obj)
-                    lookup_done = True
-
-        return types if lookup_done else self.values()
-
-    @memoize_default()
-    def values(self):
-        result = unite(self._evaluator.eval_element(v) for v in self._values())
-        result += check_array_additions(self._evaluator, self)
-        return result
-
-    def get_exact_index_types(self, mixed_index):
-        """ Here the index is an int/str. Raises IndexError/KeyError """
+    def py__getitem__(self, index):
+        """Here the index is an int/str. Raises IndexError/KeyError."""
         if self.type == 'dict':
-            for key, values in self._items():
-                # Because we only want the key to be a string.
-                keys = self._evaluator.eval_element(key)
-
-                for k in keys:
+            for key, value in self._items():
+                for k in self._evaluator.eval_element(key):
                     if isinstance(k, compiled.CompiledObject) \
-                            and mixed_index == k.obj:
-                        for value in values:
-                            return self._evaluator.eval_element(value)
+                            and index == k.obj:
+                        return self._evaluator.eval_element(value)
             raise KeyError('No key found in dictionary %s.' % self)
 
         # Can raise an IndexError
-        return self._evaluator.eval_element(self._items()[mixed_index])
-
-    def iter_content(self):
-        return self.values()
-
-    @common.safe_property
-    def parent(self):
-        return compiled.builtin
-
-    def get_parent_until(self):
-        return compiled.builtin
+        if isinstance(index, slice):
+            return set([self])
+        else:
+            return self._evaluator.eval_element(self._items()[index])
 
     def __getattr__(self, name):
         if name not in ['start_pos', 'get_only_subelement', 'parent',
@@ -270,10 +356,33 @@ class Array(IterableWrapper, ArrayMixin):
             raise AttributeError('Strange access on %s: %s.' % (self, name))
         return getattr(self.atom, name)
 
+    # @memoize_default()
+    def py__iter__(self):
+        """
+        While values returns the possible values for any array field, this
+        function returns the value for a certain index.
+        """
+        if self.type == 'dict':
+            # Get keys.
+            types = set()
+            for k, _ in self._items():
+                types |= self._evaluator.eval_element(k)
+            # We don't know which dict index comes first, therefore always
+            # yield all the types.
+            for _ in types:
+                yield types
+        else:
+            for value in self._items():
+                yield self._evaluator.eval_element(value)
+
+            additions = check_array_additions(self._evaluator, self)
+            if additions:
+                yield additions
+
     def _values(self):
         """Returns a list of a list of node."""
         if self.type == 'dict':
-            return list(chain.from_iterable(v for k, v in self._items()))
+            return unite(v for k, v in self._items())
         else:
             return self._items()
 
@@ -292,18 +401,14 @@ class Array(IterableWrapper, ArrayMixin):
                 op = next(iterator, None)
                 if op is None or op == ',':
                     kv.append(key)  # A set.
-                elif op == ':':  # A dict.
-                    kv.append((key, [next(iterator)]))
-                    next(iterator, None)  # Possible comma.
                 else:
-                    raise NotImplementedError('dict/set comprehensions')
+                    assert op == ':'  # A dict.
+                    kv.append((key, next(iterator)))
+                    next(iterator, None)  # Possible comma.
             return kv
         else:
             return [array_node]
 
-    def __iter__(self):
-        return iter(self._items())
-
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.atom)
 
@@ -326,20 +431,29 @@ class ImplicitTuple(_FakeArray):
 
 class FakeSequence(_FakeArray):
     def __init__(self, evaluator, sequence_values, type):
+        """
+        type should be one of "tuple", "list"
+        """
         super(FakeSequence, self).__init__(evaluator, sequence_values, type)
         self._sequence_values = sequence_values
 
     def _items(self):
         return self._sequence_values
 
-    def get_exact_index_types(self, index):
-        value = self._sequence_values[index]
-        return self._evaluator.eval_element(value)
+
+def create_evaluated_sequence_set(evaluator, *types_order, **kwargs):
+    """
+    ``sequence_type`` is a named argument, that doesn't work in Python2. For backwards
+    compatibility reasons, we're now using kwargs.
+    """
+    sequence_type = kwargs.get('sequence_type')
+    sets = tuple(AlreadyEvaluated(types) for types in types_order)
+    return set([FakeSequence(evaluator, sets, sequence_type)])
 
 
 class AlreadyEvaluated(frozenset):
     """A simple container to add already evaluated objects to an array."""
-    def get_code(self):
+    def get_code(self, normalized=False):
         # For debugging purposes.
         return str(self)
 
@@ -353,12 +467,16 @@ class FakeDict(_FakeArray):
         super(FakeDict, self).__init__(evaluator, dct, 'dict')
         self._dct = dct
 
-    def get_exact_index_types(self, index):
-        return list(chain.from_iterable(self._evaluator.eval_element(v)
-                                        for v in self._dct[index]))
+    def py__iter__(self):
+        yield set(compiled.create(self._evaluator, key) for key in self._dct)
+
+    def py__getitem__(self, index):
+        return unite(self._evaluator.eval_element(v) for v in self._dct[index])
 
     def _items(self):
-        return self._dct.items()
+        for key, values in self._dct.items():
+            # TODO this is not proper. The values could be multiple values?!
+            yield key, values[0]
 
 
 class MergedArray(_FakeArray):
@@ -366,56 +484,138 @@ class MergedArray(_FakeArray):
         super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].type)
         self._arrays = arrays
 
-    def get_exact_index_types(self, mixed_index):
-        raise IndexError
-
-    def values(self):
-        return list(chain(*(a.values() for a in self._arrays)))
-
-    def __iter__(self):
+    def py__iter__(self):
         for array in self._arrays:
-            for a in array:
+            for types in array.py__iter__():
+                yield types
+
+    def py__getitem__(self, index):
+        return unite(self.py__iter__())
+
+    def _items(self):
+        for array in self._arrays:
+            for a in array._items():
                 yield a
 
     def __len__(self):
         return sum(len(a) for a in self._arrays)
 
 
-def get_iterator_types(inputs):
-    """Returns the types of any iterator (arrays, yields, __iter__, etc)."""
-    iterators = []
-    # Take the first statement (for has always only
-    # one, remember `in`). And follow it.
-    for it in inputs:
-        if isinstance(it, (Generator, Array, ArrayInstance, Comprehension)):
-            iterators.append(it)
-        else:
-            if not hasattr(it, 'execute_subscope_by_name'):
-                debug.warning('iterator/for loop input wrong: %s', it)
-                continue
+def unpack_tuple_to_dict(evaluator, types, exprlist):
+    """
+    Unpacking tuple assignments in for statements and expr_stmts.
+    """
+    if exprlist.type == 'name':
+        return {exprlist.value: types}
+    elif exprlist.type == 'atom' and exprlist.children[0] in '([':
+        return unpack_tuple_to_dict(evaluator, types, exprlist.children[1])
+    elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
+                           'testlist_star_expr'):
+        dct = {}
+        parts = iter(exprlist.children[::2])
+        n = 0
+        for iter_types in py__iter__(evaluator, types, exprlist):
+            n += 1
             try:
-                iterators += it.execute_subscope_by_name('__iter__')
-            except KeyError:
-                debug.warning('iterators: No __iter__ method found.')
+                part = next(parts)
+            except StopIteration:
+                analysis.add(evaluator, 'value-error-too-many-values', part,
+                             message="ValueError: too many values to unpack (expected %s)" % n)
+            else:
+                dct.update(unpack_tuple_to_dict(evaluator, iter_types, part))
+        has_parts = next(parts, None)
+        if types and has_parts is not None:
+            analysis.add(evaluator, 'value-error-too-few-values', has_parts,
+                         message="ValueError: need more than %s values to unpack" % n)
+        return dct
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
+        # Something like ``arr[x], var = ...``.
+        # This is something that is not yet supported, would also be difficult
+        # to write into a dict.
+        return {}
+    elif exprlist.type == 'star_expr':  # `a, *b, c = x` type unpackings
+        # Currently we're not supporting them.
+        return {}
+    raise NotImplementedError
 
-    result = []
-    from jedi.evaluate.representation import Instance
-    for it in iterators:
-        if isinstance(it, Array):
-            # Array is a little bit special, since this is an internal array,
-            # but there's also the list builtin, which is another thing.
-            result += it.values()
-        elif isinstance(it, Instance):
-            # __iter__ returned an instance.
-            name = '__next__' if is_py3 else 'next'
-            try:
-                result += it.execute_subscope_by_name(name)
-            except KeyError:
-                debug.warning('Instance has no __next__ function in %s.', it)
+
+def py__iter__(evaluator, types, node=None):
+    debug.dbg('py__iter__')
+    type_iters = []
+    for typ in types:
+        try:
+            iter_method = typ.py__iter__
+        except AttributeError:
+            if node is not None:
+                analysis.add(evaluator, 'type-error-not-iterable', node,
+                             message="TypeError: '%s' object is not iterable" % typ)
         else:
-            # TODO this is not correct, __iter__ can return arbitrary input!
-            # Is a generator.
-            result += it.iter_content()
+            type_iters.append(iter_method())
+            #for result in iter_method():
+                #yield result
+
+    for t in zip_longest(*type_iters, fillvalue=set()):
+        yield unite(t)
+
+
+def py__iter__types(evaluator, types, node=None):
+    """
+    Calls `py__iter__`, but ignores the ordering in the end and just returns
+    all types that it contains.
+    """
+    return unite(py__iter__(evaluator, types, node))
+
+
+def py__getitem__(evaluator, types, trailer):
+    from jedi.evaluate.representation import Class
+    result = set()
+
+    trailer_op, node, trailer_cl = trailer.children
+    assert trailer_op == "["
+    assert trailer_cl == "]"
+
+    # special case: PEP0484 typing module, see
+    # https://github.com/davidhalter/jedi/issues/663
+    for typ in list(types):
+        if isinstance(typ, Class):
+            typing_module_types = \
+                pep0484.get_types_for_typing_module(evaluator, typ, node)
+            if typing_module_types is not None:
+                types.remove(typ)
+                result |= typing_module_types
+
+    if not types:
+        # all consumed by special cases
+        return result
+
+    for index in create_index_types(evaluator, node):
+        if isinstance(index, (compiled.CompiledObject, Slice)):
+            index = index.obj
+
+        if type(index) not in (float, int, str, unicode, slice):
+            # If the index is not clearly defined, we have to get all the
+            # possiblities.
+            for typ in list(types):
+                if isinstance(typ, Array) and typ.type == 'dict':
+                    types.remove(typ)
+                    result |= typ.dict_values()
+            return result | py__iter__types(evaluator, types)
+
+        for typ in types:
+            # The actual getitem call.
+            try:
+                getitem = typ.py__getitem__
+            except AttributeError:
+                analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op,
+                             message="TypeError: '%s' object is not subscriptable" % typ)
+            else:
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= py__iter__types(evaluator, set([typ]))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= typ.dict_values()
     return result
 
 
@@ -423,7 +623,7 @@ def check_array_additions(evaluator, array):
     """ Just a mapper function for the internal _check_array_additions """
     if array.type not in ('list', 'set'):
         # TODO also check for dict updates
-        return []
+        return set()
 
     is_list = array.type == 'list'
     try:
@@ -432,11 +632,12 @@ def check_array_additions(evaluator, array):
         # If there's no get_parent_until, it's a FakeSequence or another Fake
         # type. Those fake types are used inside Jedi's engine. No values may
         # be added to those after their creation.
-        return []
+        return set()
     return _check_array_additions(evaluator, array, current_module, is_list)
 
 
-@memoize_default([], evaluator_is_first_arg=True)
+@memoize_default(default=set(), evaluator_is_first_arg=True)
+@debug.increase_indent
 def _check_array_additions(evaluator, compare_array, module, is_list):
     """
     Checks if a `Array` has "add" (append, insert, extend) statements:
@@ -444,21 +645,24 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
     >>> a = [""]
     >>> a.append(1)
     """
+    debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA')
     if not settings.dynamic_array_additions or isinstance(module, compiled.CompiledObject):
-        return []
+        debug.dbg('Dynamic array search aborted.', color='MAGENTA')
+        return set()
 
     def check_additions(arglist, add_name):
         params = list(param.Arguments(evaluator, arglist).unpack())
-        result = []
+        result = set()
         if add_name in ['insert']:
             params = params[1:]
         if add_name in ['append', 'add', 'insert']:
             for key, nodes in params:
-                result += unite(evaluator.eval_element(node) for node in nodes)
+                result |= unite(evaluator.eval_element(node) for node in nodes)
         elif add_name in ['extend', 'update']:
             for key, nodes in params:
-                iterators = unite(evaluator.eval_element(node) for node in nodes)
-                result += get_iterator_types(iterators)
+                for node in nodes:
+                    types = evaluator.eval_element(node)
+                    result |= py__iter__types(evaluator, types, node)
         return result
 
     from jedi.evaluate import representation as er, param
@@ -469,7 +673,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
             node = element.atom
         else:
             # Is an Instance with an
-            # Arguments([AlreadyEvaluated([ArrayInstance])]) inside
+            # Arguments([AlreadyEvaluated([_ArrayInstance])]) inside
             # Yeah... I know... It's complicated ;-)
             node = list(element.var_args.argument_node[0])[0].var_args.trailer
         if isinstance(node, er.InstanceElement):
@@ -482,7 +686,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
     search_names = ['append', 'extend', 'insert'] if is_list else ['add', 'update']
     comp_arr_parent = get_execution_parent(compare_array)
 
-    added_types = []
+    added_types = set()
     for add_name in search_names:
         try:
             possible_names = module.used_names[add_name]
@@ -515,7 +719,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
                             or execution_trailer.children[0] != '(' \
                             or execution_trailer.children[1] == ')':
                         continue
-                power = helpers.call_of_name(name, cut_own_trailer=True)
+                power = helpers.call_of_leaf(name, cut_own_trailer=True)
                 # InstanceElements are special, because they don't get copied,
                 # but have this wrapper around them.
                 if isinstance(comp_arr_parent, er.InstanceElement):
@@ -527,11 +731,12 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
                     continue
                 if compare_array in evaluator.eval_element(power):
                     # The arrays match. Now add the results
-                    added_types += check_additions(execution_trailer.children[1], add_name)
+                    added_types |= check_additions(execution_trailer.children[1], add_name)
 
                 evaluator.recursion_detector.pop_stmt()
     # reset settings
     settings.dynamic_params_for_other_modules = temp_param_add
+    debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
     return added_types
 
 
@@ -540,12 +745,12 @@ def check_array_instances(evaluator, instance):
     if not settings.dynamic_array_additions:
         return instance.var_args
 
-    ai = ArrayInstance(evaluator, instance)
+    ai = _ArrayInstance(evaluator, instance)
     from jedi.evaluate import param
     return param.Arguments(evaluator, [AlreadyEvaluated([ai])])
 
 
-class ArrayInstance(IterableWrapper):
+class _ArrayInstance(IterableWrapper):
     """
     Used for the usage of set() and list().
     This is definitely a hack, but a good one :-)
@@ -561,21 +766,21 @@ class ArrayInstance(IterableWrapper):
         self.instance = instance
         self.var_args = instance.var_args
 
-    def iter_content(self):
-        """
-        The index is here just ignored, because of all the appends, etc.
-        lists/sets are too complicated too handle that.
-        """
-        items = []
-        for key, nodes in self.var_args.unpack():
-            for node in nodes:
-                for typ in self._evaluator.eval_element(node):
-                    items += get_iterator_types([typ])
+    def py__iter__(self):
+        try:
+            _, first_nodes = next(self.var_args.unpack())
+        except StopIteration:
+            types = set()
+        else:
+            types = unite(self._evaluator.eval_element(node) for node in first_nodes)
+            for types in py__iter__(self._evaluator, types, first_nodes[0]):
+                yield types
 
         module = self.var_args.get_parent_until()
         is_list = str(self.instance.name) == 'list'
-        items += _check_array_additions(self._evaluator, self.instance, module, is_list)
-        return items
+        additions = _check_array_additions(self._evaluator, self.instance, module, is_list)
+        if additions:
+            yield additions
 
 
 class Slice(object):
@@ -598,11 +803,11 @@ class Slice(object):
 
             result = self._evaluator.eval_element(element)
             if len(result) != 1:
-                # We want slices to be clear defined with just one type.
-                # Otherwise we will return an empty slice object.
+                # For simplicity, we want slices to be clear defined with just
+                # one type.  Otherwise we will return an empty slice object.
                 raise IndexError
             try:
-                return result[0].obj
+                return list(result)[0].obj
             except AttributeError:
                 return None
 
@@ -612,9 +817,15 @@ class Slice(object):
             return slice(None, None, None)
 
 
-def create_indexes_or_slices(evaluator, index):
-    if tree.is_node(index, 'subscript'):  # subscript is a slice operation.
-        start, stop, step = None, None, None
+def create_index_types(evaluator, index):
+    """
+    Handles slices in subscript nodes.
+    """
+    if index == ':':
+        # Like array[:]
+        return set([Slice(evaluator, None, None, None)])
+    elif tree.is_node(index, 'subscript'):  # subscript is a slice operation.
+        # Like array[:3]
         result = []
         for el in index.children:
             if el == ':':
@@ -627,5 +838,7 @@ def create_indexes_or_slices(evaluator, index):
                 result.append(el)
         result += [None] * (3 - len(result))
 
-        return (Slice(evaluator, *result),)
+        return set([Slice(evaluator, *result)])
+
+    # No slices
     return evaluator.eval_element(index)

jedi/evaluate/jedi_typing.py

@@ -0,0 +1,100 @@
+"""
+This module is not intended to be used in jedi, rather it will be fed to the
+jedi-parser to replace classes in the typing module
+"""
+
+try:
+    from collections import abc
+except ImportError:
+    # python 2
+    import collections as abc
+
+
+def factory(typing_name, indextypes):
+    class Iterable(abc.Iterable):
+        def __iter__(self):
+            while True:
+                yield indextypes[0]()
+
+    class Iterator(Iterable, abc.Iterator):
+        def next(self):
+            """ needed for python 2 """
+            return self.__next__()
+
+        def __next__(self):
+            return indextypes[0]()
+
+    class Sequence(abc.Sequence):
+        def __getitem__(self, index):
+            return indextypes[0]()
+
+    class MutableSequence(Sequence, abc.MutableSequence):
+        pass
+
+    class List(MutableSequence, list):
+        pass
+
+    class Tuple(Sequence, tuple):
+        def __getitem__(self, index):
+            if indextypes[1] == Ellipsis:
+                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
+                # Tuple[int, ...] means a tuple of ints of indetermined length
+                return indextypes[0]()
+            else:
+                return indextypes[index]()
+
+    class AbstractSet(Iterable, abc.Set):
+        pass
+
+    class MutableSet(AbstractSet, abc.MutableSet):
+        pass
+
+    class KeysView(Iterable, abc.KeysView):
+        pass
+
+    class ValuesView(abc.ValuesView):
+        def __iter__(self):
+            while True:
+                yield indextypes[1]()
+
+    class ItemsView(abc.ItemsView):
+        def __iter__(self):
+            while True:
+                yield (indextypes[0](), indextypes[1]())
+
+    class Mapping(Iterable, abc.Mapping):
+        def __getitem__(self, item):
+            return indextypes[1]()
+
+        def keys(self):
+            return KeysView()
+
+        def values(self):
+            return ValuesView()
+
+        def items(self):
+            return ItemsView()
+
+    class MutableMapping(Mapping, abc.MutableMapping):
+        pass
+
+    class Dict(MutableMapping, dict):
+        pass
+
+    dct = {
+        "Sequence": Sequence,
+        "MutableSequence": MutableSequence,
+        "List": List,
+        "Iterable": Iterable,
+        "Iterator": Iterator,
+        "AbstractSet": AbstractSet,
+        "MutableSet": MutableSet,
+        "Mapping": Mapping,
+        "MutableMapping": MutableMapping,
+        "Tuple": Tuple,
+        "KeysView": KeysView,
+        "ItemsView": ItemsView,
+        "ValuesView": ValuesView,
+        "Dict": Dict,
+    }
+    return dct[typing_name]

jedi/evaluate/param.py

@@ -12,6 +12,23 @@ from jedi.evaluate.helpers import FakeName
 from jedi.cache import underscore_memoization
 
 
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for iter_types in f():
+                try_iter_content(iter_types, depth + 1)
+
+
 class Arguments(tree.Base):
     def __init__(self, evaluator, argument_node, trailer=None):
         """
@@ -30,7 +47,10 @@ class Arguments(tree.Base):
             for el in self.argument_node:
                 yield 0, el
         else:
-            if not tree.is_node(self.argument_node, 'arglist'):
+            if not (tree.is_node(self.argument_node, 'arglist') or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (tree.is_node(self.argument_node, 'argument') and
+                     self.argument_node.children[0] in ('*', '**')))):
                 yield 0, self.argument_node
                 return
 
@@ -40,6 +60,10 @@ class Arguments(tree.Base):
                     continue
                 elif child in ('*', '**'):
                     yield len(child.value), next(iterator)
+                elif tree.is_node(child, 'argument') and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
                 else:
                     yield 0, child
 
@@ -49,7 +73,7 @@ class Arguments(tree.Base):
                 element = self.argument_node[0]
                 from jedi.evaluate.iterable import AlreadyEvaluated
                 if isinstance(element, AlreadyEvaluated):
-                    element = self._evaluator.eval_element(element)[0]
+                    element = list(self._evaluator.eval_element(element))[0]
             except IndexError:
                 return None
             else:
@@ -131,8 +155,8 @@ class Arguments(tree.Base):
                 debug.warning('TypeError: %s expected at least %s arguments, got %s',
                               name, len(arguments), i)
                 raise ValueError
-            values = list(chain.from_iterable(self._evaluator.eval_element(el)
-                                              for el in va_values))
+            values = set(chain.from_iterable(self._evaluator.eval_element(el)
+                                             for el in va_values))
             if not values and not optional:
                 # For the stdlib we always want values. If we don't get them,
                 # that's ok, maybe something is too hard to resolve, however,
@@ -160,6 +184,16 @@ class Arguments(tree.Base):
         else:
             return None
 
+    def eval_all(self, func=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, element_values in self.unpack():
+            for element in element_values:
+                types = self._evaluator.eval_element(element)
+                try_iter_content(types)
+
 
 class ExecutedParam(tree.Param):
     """Fake a param and give it values."""
@@ -169,9 +203,9 @@ class ExecutedParam(tree.Param):
         self._values = values
 
     def eval(self, evaluator):
-        types = []
+        types = set()
         for v in self._values:
-            types += evaluator.eval_element(v)
+            types |= evaluator.eval_element(v)
         return types
 
     @property
@@ -242,7 +276,7 @@ def get_params(evaluator, func, var_args):
             try:
                 key_param = param_dict[unicode(key)]
             except KeyError:
-                non_matching_keys[key] += va_values
+                non_matching_keys[key] = va_values
             else:
                 param_names.append(ExecutedParam(key_param, var_args, va_values).name)
 
@@ -353,11 +387,12 @@ def get_params(evaluator, func, var_args):
 def _iterate_star_args(evaluator, array, input_node, func=None):
     from jedi.evaluate.representation import Instance
     if isinstance(array, iterable.Array):
-        for field_stmt in array:  # yield from plz!
-            yield field_stmt
+        # TODO ._items is not the call we want here. Replace in the future.
+        for node in array._items():
+            yield node
     elif isinstance(array, iterable.Generator):
-        for field_stmt in array.iter_content():
-            yield iterable.AlreadyEvaluated([field_stmt])
+        for types in array.py__iter__():
+            yield iterable.AlreadyEvaluated(types)
     elif isinstance(array, Instance) and array.name.get_code() == 'tuple':
         debug.warning('Ignored a tuple *args input %s' % array)
     else:
@@ -379,10 +414,10 @@ def _star_star_dict(evaluator, array, input_node, func):
         return array._dct
     elif isinstance(array, iterable.Array) and array.type == 'dict':
         # TODO bad call to non-public API
-        for key_node, values in array._items():
+        for key_node, value in array._items():
             for key in evaluator.eval_element(key_node):
                 if precedence.is_string(key):
-                    dct[key.obj] += values
+                    dct[key.obj].append(value)
 
     else:
         if func is not None:

jedi/evaluate/pep0484.py

@@ -0,0 +1,195 @@
+"""
+PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
+through function annotations. There is a strong suggestion in this document
+that only the type of type hinting defined in PEP0484 should be allowed
+as annotations in future python versions.
+
+The (initial / probably incomplete) implementation todo list for pep-0484:
+v Function parameter annotations with builtin/custom type classes
+v Function returntype annotations with builtin/custom type classes
+v Function parameter annotations with strings (forward reference)
+v Function return type annotations with strings (forward reference)
+v Local variable type hints
+v Assigned types: `Url = str\ndef get(url:Url) -> str:`
+v Type hints in `with` statements
+x Stub files support
+x support `@no_type_check` and `@no_type_check_decorator`
+x support for typing.cast() operator
+x support for type hint comments for functions, `# type: (int, str) -> int`.
+    See comment from Guido https://github.com/davidhalter/jedi/issues/662
+"""
+
+import itertools
+
+import os
+from jedi.parser import \
+    Parser, load_grammar, ParseError, ParserWithRecovery, tree
+from jedi.evaluate.cache import memoize_default
+from jedi.common import unite
+from jedi.evaluate import compiled
+from jedi import debug
+from jedi import _compatibility
+import re
+
+
+def _evaluate_for_annotation(evaluator, annotation, index=None):
+    """
+    Evaluates a string-node, looking for an annotation
+    If index is not None, the annotation is expected to be a tuple
+    and we're interested in that index
+    """
+    if annotation is not None:
+        definitions = evaluator.eval_element(
+            _fix_forward_reference(evaluator, annotation))
+        if index is not None:
+            definitions = list(itertools.chain.from_iterable(
+                definition.py__getitem__(index) for definition in definitions
+                if definition.type == 'tuple' and
+                len(list(definition.py__iter__())) >= index))
+        return list(itertools.chain.from_iterable(
+            evaluator.execute(d) for d in definitions))
+    else:
+        return []
+
+
+def _fix_forward_reference(evaluator, node):
+    evaled_nodes = evaluator.eval_element(node)
+    if len(evaled_nodes) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (node, evaled_nodes))
+        return node
+    evaled_node = list(evaled_nodes)[0]
+    if isinstance(evaled_node, compiled.CompiledObject) and \
+            isinstance(evaled_node.obj, str):
+        try:
+            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
+                       start_symbol='eval_input')
+            newnode = p.get_parsed_node()
+        except ParseError:
+            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
+            return node
+        else:
+            module = node.get_parent_until()
+            p.position_modifier.line = module.end_pos[0]
+            newnode.parent = module
+            return newnode
+    else:
+        return node
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def follow_param(evaluator, param):
+    annotation = param.annotation()
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def find_return_types(evaluator, func):
+    annotation = func.py__annotations__().get("return", None)
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+_typing_module = None
+
+
+def _get_typing_replacement_module():
+    """
+    The idea is to return our jedi replacement for the PEP-0484 typing module
+    as discussed at https://github.com/davidhalter/jedi/issues/663
+    """
+    global _typing_module
+    if _typing_module is None:
+        typing_path = \
+            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
+        with open(typing_path) as f:
+            code = _compatibility.unicode(f.read())
+        p = ParserWithRecovery(load_grammar(), code)
+        _typing_module = p.module
+    return _typing_module
+
+
+def get_types_for_typing_module(evaluator, typ, node):
+    from jedi.evaluate.iterable import FakeSequence
+    if not typ.base.get_parent_until().name.value == "typing":
+        return None
+    # we assume that any class using [] in a module called
+    # "typing" with a name for which we have a replacement
+    # should be replaced by that class. This is not 100%
+    # airtight but I don't have a better idea to check that it's
+    # actually the PEP-0484 typing module and not some other
+    if tree.is_node(node, "subscriptlist"):
+        nodes = node.children[::2]  # skip the commas
+    else:
+        nodes = [node]
+    del node
+
+    nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
+
+    # hacked in Union and Optional, since it's hard to do nicely in parsed code
+    if typ.name.value == "Union":
+        return unite(evaluator.eval_element(node) for node in nodes)
+    if typ.name.value == "Optional":
+        return evaluator.eval_element(nodes[0])
+
+    typing = _get_typing_replacement_module()
+    factories = evaluator.find_types(typing, "factory")
+    assert len(factories) == 1
+    factory = list(factories)[0]
+    assert factory
+    function_body_nodes = factory.children[4].children
+    valid_classnames = set(child.name.value
+                           for child in function_body_nodes
+                           if isinstance(child, tree.Class))
+    if typ.name.value not in valid_classnames:
+        return None
+    compiled_classname = compiled.create(evaluator, typ.name.value)
+
+    args = FakeSequence(evaluator, nodes, "tuple")
+
+    result = evaluator.execute_evaluated(factory, compiled_classname, args)
+    return result
+
+
+def find_type_from_comment_hint_for(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[1], name)
+
+
+def find_type_from_comment_hint_with(evaluator, node, name):
+    assert len(node.children[1].children) == 3, \
+        "Can only be here when children[1] is 'foo() as f'"
+    return _find_type_from_comment_hint(
+        evaluator, node, node.children[1].children[2], name)
+
+
+def find_type_from_comment_hint_assign(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[0], name)
+
+
+def _find_type_from_comment_hint(evaluator, node, varlist, name):
+    index = None
+    if varlist.type in ("testlist_star_expr", "exprlist"):
+        # something like "a, b = 1, 2"
+        index = 0
+        for child in varlist.children:
+            if child == name:
+                break
+            if child.type == "operator":
+                continue
+            index += 1
+        else:
+            return []
+
+    comment = node.get_following_comment_same_line()
+    if comment is None:
+        return []
+    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
+    if not match:
+        return []
+    annotation = tree.String(
+        tree.zero_position_modifier,
+        repr(str(match.group(1).strip())),
+        node.start_pos)
+    annotation.parent = node.parent
+    return _evaluate_for_annotation(evaluator, annotation, index)

jedi/evaluate/precedence.py

@@ -6,8 +6,7 @@ import operator
 from jedi._compatibility import unicode
 from jedi.parser import tree
 from jedi import debug
-from jedi.evaluate.compiled import (CompiledObject, create, builtin,
-                                    keyword_from_value, true_obj, false_obj)
+from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
 from jedi.evaluate import analysis
 
 # Maps Python syntax to the operator module.
@@ -23,16 +22,19 @@ COMPARISON_OPERATORS = {
 }
 
 
-def _literals_to_types(evaluator, result):
+def literals_to_types(evaluator, result):
     # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
     # int(), float(), etc).
-    for i, r in enumerate(result):
-        if is_literal(r):
+    new_result = set()
+    for typ in result:
+        if is_literal(typ):
             # Literals are only valid as long as the operations are
             # correct. Otherwise add a value-free instance.
-            cls = builtin.get_by_name(r.name.get_code())
-            result[i] = evaluator.execute(cls)[0]
-    return list(set(result))
+            cls = builtin_from_name(evaluator, typ.name.value)
+            new_result |= evaluator.execute(cls)
+        else:
+            new_result.add(typ)
+    return new_result
 
 
 def calculate_children(evaluator, children):
@@ -64,21 +66,21 @@ def calculate_children(evaluator, children):
 
 
 def calculate(evaluator, left_result, operator, right_result):
-    result = []
+    result = set()
     if not left_result or not right_result:
         # illegal slices e.g. cause left/right_result to be None
-        result = (left_result or []) + (right_result or [])
-        result = _literals_to_types(evaluator, result)
+        result = (left_result or set()) | (right_result or set())
+        result = literals_to_types(evaluator, result)
     else:
         # I don't think there's a reasonable chance that a string
         # operation is still correct, once we pass something like six
         # objects.
         if len(left_result) * len(right_result) > 6:
-            result = _literals_to_types(evaluator, left_result + right_result)
+            result = literals_to_types(evaluator, left_result | right_result)
         else:
             for left in left_result:
                 for right in right_result:
-                    result += _element_calculate(evaluator, left, operator, right)
+                    result |= _element_calculate(evaluator, left, operator, right)
     return result
 
 
@@ -94,7 +96,7 @@ def factor_calculate(evaluator, types, operator):
             value = typ.py__bool__()
             if value is None:  # Uncertainty.
                 return
-            yield keyword_from_value(not value)
+            yield create(evaluator, not value)
         else:
             yield typ
 
@@ -130,21 +132,21 @@ def _element_calculate(evaluator, left, operator, right):
     if operator == '*':
         # for iterables, ignore * operations
         if isinstance(left, iterable.Array) or is_string(left):
-            return [left]
+            return set([left])
         elif isinstance(right, iterable.Array) or is_string(right):
-            return [right]
+            return set([right])
     elif operator == '+':
         if l_is_num and r_is_num or is_string(left) and is_string(right):
-            return [create(evaluator, left.obj + right.obj)]
+            return set([create(evaluator, left.obj + right.obj)])
         elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
-            return [iterable.MergedArray(evaluator, (left, right))]
+            return set([iterable.MergedArray(evaluator, (left, right))])
     elif operator == '-':
         if l_is_num and r_is_num:
-            return [create(evaluator, left.obj - right.obj)]
+            return set([create(evaluator, left.obj - right.obj)])
     elif operator == '%':
         # With strings and numbers the left type typically remains. Except for
         # `int() % float()`.
-        return [left]
+        return set([left])
     elif operator in COMPARISON_OPERATORS:
         operation = COMPARISON_OPERATORS[operator]
         if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
@@ -153,12 +155,14 @@ def _element_calculate(evaluator, left, operator, right):
             right = right.obj
 
         try:
-            return [keyword_from_value(operation(left, right))]
+            result = operation(left, right)
         except TypeError:
             # Could be True or False.
-            return [true_obj, false_obj]
+            return set([create(evaluator, True), create(evaluator, False)])
+        else:
+            return set([create(evaluator, result)])
     elif operator == 'in':
-        return []
+        return set()
 
     def check(obj):
         """Checks if a Jedi object is either a float or an int."""
@@ -171,4 +175,4 @@ def _element_calculate(evaluator, left, operator, right):
         analysis.add(evaluator, 'type-error-operation', operator,
                      message % (left, right))
 
-    return [left, right]
+    return set([left, right])

jedi/evaluate/recursion.py

@@ -9,7 +9,6 @@ count the function calls.
 """
 from jedi import debug
 from jedi import settings
-from jedi.evaluate import compiled
 from jedi.evaluate import iterable
 
 
@@ -18,7 +17,7 @@ def recursion_decorator(func):
         rec_detect = evaluator.recursion_detector
         # print stmt, len(self.node_statements())
         if rec_detect.push_stmt(stmt):
-            return []
+            return set()
         else:
             result = func(evaluator, stmt, *args, **kwargs)
             rec_detect.pop_stmt()
@@ -31,12 +30,13 @@ class RecursionDetector(object):
     A decorator to detect recursions in statements. In a recursion a statement
     at the same place, in the same module may not be executed two times.
     """
-    def __init__(self):
+    def __init__(self, evaluator):
         self.top = None
         self.current = None
+        self._evaluator = evaluator
 
     def push_stmt(self, stmt):
-        self.current = _RecursionNode(stmt, self.current)
+        self.current = _RecursionNode(self._evaluator, stmt, self.current)
         check = self._check_recursion()
         if check:
             debug.warning('catched stmt recursion: %s against %s @%s', stmt,
@@ -71,7 +71,8 @@ class RecursionDetector(object):
 
 class _RecursionNode(object):
     """ A node of the RecursionDecorator. """
-    def __init__(self, stmt, parent):
+    def __init__(self, evaluator, stmt, parent):
+        self._evaluator = evaluator
         self.script = stmt.get_parent_until()
         self.position = stmt.start_pos
         self.parent = parent
@@ -80,7 +81,7 @@ class _RecursionNode(object):
         # Don't check param instances, they are not causing recursions
         # The same's true for the builtins, because the builtins are really
         # simple.
-        self.is_ignored = self.script == compiled.builtin
+        self.is_ignored = self.script == self._evaluator.BUILTINS
 
     def __eq__(self, other):
         if not other:
@@ -95,7 +96,7 @@ def execution_recursion_decorator(func):
     def run(execution, **kwargs):
         detector = execution._evaluator.execution_recursion_detector
         if detector.push_execution(execution):
-            result = []
+            result = set()
         else:
             result = func(execution, **kwargs)
         detector.pop_execution()
@@ -107,51 +108,51 @@ def execution_recursion_decorator(func):
 class ExecutionRecursionDetector(object):
     """
     Catches recursions of executions.
-    It is designed like a Singelton. Only one instance should exist.
     """
-    def __init__(self):
+    def __init__(self, evaluator):
         self.recursion_level = 0
         self.parent_execution_funcs = []
         self.execution_funcs = set()
         self.execution_count = 0
+        self._evaluator = evaluator
 
     def __call__(self, execution):
         debug.dbg('Execution recursions: %s', execution, self.recursion_level,
                   self.execution_count, len(self.execution_funcs))
         if self.check_recursion(execution):
-            result = []
+            result = set()
         else:
             result = self.func(execution)
         self.pop_execution()
         return result
 
-    def pop_execution(cls):
-        cls.parent_execution_funcs.pop()
-        cls.recursion_level -= 1
+    def pop_execution(self):
+        self.parent_execution_funcs.pop()
+        self.recursion_level -= 1
 
-    def push_execution(cls, execution):
-        in_par_execution_funcs = execution.base in cls.parent_execution_funcs
-        in_execution_funcs = execution.base in cls.execution_funcs
-        cls.recursion_level += 1
-        cls.execution_count += 1
-        cls.execution_funcs.add(execution.base)
-        cls.parent_execution_funcs.append(execution.base)
+    def push_execution(self, execution):
+        in_par_execution_funcs = execution.base in self.parent_execution_funcs
+        in_execution_funcs = execution.base in self.execution_funcs
+        self.recursion_level += 1
+        self.execution_count += 1
+        self.execution_funcs.add(execution.base)
+        self.parent_execution_funcs.append(execution.base)
 
-        if cls.execution_count > settings.max_executions:
+        if self.execution_count > settings.max_executions:
             return True
 
         if isinstance(execution.base, (iterable.Array, iterable.Generator)):
             return False
         module = execution.get_parent_until()
-        if module == compiled.builtin:
+        if module == self._evaluator.BUILTINS:
             return False
 
         if in_par_execution_funcs:
-            if cls.recursion_level > settings.max_function_recursion_level:
+            if self.recursion_level > settings.max_function_recursion_level:
                 return True
         if in_execution_funcs and \
-                len(cls.execution_funcs) > settings.max_until_execution_unique:
+                len(self.execution_funcs) > settings.max_until_execution_unique:
             return True
-        if cls.execution_count > settings.max_executions_without_builtins:
+        if self.execution_count > settings.max_executions_without_builtins:
             return True
         return False

jedi/evaluate/representation.py

@@ -17,13 +17,18 @@ and others. Here's a list:
 ====================================== ========================================
 **Method**                             **Description**
 -------------------------------------- ----------------------------------------
-py__call__(evaluator, params: Array)   On callable objects, returns types.
+py__call__(params: Array)              On callable objects, returns types.
 py__bool__()                           Returns True/False/None; None means that
                                        there's no certainty.
-py__bases__(evaluator)                 Returns a list of base classes.
-py__mro__(evaluator)                   Returns a list of classes (the mro).
-py__getattribute__(evaluator, name)    Returns a list of attribute values. The
-                                       name can be str or Name.
+py__bases__()                          Returns a list of base classes.
+py__mro__()                            Returns a list of classes (the mro).
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__getitem__(index: int/str)          Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__file__()                           Only on modules.
+py__package__()                        Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
 ====================================== ========================================
 
 __
@@ -34,16 +39,18 @@ import imp
 import re
 from itertools import chain
 
-from jedi._compatibility import use_metaclass, unicode, Python3Method
+from jedi._compatibility import use_metaclass, unicode, Python3Method, is_py3
 from jedi.parser import tree
 from jedi import debug
 from jedi import common
 from jedi.cache import underscore_memoization, cache_star_import
 from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT
 from jedi.evaluate import compiled
+from jedi.evaluate.compiled import mixed
 from jedi.evaluate import recursion
 from jedi.evaluate import iterable
 from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
 from jedi.evaluate import helpers
 from jedi.evaluate import param
 from jedi.evaluate import flow_analysis
@@ -83,7 +90,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         self.is_generated = is_generated
 
         if base.name.get_code() in ['list', 'set'] \
-                and compiled.builtin == base.get_parent_until():
+                and evaluator.BUILTINS == base.get_parent_until():
             # compare the module path with the builtin name.
             self.var_args = iterable.check_array_instances(evaluator, self)
         elif not is_generated:
@@ -96,10 +103,13 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
             else:
                 evaluator.execute(method, self.var_args)
 
+    def is_class(self):
+        return False
+
     @property
     def py__call__(self):
-        def actual(evaluator, params):
-            return evaluator.execute(method, params)
+        def actual(params):
+            return self._evaluator.execute(method, params)
 
         try:
             method = self.get_subscope_by_name('__call__')
@@ -109,7 +119,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
 
         return actual
 
-    def py__class__(self, evaluator):
+    def py__class__(self):
         return self.base
 
     def py__bool__(self):
@@ -153,8 +163,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
                     sub = self._get_method_execution(sub)
             for name_list in sub.names_dict.values():
                 for name in name_list:
-                    if name.value == self_name and name.prev_sibling() is None:
-                        trailer = name.next_sibling()
+                    if name.value == self_name and name.get_previous_sibling() is None:
+                        trailer = name.get_next_sibling()
                         if tree.is_node(trailer, 'trailer') \
                                 and len(trailer.children) == 2 \
                                 and trailer.children[0] == '.':
@@ -176,17 +186,18 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         """ Throws a KeyError if there's no method. """
         # Arguments in __get__ descriptors are obj, class.
         # `method` is the new parent of the array, don't know if that's good.
-        args = [obj, obj.base] if isinstance(obj, Instance) else [compiled.none_obj, obj]
+        none_obj = compiled.create(self._evaluator, None)
+        args = [obj, obj.base] if isinstance(obj, Instance) else [none_obj, obj]
         try:
             return self.execute_subscope_by_name('__get__', *args)
         except KeyError:
-            return [self]
+            return set([self])
 
     @memoize_default()
     def names_dicts(self, search_global):
         yield self._self_names_dict()
 
-        for s in self.base.py__mro__(self._evaluator)[1:]:
+        for s in self.base.py__mro__()[1:]:
             if not isinstance(s, compiled.CompiledObject):
                 # Compiled objects don't have `self.` names.
                 for inst in self._evaluator.execute(s):
@@ -195,21 +206,35 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         for names_dict in self.base.names_dicts(search_global=False, is_instance=True):
             yield LazyInstanceDict(self._evaluator, self, names_dict)
 
-    def get_index_types(self, evaluator, index_array):
-        indexes = iterable.create_indexes_or_slices(self._evaluator, index_array)
-        if any([isinstance(i, iterable.Slice) for i in indexes]):
-            # Slice support in Jedi is very marginal, at the moment, so just
-            # ignore them in case of __getitem__.
-            # TODO support slices in a more general way.
-            indexes = []
-
+    def py__getitem__(self, index):
         try:
             method = self.get_subscope_by_name('__getitem__')
         except KeyError:
             debug.warning('No __getitem__, cannot access the array.')
-            return []
+            return set()
         else:
-            return self._evaluator.execute(method, [iterable.AlreadyEvaluated(indexes)])
+            index_obj = compiled.create(self._evaluator, index)
+            return self._evaluator.execute_evaluated(method, index_obj)
+
+    def py__iter__(self):
+        try:
+            method = self.get_subscope_by_name('__iter__')
+        except KeyError:
+            debug.warning('No __iter__ on %s.' % self)
+            return
+        else:
+            iters = self._evaluator.execute(method)
+            for generator in iters:
+                if isinstance(generator, Instance):
+                    # `__next__` logic.
+                    name = '__next__' if is_py3 else 'next'
+                    try:
+                        yield generator.execute_subscope_by_name(name)
+                    except KeyError:
+                        debug.warning('Instance has no __next__ function in %s.', generator)
+                else:
+                    for typ in generator.py__iter__():
+                        yield typ
 
     @property
     @underscore_memoization
@@ -228,8 +253,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)):
         dec = ''
         if self.decorates is not None:
             dec = " decorates " + repr(self.decorates)
-        return "<e%s of %s(%s)%s>" % (type(self).__name__, self.base,
-                                      self.var_args, dec)
+        return "<%s of %s(%s)%s>" % (type(self).__name__, self.base,
+                                     self.var_args, dec)
 
 
 class LazyInstanceDict(object):
@@ -355,13 +380,13 @@ class InstanceElement(use_metaclass(CachedMetaClass, tree.Base)):
         """
         return self.var.is_scope()
 
-    def py__call__(self, evaluator, params):
+    def py__call__(self, params):
         if isinstance(self.var, compiled.CompiledObject):
             # This check is a bit strange, but CompiledObject itself is a bit
             # more complicated than we would it actually like to be.
-            return self.var.py__call__(evaluator, params)
+            return self.var.py__call__(params)
         else:
-            return Function.py__call__(self, evaluator, params)
+            return Function.py__call__(self, params)
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.var)
@@ -398,7 +423,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         self.base = base
 
     @memoize_default(default=())
-    def py__mro__(self, evaluator):
+    def py__mro__(self):
         def add(cls):
             if cls not in mro:
                 mro.append(cls)
@@ -406,7 +431,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         mro = [self]
         # TODO Do a proper mro resolution. Currently we are just listing
         # classes. However, it's a complicated algorithm.
-        for cls in self.py__bases__(self._evaluator):
+        for cls in self.py__bases__():
             # TODO detect for TypeError: duplicate base class str,
             # e.g.  `class X(str, str): pass`
             try:
@@ -426,24 +451,24 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
                 pass
             else:
                 add(cls)
-                for cls_new in mro_method(evaluator):
+                for cls_new in mro_method():
                     add(cls_new)
         return tuple(mro)
 
     @memoize_default(default=())
-    def py__bases__(self, evaluator):
+    def py__bases__(self):
         arglist = self.base.get_super_arglist()
         if arglist:
             args = param.Arguments(self._evaluator, arglist)
             return list(chain.from_iterable(args.eval_args()))
         else:
-            return [compiled.object_obj]
+            return [compiled.create(self._evaluator, object)]
 
-    def py__call__(self, evaluator, params):
-        return [Instance(evaluator, self, params)]
+    def py__call__(self, params):
+        return set([Instance(self._evaluator, self, params)])
 
-    def py__getattribute__(self, name):
-        return self._evaluator.find_types(self, name)
+    def py__class__(self):
+        return compiled.create(self._evaluator, type)
 
     @property
     def params(self):
@@ -453,7 +478,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         if search_global:
             yield self.names_dict
         else:
-            for scope in self.py__mro__(self._evaluator):
+            for scope in self.py__mro__():
                 if isinstance(scope, compiled.CompiledObject):
                     yield scope.names_dicts(False, is_instance)[0]
                 else:
@@ -463,7 +488,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)):
         return True
 
     def get_subscope_by_name(self, name):
-        for s in self.py__mro__(self._evaluator):
+        for s in self.py__mro__():
             for sub in reversed(s.subscopes):
                 if sub.name.value == name:
                         return sub
@@ -527,8 +552,10 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
                 # Create param array.
                 if isinstance(f, Function):
                     old_func = f  # TODO this is just hacky. change.
-                else:
+                elif f.type == 'funcdef':
                     old_func = Function(self._evaluator, f, is_decorated=True)
+                else:
+                    old_func = f
 
                 wrappers = self._evaluator.execute_evaluated(decorator, old_func)
                 if not len(wrappers):
@@ -538,7 +565,7 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
                     # TODO resolve issue with multiple wrappers -> multiple types
                     debug.warning('multiple wrappers found %s %s',
                                   self.base_func, wrappers)
-                f = wrappers[0]
+                f = list(wrappers)[0]
                 if isinstance(f, (Instance, Function)):
                     f.decorates = self
 
@@ -549,15 +576,33 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
         if search_global:
             yield self.names_dict
         else:
-            for names_dict in compiled.magic_function_class.names_dicts(False):
+            scope = compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS')
+            for names_dict in scope.names_dicts(False):
                 yield names_dict
 
     @Python3Method
-    def py__call__(self, evaluator, params):
+    def py__call__(self, params):
         if self.base.is_generator():
-            return [iterable.Generator(evaluator, self, params)]
+            return set([iterable.Generator(self._evaluator, self, params)])
         else:
-            return FunctionExecution(evaluator, self, params).get_return_types()
+            return FunctionExecution(self._evaluator, self, params).get_return_types()
+
+    @memoize_default()
+    def py__annotations__(self):
+        parser_func = self.base
+        return_annotation = parser_func.annotation()
+        if return_annotation:
+            dct = {'return': return_annotation}
+        else:
+            dct = {}
+        for function_param in parser_func.params:
+            param_annotation = function_param.annotation()
+            if param_annotation is not None:
+                dct[function_param.name.value] = param_annotation
+        return dct
+
+    def py__class__(self):
+        return compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS')
 
     def __getattr__(self, name):
         return getattr(self.base_func, name)
@@ -588,11 +633,22 @@ class FunctionExecution(Executed):
     def __init__(self, evaluator, base, *args, **kwargs):
         super(FunctionExecution, self).__init__(evaluator, base, *args, **kwargs)
         self._copy_dict = {}
-        new_func = helpers.deep_ast_copy(base.base_func, self, self._copy_dict)
-        self.children = new_func.children
-        self.names_dict = new_func.names_dict
+        funcdef = base.base_func
+        if isinstance(funcdef, mixed.MixedObject):
+            # The extra information in mixed is not needed anymore. We can just
+            # unpack it and give it the tree object.
+            funcdef = funcdef.definition
 
-    @memoize_default(default=())
+        # Just overwrite the old version. We don't need it anymore.
+        funcdef = helpers.deep_ast_copy(funcdef, new_elements=self._copy_dict)
+        for child in funcdef.children:
+            if child.type not in ('operator', 'keyword'):
+                # Not all nodes are properly copied by deep_ast_copy.
+                child.parent = self
+        self.children = funcdef.children
+        self.names_dict = funcdef.names_dict
+
+    @memoize_default(default=set())
     @recursion.execution_recursion_decorator
     def get_return_types(self, check_yields=False):
         func = self.base
@@ -607,26 +663,87 @@ class FunctionExecution(Executed):
             # If we do have listeners, that means that there's not a regular
             # execution ongoing. In this case Jedi is interested in the
             # inserted params, not in the actual execution of the function.
-            return []
+            return set()
 
         if check_yields:
-            types = []
+            types = set()
             returns = self.yields
         else:
             returns = self.returns
-            types = list(docstrings.find_return_types(self._evaluator, func))
+            types = set(docstrings.find_return_types(self._evaluator, func))
+            types |= set(pep0484.find_return_types(self._evaluator, func))
 
         for r in returns:
             check = flow_analysis.break_check(self._evaluator, self, r)
             if check is flow_analysis.UNREACHABLE:
                 debug.dbg('Return unreachable: %s', r)
             else:
-                types += self._evaluator.eval_element(r.children[1])
+                if check_yields:
+                    types |= iterable.unite(self._eval_yield(r))
+                else:
+                    types |= self._evaluator.eval_element(r.children[1])
             if check is flow_analysis.REACHABLE:
                 debug.dbg('Return reachable: %s', r)
                 break
         return types
 
+    def _eval_yield(self, yield_expr):
+        element = yield_expr.children[1]
+        if element.type == 'yield_arg':
+            # It must be a yield from.
+            yield_from_types = self._evaluator.eval_element(element.children[1])
+            for result in iterable.py__iter__(self._evaluator, yield_from_types, element):
+                yield result
+        else:
+            yield self._evaluator.eval_element(element)
+
+    @recursion.execution_recursion_decorator
+    def get_yield_types(self):
+        yields = self.yields
+        stopAt = tree.ForStmt, tree.WhileStmt, FunctionExecution, tree.IfStmt
+        for_parents = [(x, x.get_parent_until((stopAt))) for x in yields]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self \
+                    and for_stmt.defines_one_name():  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self:
+                yields_order.append((None, [yield_]))
+            else:
+                yield self.get_return_types(check_yields=True)
+                return
+            last_for_stmt = for_stmt
+
+        evaluator = self._evaluator
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._eval_yield(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_input_node()
+                for_types = evaluator.eval_element(input_node)
+                ordered = iterable.py__iter__(evaluator, for_types, input_node)
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    evaluator.predefined_if_name_dict_dict[for_stmt] = dct
+                    for yield_in_same_for_stmt in yields:
+                        for result in self._eval_yield(yield_in_same_for_stmt):
+                            yield result
+                    del evaluator.predefined_if_name_dict_dict[for_stmt]
+
     def names_dicts(self, search_global):
         yield self.names_dict
 
@@ -646,50 +763,28 @@ class FunctionExecution(Executed):
     def name_for_position(self, position):
         return tree.Function.name_for_position(self, position)
 
-    def _copy_list(self, lst):
-        """
-        Copies a list attribute of a parser Function. Copying is very
-        expensive, because it is something like `copy.deepcopy`. However, these
-        copied objects can be used for the executions, as if they were in the
-        execution.
-        """
-        objects = []
-        for element in lst:
-            self._scope_copy(element.parent)
-            copied = helpers.deep_ast_copy(element, self._copy_dict)
-            objects.append(copied)
-        return objects
-
     def __getattr__(self, name):
         if name not in ['start_pos', 'end_pos', 'imports', 'name', 'type']:
             raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
         return getattr(self.base, name)
 
-    def _scope_copy(self, scope):
-        raise NotImplementedError
-        """ Copies a scope (e.g. `if foo:`) in an execution """
-        if scope != self.base.base_func:
-            # Just make sure the parents been copied.
-            self._scope_copy(scope.parent)
-            helpers.deep_ast_copy(scope, self._copy_dict)
-
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def returns(self):
         return tree.Scope._search_in_scope(self, tree.ReturnStmt)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def yields(self):
         return tree.Scope._search_in_scope(self, tree.YieldExpr)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def statements(self):
         return tree.Scope._search_in_scope(self, tree.ExprStmt)
 
     @common.safe_property
-    @memoize_default([])
+    @memoize_default()
     def subscopes(self):
         return tree.Scope._search_in_scope(self, tree.Scope)
 
@@ -742,7 +837,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
     @memoize_default()
     def _module_attributes_dict(self):
         def parent_callback():
-            return self._evaluator.execute(compiled.create(self._evaluator, str))[0]
+            # Create a string type object (without a defined string in it):
+            return list(self._evaluator.execute(compiled.create(self._evaluator, str)))[0]
 
         names = ['__file__', '__package__', '__doc__', '__name__']
         # All the additional module attributes are strings.
@@ -757,7 +853,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
     def _get_init_directory(self):
         for suffix, _, _ in imp.get_suffixes():
             ending = '__init__' + suffix
-            if self.py__file__().endswith(ending):
+            py__file__ = self.py__file__()
+            if py__file__ is not None and py__file__.endswith(ending):
                 # Remove the ending, including the separator.
                 return self.py__file__()[:-len(ending) - 1]
         return None
@@ -850,6 +947,9 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
 
         return names
 
+    def py__class__(self):
+        return compiled.get_special_object(self._evaluator, 'MODULE_CLASS')
+
     def __getattr__(self, name):
         return getattr(self._module, name)
 

jedi/evaluate/stdlib.py

@@ -5,32 +5,37 @@ understand them with Jedi.
 To add a new implementation, create a function and add it to the
 ``_implemented`` dict at the bottom of this module.
 
+Note that this module exists only to implement very specific functionality in
+the standard library. The usual way to understand the standard library is the
+compiled module that returns the types for C-builtins.
 """
 import collections
 import re
 
 from jedi._compatibility import unicode
+from jedi.common import unite
 from jedi.evaluate import compiled
 from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
 from jedi import debug
 from jedi.evaluate import precedence
 from jedi.evaluate import param
+from jedi.evaluate import analysis
 
 
 class NotInStdLib(LookupError):
     pass
 
 
-def execute(evaluator, obj, params):
+def execute(evaluator, obj, arguments):
     try:
         obj_name = str(obj.name)
     except AttributeError:
         pass
     else:
-        if obj.parent == compiled.builtin:
+        if obj.parent == evaluator.BUILTINS:
             module_name = 'builtins'
         elif isinstance(obj.parent, tree.Module):
             module_name = str(obj.parent.name)
@@ -39,22 +44,24 @@ def execute(evaluator, obj, params):
 
         # for now we just support builtin functions.
         try:
-            return _implemented[module_name][obj_name](evaluator, obj, params)
+            func = _implemented[module_name][obj_name]
         except KeyError:
             pass
+        else:
+            return func(evaluator, obj, arguments)
     raise NotInStdLib()
 
 
-def _follow_param(evaluator, params, index):
+def _follow_param(evaluator, arguments, index):
     try:
-        key, values = list(params.unpack())[index]
+        key, values = list(arguments.unpack())[index]
     except IndexError:
-        return []
+        return set()
     else:
-        return iterable.unite(evaluator.eval_element(v) for v in values)
+        return unite(evaluator.eval_element(v) for v in values)
 
 
-def argument_clinic(string, want_obj=False, want_scope=False):
+def argument_clinic(string, want_obj=False, want_scope=False, want_arguments=False):
     """
     Works like Argument Clinic (PEP 436), to validate function params.
     """
@@ -77,17 +84,22 @@ def argument_clinic(string, want_obj=False, want_scope=False):
 
     def f(func):
         def wrapper(evaluator, obj, arguments):
+            debug.dbg('builtin start %s' % obj, color='MAGENTA')
             try:
                 lst = list(arguments.eval_argument_clinic(clinic_args))
             except ValueError:
-                return []
+                return set()
             else:
                 kwargs = {}
                 if want_scope:
                     kwargs['scope'] = arguments.scope()
                 if want_obj:
                     kwargs['obj'] = obj
+                if want_arguments:
+                    kwargs['arguments'] = arguments
                 return func(evaluator, *lst, **kwargs)
+            finally:
+                debug.dbg('builtin end', color='MAGENTA')
 
         return wrapper
     return f
@@ -95,7 +107,6 @@ def argument_clinic(string, want_obj=False, want_scope=False):
 
 @argument_clinic('object, name[, default], /')
 def builtins_getattr(evaluator, objects, names, defaults=None):
-    types = []
     # follow the first param
     for obj in objects:
         if not isinstance(obj, (er.Instance, er.Class, tree.Module, compiled.CompiledObject)):
@@ -108,16 +119,16 @@ def builtins_getattr(evaluator, objects, names, defaults=None):
             else:
                 debug.warning('getattr called without str')
                 continue
-    return types
+    return set()
 
 
 @argument_clinic('object[, bases, dict], /')
 def builtins_type(evaluator, objects, bases, dicts):
     if bases or dicts:
-        # metaclass... maybe someday...
-        return []
+        # It's a type creation... maybe someday...
+        return set()
     else:
-        return [o.base for o in objects if isinstance(o, er.Instance)]
+        return set([o.py__class__() for o in objects])
 
 
 class SuperInstance(er.Instance):
@@ -140,17 +151,21 @@ def builtins_super(evaluator, types, objects, scope):
                 cls = er.Class(evaluator, cls)
             elif isinstance(cls, er.Instance):
                 cls = cls.base
-            su = cls.py__bases__(evaluator)
+            su = cls.py__bases__()
             if su:
                 return evaluator.execute(su[0])
-    return []
+    return set()
 
 
-@argument_clinic('sequence, /', want_obj=True)
-def builtins_reversed(evaluator, sequences, obj):
-    # Unpack the iterator values
-    objects = tuple(iterable.get_iterator_types(sequences))
-    rev = [iterable.AlreadyEvaluated([o]) for o in reversed(objects)]
+@argument_clinic('sequence, /', want_obj=True, want_arguments=True)
+def builtins_reversed(evaluator, sequences, obj, arguments):
+    # While we could do without this variable (just by using sequences), we
+    # want static analysis to work well. Therefore we need to generated the
+    # values again.
+    first_arg = next(arguments.as_tuple())[0]
+    ordered = list(iterable.py__iter__(evaluator, sequences, first_arg))
+
+    rev = [iterable.AlreadyEvaluated(o) for o in reversed(ordered)]
     # Repack iterator values and then run it the normal way. This is
     # necessary, because `reversed` is a function and autocompletion
     # would fail in certain cases like `reversed(x).__iter__` if we
@@ -158,35 +173,43 @@ def builtins_reversed(evaluator, sequences, obj):
     rev = iterable.AlreadyEvaluated(
         [iterable.FakeSequence(evaluator, rev, 'list')]
     )
-    return [er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))]
+    return set([er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))])
 
 
-@argument_clinic('obj, type, /')
-def builtins_isinstance(evaluator, objects, types):
+@argument_clinic('obj, type, /', want_arguments=True)
+def builtins_isinstance(evaluator, objects, types, arguments):
     bool_results = set([])
     for o in objects:
         try:
-            mro_func = o.py__class__(evaluator).py__mro__
+            mro_func = o.py__class__().py__mro__
         except AttributeError:
             # This is temporary. Everything should have a class attribute in
             # Python?! Maybe we'll leave it here, because some numpy objects or
             # whatever might not.
-            return [compiled.true_obj, compiled.false_obj]
+            return set([compiled.create(True), compiled.create(False)])
 
-        mro = mro_func(evaluator)
+        mro = mro_func()
 
         for cls_or_tup in types:
             if cls_or_tup.is_class():
                 bool_results.add(cls_or_tup in mro)
-            else:
+            elif str(cls_or_tup.name) == 'tuple' \
+                    and cls_or_tup.get_parent_scope() == evaluator.BUILTINS:
                 # Check for tuples.
-                classes = iterable.get_iterator_types([cls_or_tup])
+                classes = unite(cls_or_tup.py__iter__())
                 bool_results.add(any(cls in mro for cls in classes))
+            else:
+                _, nodes = list(arguments.unpack())[1]
+                for node in nodes:
+                    message = 'TypeError: isinstance() arg 2 must be a ' \
+                              'class, type, or tuple of classes and types, ' \
+                              'not %s.' % cls_or_tup
+                    analysis.add(evaluator, 'type-error-isinstance', node, message)
 
-    return [compiled.keyword_from_value(x) for x in bool_results]
+    return set(compiled.create(evaluator, x) for x in bool_results)
 
 
-def collections_namedtuple(evaluator, obj, params):
+def collections_namedtuple(evaluator, obj, arguments):
     """
     Implementation of the namedtuple function.
 
@@ -198,20 +221,21 @@ def collections_namedtuple(evaluator, obj, params):
     """
     # Namedtuples are not supported on Python 2.6
     if not hasattr(collections, '_class_template'):
-        return []
+        return set()
 
     # Process arguments
-    name = _follow_param(evaluator, params, 0)[0].obj
-    _fields = _follow_param(evaluator, params, 1)[0]
+    # TODO here we only use one of the types, we should use all.
+    name = list(_follow_param(evaluator, arguments, 0))[0].obj
+    _fields = list(_follow_param(evaluator, arguments, 1))[0]
     if isinstance(_fields, compiled.CompiledObject):
         fields = _fields.obj.replace(',', ' ').split()
     elif isinstance(_fields, iterable.Array):
         try:
-            fields = [v.obj for v in _fields.values()]
+            fields = [v.obj for v in unite(_fields.py__iter__())]
         except AttributeError:
-            return []
+            return set()
     else:
-        return []
+        return set()
 
     # Build source
     source = collections._class_template.format(
@@ -225,8 +249,8 @@ def collections_namedtuple(evaluator, obj, params):
     )
 
     # Parse source
-    generated_class = Parser(evaluator.grammar, unicode(source)).module.subscopes[0]
-    return [er.Class(evaluator, generated_class)]
+    generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0]
+    return set([er.Class(evaluator, generated_class)])
 
 
 @argument_clinic('first, /')
@@ -247,8 +271,8 @@ _implemented = {
         'deepcopy': _return_first_param,
     },
     'json': {
-        'load': lambda *args: [],
-        'loads': lambda *args: [],
+        'load': lambda *args: set(),
+        'loads': lambda *args: set(),
     },
     'collections': {
         'namedtuple': collections_namedtuple,

jedi/evaluate/sys_path.py

@@ -5,11 +5,11 @@ from jedi.evaluate.site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.evaluate.cache import memoize_default
 from jedi import debug
 from jedi import common
-from jedi import cache
+from jedi.parser.utils import load_parser, save_parser
 
 
 def get_venv_path(venv):
@@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt):
     for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
         try:
             assert operator in ['=', '+=']
-            assert tree.is_node(assignee, 'power') and len(assignee.children) > 1
+            assert tree.is_node(assignee, 'power', 'atom_expr') and \
+                len(assignee.children) > 1
             c = assignee.children
             assert c[0].type == 'name' and c[0].value == 'sys'
             trailer = c[1]
@@ -118,11 +119,13 @@ def _paths_from_assignment(evaluator, expr_stmt):
         except AssertionError:
             continue
 
-        from jedi.evaluate.iterable import get_iterator_types
+        from jedi.evaluate.iterable import py__iter__
         from jedi.evaluate.precedence import is_string
-        for val in get_iterator_types(evaluator.eval_statement(expr_stmt)):
-            if is_string(val):
-                yield val.obj
+        types = evaluator.eval_element(expr_stmt)
+        for types in py__iter__(evaluator, types, expr_stmt):
+            for typ in types:
+                if is_string(typ):
+                    yield typ.obj
 
 
 def _paths_from_list_modifications(module_path, trailer1, trailer2):
@@ -150,7 +153,7 @@ def _check_module(evaluator, module):
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
-            if tree.is_node(power, 'power'):
+            if tree.is_node(power, 'power', 'atom_expr'):
                 c = power.children
                 if isinstance(c[0], tree.Name) and c[0].value == 'sys' \
                         and tree.is_node(c[1], 'trailer'):
@@ -183,6 +186,7 @@ def sys_path_with_modifications(evaluator, module):
         return list(evaluator.sys_path)
 
     curdir = os.path.abspath(os.curdir)
+    #TODO why do we need a chdir?
     with common.ignored(OSError):
         os.chdir(os.path.dirname(module.path))
 
@@ -207,11 +211,11 @@ def _get_paths_from_buildout_script(evaluator, buildout_script):
             debug.dbg('Error trying to read buildout_script: %s', buildout_script)
             return
 
-        p = Parser(evaluator.grammar, source, buildout_script)
-        cache.save_parser(buildout_script, p)
+        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
+        save_parser(buildout_script, p)
         return p.module
 
-    cached = cache.load_parser(buildout_script)
+    cached = load_parser(buildout_script)
     module = cached and cached.module or load(buildout_script)
     if not module:
         return
@@ -271,8 +275,8 @@ def _get_buildout_scripts(module_path):
                 firstline = f.readline()
                 if firstline.startswith('#!') and 'python' in firstline:
                     extra_module_paths.append(filepath)
-        except IOError as e:
-            # either permission error or race cond. because file got deleted
+        except (UnicodeDecodeError, IOError) as e:
+            # Probably a binary file; permission error or race cond. because file got deleted
             # ignore
             debug.warning(unicode(e))
             continue

jedi/parser/__init__.py

@@ -20,9 +20,8 @@ import re
 
 from jedi.parser import tree as pt
 from jedi.parser import tokenize
-from jedi.parser import token
 from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER,
-                               STRING, OP, ERRORTOKEN)
+                               STRING)
 from jedi.parser.pgen2.pgen import generate_grammar
 from jedi.parser.pgen2.parse import PgenParser
 
@@ -35,45 +34,30 @@ STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \
 _loaded_grammars = {}
 
 
-def load_grammar(file='grammar3.4'):
+class ParseError(Exception):
+    """
+    Signals you that the code you fed the Parser was not correct Python code.
+    """
+
+
+def load_grammar(version='3.4'):
     # For now we only support two different Python syntax versions: The latest
     # Python 3 and Python 2. This may change.
-    if file.startswith('grammar3'):
-        file = 'grammar3.4'
-    else:
-        file = 'grammar2.7'
+    if version in ('3.2', '3.3'):
+        version = '3.4'
+    elif version == '2.6':
+        version = '2.7'
+
+    file = 'grammar' + version + '.txt'
 
     global _loaded_grammars
-    path = os.path.join(os.path.dirname(__file__), file) + '.txt'
+    path = os.path.join(os.path.dirname(__file__), file)
     try:
         return _loaded_grammars[path]
     except KeyError:
         return _loaded_grammars.setdefault(path, generate_grammar(path))
 
 
-class ErrorStatement(object):
-    def __init__(self, stack, next_token, position_modifier, next_start_pos):
-        self.stack = stack
-        self._position_modifier = position_modifier
-        self.next_token = next_token
-        self._next_start_pos = next_start_pos
-
-    @property
-    def next_start_pos(self):
-        s = self._next_start_pos
-        return s[0] + self._position_modifier.line, s[1]
-
-    @property
-    def first_pos(self):
-        first_type, nodes = self.stack[0]
-        return nodes[0].start_pos
-
-    @property
-    def first_type(self):
-        first_type, nodes = self.stack[0]
-        return first_type
-
-
 class ParserSyntaxError(object):
     def __init__(self, message, position):
         self.message = message
@@ -81,91 +65,96 @@ class ParserSyntaxError(object):
 
 
 class Parser(object):
-    """
-    This class is used to parse a Python file, it then divides them into a
-    class structure of different scopes.
+    AST_MAPPING = {
+        'expr_stmt': pt.ExprStmt,
+        'classdef': pt.Class,
+        'funcdef': pt.Function,
+        'file_input': pt.Module,
+        'import_name': pt.ImportName,
+        'import_from': pt.ImportFrom,
+        'break_stmt': pt.KeywordStatement,
+        'continue_stmt': pt.KeywordStatement,
+        'return_stmt': pt.ReturnStmt,
+        'raise_stmt': pt.KeywordStatement,
+        'yield_expr': pt.YieldExpr,
+        'del_stmt': pt.KeywordStatement,
+        'pass_stmt': pt.KeywordStatement,
+        'global_stmt': pt.GlobalStmt,
+        'nonlocal_stmt': pt.KeywordStatement,
+        'print_stmt': pt.KeywordStatement,
+        'assert_stmt': pt.AssertStmt,
+        'if_stmt': pt.IfStmt,
+        'with_stmt': pt.WithStmt,
+        'for_stmt': pt.ForStmt,
+        'while_stmt': pt.WhileStmt,
+        'try_stmt': pt.TryStmt,
+        'comp_for': pt.CompFor,
+        'decorator': pt.Decorator,
+        'lambdef': pt.Lambda,
+        'old_lambdef': pt.Lambda,
+        'lambdef_nocond': pt.Lambda,
+    }
 
-    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
-    :param source: The codebase for the parser. Must be unicode.
-    :param module_path: The path of the module in the file system, may be None.
-    :type module_path: str
-    :param top_module: Use this module as a parent instead of `self.module`.
-    """
-    def __init__(self, grammar, source, module_path=None, tokenizer=None):
-        self._ast_mapping = {
-            'expr_stmt': pt.ExprStmt,
-            'classdef': pt.Class,
-            'funcdef': pt.Function,
-            'file_input': pt.Module,
-            'import_name': pt.ImportName,
-            'import_from': pt.ImportFrom,
-            'break_stmt': pt.KeywordStatement,
-            'continue_stmt': pt.KeywordStatement,
-            'return_stmt': pt.ReturnStmt,
-            'raise_stmt': pt.KeywordStatement,
-            'yield_expr': pt.YieldExpr,
-            'del_stmt': pt.KeywordStatement,
-            'pass_stmt': pt.KeywordStatement,
-            'global_stmt': pt.GlobalStmt,
-            'nonlocal_stmt': pt.KeywordStatement,
-            'assert_stmt': pt.AssertStmt,
-            'if_stmt': pt.IfStmt,
-            'with_stmt': pt.WithStmt,
-            'for_stmt': pt.ForStmt,
-            'while_stmt': pt.WhileStmt,
-            'try_stmt': pt.TryStmt,
-            'comp_for': pt.CompFor,
-            'decorator': pt.Decorator,
-            'lambdef': pt.Lambda,
-            'old_lambdef': pt.Lambda,
-            'lambdef_nocond': pt.Lambda,
-        }
+    def __init__(self, grammar, source, start_symbol='file_input',
+                 tokenizer=None, start_parsing=True):
+        # Todo Remove start_parsing (with False)
 
-        self.syntax_errors = []
-
-        self._global_names = []
-        self._omit_dedent_list = []
-        self._indent_counter = 0
-        self._last_failed_start_pos = (0, 0)
-
-        # TODO do print absolute import detection here.
-        #try:
-        #    del python_grammar_no_print_statement.keywords["print"]
-        #except KeyError:
-        #    pass  # Doesn't exist in the Python 3 grammar.
-
-        #if self.options["print_function"]:
-        #    python_grammar = pygram.python_grammar_no_print_statement
-        #else:
         self._used_names = {}
         self._scope_names_stack = [{}]
-        self._error_statement_stacks = []
-
-        added_newline = False
-        # The Python grammar needs a newline at the end of each statement.
-        if not source.endswith('\n'):
-            source += '\n'
-            added_newline = True
+        self._last_failed_start_pos = (0, 0)
+        self._global_names = []
 
         # For the fast parser.
         self.position_modifier = pt.PositionModifier()
-        p = PgenParser(grammar, self.convert_node, self.convert_leaf,
-                       self.error_recovery)
-        tokenizer = tokenizer or tokenize.source_tokens(source)
-        self.module = p.parse(self._tokenize(tokenizer))
-        if self.module.type != 'file_input':
+
+        self._added_newline = False
+        # The Python grammar needs a newline at the end of each statement.
+        if not source.endswith('\n') and start_symbol == 'file_input':
+            source += '\n'
+            self._added_newline = True
+
+        self._start_symbol = start_symbol
+        self._grammar = grammar
+
+        self._parsed = None
+
+        if start_parsing:
+            if tokenizer is None:
+                tokenizer = tokenize.source_tokens(source, use_exact_op_types=True)
+            self.parse(tokenizer)
+
+    def parse(self, tokenizer):
+        if self._parsed is not None:
+            return self._parsed
+
+        start_number = self._grammar.symbol2number[self._start_symbol]
+        pgen_parser = PgenParser(
+            self._grammar, self.convert_node, self.convert_leaf,
+            self.error_recovery, start_number
+        )
+
+        try:
+            self._parsed = pgen_parser.parse(tokenizer)
+        finally:
+            self.stack = pgen_parser.stack
+
+        if self._start_symbol == 'file_input' != self._parsed.type:
             # If there's only one statement, we get back a non-module. That's
             # not what we want, we want a module, so we add it here:
-            self.module = self.convert_node(grammar,
-                                            grammar.symbol2number['file_input'],
-                                            [self.module])
+            self._parsed = self.convert_node(self._grammar,
+                                             self._grammar.symbol2number['file_input'],
+                                             [self._parsed])
 
-        if added_newline:
+        if self._added_newline:
             self.remove_last_newline()
-        self.module.used_names = self._used_names
-        self.module.path = module_path
-        self.module.global_names = self._global_names
-        self.module.error_statement_stacks = self._error_statement_stacks
+
+    def get_parsed_node(self):
+        # TODO rename to get_root_node
+        return self._parsed
+
+    def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
+                       add_token_callback):
+        raise ParseError
 
     def convert_node(self, grammar, type, children):
         """
@@ -177,7 +166,7 @@ class Parser(object):
         """
         symbol = grammar.number2symbol[type]
         try:
-            new_node = self._ast_mapping[symbol](children)
+            new_node = Parser.AST_MAPPING[symbol](children)
         except KeyError:
             new_node = pt.Node(symbol, children)
 
@@ -206,7 +195,7 @@ class Parser(object):
         return new_node
 
     def convert_leaf(self, grammar, type, value, prefix, start_pos):
-        #print('leaf', value, pytree.type_repr(type))
+        # print('leaf', repr(value), token.tok_name[type])
         if type == tokenize.NAME:
             if value in grammar.keywords:
                 if value in ('def', 'class', 'lambda'):
@@ -227,156 +216,51 @@ class Parser(object):
             return pt.Number(self.position_modifier, value, start_pos, prefix)
         elif type in (NEWLINE, ENDMARKER):
             return pt.Whitespace(self.position_modifier, value, start_pos, prefix)
+        elif type == INDENT:
+            return pt.Indent(self.position_modifier, value, start_pos, prefix)
+        elif type == DEDENT:
+            return pt.Dedent(self.position_modifier, value, start_pos, prefix)
         else:
             return pt.Operator(self.position_modifier, value, start_pos, prefix)
 
-    def error_recovery(self, grammar, stack, typ, value, start_pos, prefix,
-                       add_token_callback):
-        """
-        This parser is written in a dynamic way, meaning that this parser
-        allows using different grammars (even non-Python). However, error
-        recovery is purely written for Python.
-        """
-        def current_suite(stack):
-            # For now just discard everything that is not a suite or
-            # file_input, if we detect an error.
-            for index, (dfa, state, (typ, nodes)) in reversed(list(enumerate(stack))):
-                # `suite` can sometimes be only simple_stmt, not stmt.
-                symbol = grammar.number2symbol[typ]
-                if symbol == 'file_input':
-                    break
-                elif symbol == 'suite' and len(nodes) > 1:
-                    # suites without an indent in them get discarded.
-                    break
-                elif symbol == 'simple_stmt' and len(nodes) > 1:
-                    # simple_stmt can just be turned into a Node, if there are
-                    # enough statements. Ignore the rest after that.
-                    break
-            return index, symbol, nodes
-
-        index, symbol, nodes = current_suite(stack)
-        if symbol == 'simple_stmt':
-            index -= 2
-            (_, _, (typ, suite_nodes)) = stack[index]
-            symbol = grammar.number2symbol[typ]
-            suite_nodes.append(pt.Node(symbol, list(nodes)))
-            # Remove
-            nodes[:] = []
-            nodes = suite_nodes
-            stack[index]
-
-        #print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index)
-        self._stack_removal(grammar, stack, index + 1, value, start_pos)
-        if typ == INDENT:
-            # For every deleted INDENT we have to delete a DEDENT as well.
-            # Otherwise the parser will get into trouble and DEDENT too early.
-            self._omit_dedent_list.append(self._indent_counter)
-
-        if value in ('import', 'from', 'class', 'def', 'try', 'while', 'return'):
-            # Those can always be new statements.
-            add_token_callback(typ, value, prefix, start_pos)
-        elif typ == DEDENT and symbol == 'suite':
-            # Close the current suite, with DEDENT.
-            # Note that this may cause some suites to not contain any
-            # statements at all. This is contrary to valid Python syntax. We
-            # keep incomplete suites in Jedi to be able to complete param names
-            # or `with ... as foo` names. If we want to use this parser for
-            # syntax checks, we have to check in a separate turn if suites
-            # contain statements or not. However, a second check is necessary
-            # anyway (compile.c does that for Python), because Python's grammar
-            # doesn't stop you from defining `continue` in a module, etc.
-            add_token_callback(typ, value, prefix, start_pos)
-
-    def _stack_removal(self, grammar, stack, start_index, value, start_pos):
-        def clear_names(children):
-            for c in children:
-                try:
-                    clear_names(c.children)
-                except AttributeError:
-                    if isinstance(c, pt.Name):
-                        try:
-                            self._scope_names_stack[-1][c.value].remove(c)
-                            self._used_names[c.value].remove(c)
-                        except ValueError:
-                            pass  # This may happen with CompFor.
-
-        for dfa, state, node in stack[start_index:]:
-            clear_names(children=node[1])
-
-        failed_stack = []
-        found = False
-        for dfa, state, (typ, nodes) in stack[start_index:]:
-            if nodes:
-                found = True
-            if found:
-                symbol = grammar.number2symbol[typ]
-                failed_stack.append((symbol, nodes))
-            if nodes and nodes[0] in ('def', 'class', 'lambda'):
-                self._scope_names_stack.pop()
-        if failed_stack:
-            err = ErrorStatement(failed_stack, value, self.position_modifier, start_pos)
-            self._error_statement_stacks.append(err)
-
-        self._last_failed_start_pos = start_pos
-
-        stack[start_index:] = []
-
-    def _tokenize(self, tokenizer):
-        for typ, value, start_pos, prefix in tokenizer:
-            #print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
-            if typ == DEDENT:
-                # We need to count indents, because if we just omit any DEDENT,
-                # we might omit them in the wrong place.
-                o = self._omit_dedent_list
-                if o and o[-1] == self._indent_counter:
-                    o.pop()
-                    continue
-
-                self._indent_counter -= 1
-            elif typ == INDENT:
-                self._indent_counter += 1
-            elif typ == ERRORTOKEN:
-                self._add_syntax_error('Strange token', start_pos)
-                continue
-
-            if typ == OP:
-                typ = token.opmap[value]
-            yield typ, value, prefix, start_pos
-
-    def _add_syntax_error(self, message, position):
-        self.syntax_errors.append(ParserSyntaxError(message, position))
-
-    def __repr__(self):
-        return "<%s: %s>" % (type(self).__name__, self.module)
-
     def remove_last_newline(self):
         """
         In all of this we need to work with _start_pos, because if we worked
         with start_pos, we would need to check the position_modifier as well
         (which is accounted for in the start_pos property).
         """
-        endmarker = self.module.children[-1]
+        endmarker = self._parsed.children[-1]
         # The newline is either in the endmarker as a prefix or the previous
         # leaf as a newline token.
-        if endmarker.prefix.endswith('\n'):
-            endmarker.prefix = endmarker.prefix[:-1]
-            last_line = re.sub('.*\n', '', endmarker.prefix)
-            endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line)
+        prefix = endmarker.prefix
+        if prefix.endswith('\n'):
+            endmarker.prefix = prefix = prefix[:-1]
+            last_end = 0
+            if '\n' not in prefix:
+                # Basically if the last line doesn't end with a newline. we
+                # have to add the previous line's end_position.
+                try:
+                    last_end = endmarker.get_previous_leaf().end_pos[1]
+                except IndexError:
+                    pass
+            last_line = re.sub('.*\n', '', prefix)
+            endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line)
         else:
             try:
-                newline = endmarker.get_previous()
+                newline = endmarker.get_previous_leaf()
             except IndexError:
                 return  # This means that the parser is empty.
             while True:
                 if newline.value == '':
                     # Must be a DEDENT, just continue.
                     try:
-                        newline = newline.get_previous()
+                        newline = newline.get_previous_leaf()
                     except IndexError:
                         # If there's a statement that fails to be parsed, there
                         # will be no previous leaf. So just ignore it.
                         break
                 elif newline.value != '\n':
+                    # TODO REMOVE, error recovery was simplified.
                     # This may happen if error correction strikes and removes
                     # a whole statement including '\n'.
                     break
@@ -391,3 +275,127 @@ class Parser(object):
                     else:
                         endmarker._start_pos = newline._start_pos
                     break
+
+
+class ParserWithRecovery(Parser):
+    """
+    This class is used to parse a Python file, it then divides them into a
+    class structure of different scopes.
+
+    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    :param source: The codebase for the parser. Must be unicode.
+    :param module_path: The path of the module in the file system, may be None.
+    :type module_path: str
+    """
+    def __init__(self, grammar, source, module_path=None, tokenizer=None):
+        self.syntax_errors = []
+
+        self._omit_dedent_list = []
+        self._indent_counter = 0
+
+        # TODO do print absolute import detection here.
+        # try:
+        #     del python_grammar_no_print_statement.keywords["print"]
+        # except KeyError:
+        #     pass  # Doesn't exist in the Python 3 grammar.
+
+        # if self.options["print_function"]:
+        #     python_grammar = pygram.python_grammar_no_print_statement
+        # else:
+        super(ParserWithRecovery, self).__init__(grammar, source, tokenizer=tokenizer)
+
+        self.module = self._parsed
+        self.module.used_names = self._used_names
+        self.module.path = module_path
+        self.module.global_names = self._global_names
+
+    def parse(self, tokenizer):
+        return super(ParserWithRecovery, self).parse(self._tokenize(self._tokenize(tokenizer)))
+
+    def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
+                       add_token_callback):
+        """
+        This parser is written in a dynamic way, meaning that this parser
+        allows using different grammars (even non-Python). However, error
+        recovery is purely written for Python.
+        """
+        def current_suite(stack):
+            # For now just discard everything that is not a suite or
+            # file_input, if we detect an error.
+            for index, (dfa, state, (type_, nodes)) in reversed(list(enumerate(stack))):
+                # `suite` can sometimes be only simple_stmt, not stmt.
+                symbol = grammar.number2symbol[type_]
+                if symbol == 'file_input':
+                    break
+                elif symbol == 'suite' and len(nodes) > 1:
+                    # suites without an indent in them get discarded.
+                    break
+                elif symbol == 'simple_stmt' and len(nodes) > 1:
+                    # simple_stmt can just be turned into a Node, if there are
+                    # enough statements. Ignore the rest after that.
+                    break
+            return index, symbol, nodes
+
+        index, symbol, nodes = current_suite(stack)
+        if symbol == 'simple_stmt':
+            index -= 2
+            (_, _, (type_, suite_nodes)) = stack[index]
+            symbol = grammar.number2symbol[type_]
+            suite_nodes.append(pt.Node(symbol, list(nodes)))
+            # Remove
+            nodes[:] = []
+            nodes = suite_nodes
+            stack[index]
+
+        # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index)
+        if self._stack_removal(grammar, stack, arcs, index + 1, value, start_pos):
+            add_token_callback(typ, value, start_pos, prefix)
+        else:
+            if typ == INDENT:
+                # For every deleted INDENT we have to delete a DEDENT as well.
+                # Otherwise the parser will get into trouble and DEDENT too early.
+                self._omit_dedent_list.append(self._indent_counter)
+            else:
+                error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix)
+                stack[-1][2][1].append(error_leaf)
+
+    def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):
+        failed_stack = []
+        found = False
+        all_nodes = []
+        for dfa, state, (typ, nodes) in stack[start_index:]:
+            if nodes:
+                found = True
+            if found:
+                symbol = grammar.number2symbol[typ]
+                failed_stack.append((symbol, nodes))
+                all_nodes += nodes
+            if nodes and nodes[0] in ('def', 'class', 'lambda'):
+                self._scope_names_stack.pop()
+        if failed_stack:
+            stack[start_index - 1][2][1].append(pt.ErrorNode(all_nodes))
+
+        self._last_failed_start_pos = start_pos
+
+        stack[start_index:] = []
+        return failed_stack
+
+    def _tokenize(self, tokenizer):
+        for typ, value, start_pos, prefix in tokenizer:
+            # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
+            if typ == DEDENT:
+                # We need to count indents, because if we just omit any DEDENT,
+                # we might omit them in the wrong place.
+                o = self._omit_dedent_list
+                if o and o[-1] == self._indent_counter:
+                    o.pop()
+                    continue
+
+                self._indent_counter -= 1
+            elif typ == INDENT:
+                self._indent_counter += 1
+
+            yield typ, value, start_pos, prefix
+
+    def __repr__(self):
+        return "<%s: %s>" % (type(self).__name__, self.module)

jedi/parser/fast.py

@@ -8,9 +8,9 @@ from itertools import chain
 
 from jedi._compatibility import use_metaclass
 from jedi import settings
-from jedi.parser import Parser
+from jedi.parser import ParserWithRecovery
 from jedi.parser import tree
-from jedi import cache
+from jedi.parser.utils import underscore_memoization, parser_cache
 from jedi import debug
 from jedi.parser.tokenize import (source_tokens, NEWLINE,
                                   ENDMARKER, INDENT, DEDENT)
@@ -36,7 +36,7 @@ class FastModule(tree.Module):
             pass  # It was never used.
 
     @property
-    @cache.underscore_memoization
+    @underscore_memoization
     def used_names(self):
         return MergedNamesDict([m.used_names for m in self.modules])
 
@@ -45,21 +45,22 @@ class FastModule(tree.Module):
         return [name for m in self.modules for name in m.global_names]
 
     @property
-    def error_statement_stacks(self):
-        return [e for m in self.modules for e in m.error_statement_stacks]
+    def error_statements(self):
+        return [e for m in self.modules for e in m.error_statements]
 
     def __repr__(self):
         return "<fast.%s: %s@%s-%s>" % (type(self).__name__, self.name,
                                         self.start_pos[0], self.end_pos[0])
 
-    # To avoid issues with with the `parser.Parser`, we need setters that do
-    # nothing, because if pickle comes along and sets those values.
+    # To avoid issues with with the `parser.ParserWithRecovery`, we need
+    # setters that do nothing, because if pickle comes along and sets those
+    # values.
     @global_names.setter
     def global_names(self, value):
         pass
 
-    @error_statement_stacks.setter
-    def error_statement_stacks(self, value):
+    @error_statements.setter
+    def error_statements(self, value):
         pass
 
     @used_names.setter
@@ -99,10 +100,10 @@ class CachedFastParser(type):
     """ This is a metaclass for caching `FastParser`. """
     def __call__(self, grammar, source, module_path=None):
         if not settings.fast_parser:
-            return Parser(grammar, source, module_path)
+            return ParserWithRecovery(grammar, source, module_path)
 
-        pi = cache.parser_cache.get(module_path, None)
-        if pi is None or isinstance(pi.parser, Parser):
+        pi = parser_cache.get(module_path, None)
+        if pi is None or isinstance(pi.parser, ParserWithRecovery):
             p = super(CachedFastParser, self).__call__(grammar, source, module_path)
         else:
             p = pi.parser  # pi is a `cache.ParserCacheItem`
@@ -123,14 +124,21 @@ class ParserNode(object):
         try:
             # With fast_parser we have either 1 subscope or only statements.
             self._content_scope = parser.module.subscopes[0]
+            # A parsed node's content will be in the first indent, because
+            # everything that's parsed is within this subscope.
+            self._is_class_or_def = True
         except IndexError:
             self._content_scope = parser.module
+            self._is_class_or_def = False
         else:
             self._rewrite_last_newline()
 
         # We need to be able to reset the original children of a parser.
         self._old_children = list(self._content_scope.children)
 
+    def is_root_node(self):
+        return self.parent is None
+
     def _rewrite_last_newline(self):
         """
         The ENDMARKER can contain a newline in the prefix. However this prefix
@@ -181,25 +189,30 @@ class ParserNode(object):
             dcts.insert(0, self._content_scope.names_dict)
             self._content_scope.names_dict = MergedNamesDict(dcts)
 
-    def parent_until_indent(self, indent=None):
-        if (indent is None or self._indent >= indent) and self.parent is not None:
-            self.close()
-            return self.parent.parent_until_indent(indent)
-        return self
-
     @property
     def _indent(self):
-        if not self.parent:
+        if self.is_root_node():
             return 0
 
         return self.parser.module.children[0].start_pos[1]
 
-    def add_node(self, node, line_offset):
-        """Adding a node means adding a node that was already added earlier"""
+    def add_node(self, node, start_line, indent):
+        """
+        Adding a node means adding a node that was either just parsed or one
+        that can be reused.
+        """
+        # Content that is not a subscope can never be part of the current node,
+        # because it's basically a sister node, that sits next to it and not
+        # within it.
+        if (self._indent >= indent or not self._is_class_or_def) and \
+                not self.is_root_node():
+            self.close()
+            return self.parent.add_node(node, start_line, indent)
+
         # Changing the line offsets is very important, because if they don't
         # fit, all the start_pos values will be wrong.
         m = node.parser.module
-        node.parser.position_modifier.line = line_offset
+        node.parser.position_modifier.line = start_line - 1
         self._fast_module.modules.append(m)
         node.parent = self
 
@@ -223,7 +236,7 @@ class ParserNode(object):
             for y in n.all_sub_nodes():
                 yield y
 
-    @cache.underscore_memoization  # Should only happen once!
+    @underscore_memoization  # Should only happen once!
     def remove_last_newline(self):
         self.parser.remove_last_newline()
 
@@ -244,11 +257,15 @@ class FastParser(use_metaclass(CachedFastParser)):
 
     def _reset_caches(self):
         self.module = FastModule(self.module_path)
-        self.current_node = ParserNode(self.module, self, '')
+        self.root_node = self.current_node = ParserNode(self.module, self, '')
+
+    def get_parsed_node(self):
+        return self.module
 
     def update(self, source):
-        # For testing purposes: It is important that the number of parsers used
-        # can be minimized. With these variables we can test against that.
+        # Variables for testing purposes: It is important that the number of
+        # parsers used can be minimized. With these variables we can test
+        # against that.
         self.number_parsers_used = 0
         self.number_of_splits = 0
         self.number_of_misses = 0
@@ -312,13 +329,13 @@ class FastParser(use_metaclass(CachedFastParser)):
                 current_lines.append(l)  # Just ignore comments and blank lines
                 continue
 
-            if new_indent:
+            if new_indent and not parentheses_level:
                 if indent > indent_list[-2]:
                     # Set the actual indent, not just the random old indent + 1.
                     indent_list[-1] = indent
                 new_indent = False
 
-            while indent <= indent_list[-2]:  # -> dedent
+            while indent < indent_list[-1]:  # -> dedent
                 indent_list.pop()
                 # This automatically resets the flow_indent if there was a
                 # dedent or a flow just on one line (with one simple_stmt).
@@ -348,10 +365,13 @@ class FastParser(use_metaclass(CachedFastParser)):
                     is_decorator = False
 
             parentheses_level = \
-                max(0, (l.count('(') + l.count('[') + l.count('{')
-                        - l.count(')') - l.count(']') - l.count('}')))
+                max(0, (l.count('(') + l.count('[') + l.count('{') -
+                        l.count(')') - l.count(']') - l.count('}')))
 
             current_lines.append(l)
+
+        if previous_line is not None:
+            current_lines.append(previous_line)
         if current_lines:
             yield gen_part()
 
@@ -366,41 +386,44 @@ class FastParser(use_metaclass(CachedFastParser)):
             source += '\n'
             added_newline = True
 
-        next_line_offset = line_offset = 0
+        next_code_part_end_line = code_part_end_line = 1
         start = 0
-        nodes = list(self.current_node.all_sub_nodes())
+        nodes = list(self.root_node.all_sub_nodes())
         # Now we can reset the node, because we have all the old nodes.
-        self.current_node.reset_node()
+        self.root_node.reset_node()
+        self.current_node = self.root_node
         last_end_line = 1
 
         for code_part in self._split_parts(source):
-            next_line_offset += code_part.count('\n')
+            next_code_part_end_line += code_part.count('\n')
             # If the last code part parsed isn't equal to the current end_pos,
             # we know that the parser went further (`def` start in a
             # docstring). So just parse the next part.
-            if line_offset + 1 == last_end_line:
-                self.current_node = self._get_node(code_part, source[start:],
-                                                   line_offset, nodes)
+            if code_part_end_line == last_end_line:
+                self._parse_part(code_part, source[start:], code_part_end_line, nodes)
             else:
+                self.number_of_misses += 1
                 # Means that some lines where not fully parsed. Parse it now.
                 # This is a very rare case. Should only happens with very
                 # strange code bits.
-                self.number_of_misses += 1
-                while last_end_line < next_line_offset + 1:
-                    line_offset = last_end_line - 1
+                while last_end_line < next_code_part_end_line:
+                    code_part_end_line = last_end_line
                     # We could calculate the src in a more complicated way to
                     # make caching here possible as well. However, this is
                     # complicated and error-prone. Since this is not very often
                     # called - just ignore it.
-                    src = ''.join(self._lines[line_offset:])
-                    self.current_node = self._get_node(code_part, src,
-                                                       line_offset, nodes)
+                    src = ''.join(self._lines[code_part_end_line - 1:])
+                    self._parse_part(code_part, src, code_part_end_line, nodes)
                     last_end_line = self.current_node.parser.module.end_pos[0]
-
+                debug.dbg("While parsing %s, starting with line %s wasn't included in split.",
+                          self.module_path, code_part_end_line)
+                #assert code_part_end_line > last_end_line
+                # This means that the parser parsed faster than the last given
+                # `code_part`.
                 debug.dbg('While parsing %s, line %s slowed down the fast parser.',
-                          self.module_path, line_offset + 1)
+                          self.module_path, code_part_end_line)
 
-            line_offset = next_line_offset
+            code_part_end_line = next_code_part_end_line
             start += len(code_part)
 
             last_end_line = self.current_node.parser.module.end_pos[0]
@@ -409,39 +432,41 @@ class FastParser(use_metaclass(CachedFastParser)):
             self.current_node.remove_last_newline()
 
         # Now that the for loop is finished, we still want to close all nodes.
-        self.current_node = self.current_node.parent_until_indent()
-        self.current_node.close()
+        node = self.current_node
+        while node is not None:
+            node.close()
+            node = node.parent
 
         debug.dbg('Parsed %s, with %s parsers in %s splits.'
                   % (self.module_path, self.number_parsers_used,
                      self.number_of_splits))
 
-    def _get_node(self, source, parser_code, line_offset, nodes):
+    def _parse_part(self, source, parser_code, code_part_end_line, nodes):
         """
         Side effect: Alters the list of nodes.
         """
-        indent = len(source) - len(source.lstrip('\t '))
-        self.current_node = self.current_node.parent_until_indent(indent)
-
         h = hash(source)
         for index, node in enumerate(nodes):
             if node.hash == h and node.source == source:
                 node.reset_node()
                 nodes.remove(node)
+                parser_code = source
                 break
         else:
             tokenizer = FastTokenizer(parser_code)
             self.number_parsers_used += 1
-            p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
+            p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
 
-            end = line_offset + p.module.end_pos[0]
-            used_lines = self._lines[line_offset:end - 1]
+            end = code_part_end_line - 1 + p.module.end_pos[0]
+            used_lines = self._lines[code_part_end_line - 1:end - 1]
             code_part_actually_used = ''.join(used_lines)
 
             node = ParserNode(self.module, p, code_part_actually_used)
 
-        self.current_node.add_node(node, line_offset)
-        return node
+        indent = len(parser_code) - len(parser_code.lstrip('\t '))
+
+        self.current_node.add_node(node, code_part_end_line, indent)
+        self.current_node = node
 
 
 class FastTokenizer(object):
@@ -450,7 +475,7 @@ class FastTokenizer(object):
     """
     def __init__(self, source):
         self.source = source
-        self._gen = source_tokens(source)
+        self._gen = source_tokens(source, use_exact_op_types=True)
         self._closed = False
 
         # fast parser options
@@ -501,19 +526,22 @@ class FastTokenizer(object):
                 self._closed = True
             return current
 
-        if value in ('def', 'class') and self._parentheses_level \
-                and re.search(r'\n[ \t]*\Z', prefix):
+        previous_type = self.previous[0]
+        if value in ('def', 'class') and self._parentheses_level:
             # Account for the fact that an open parentheses before a function
             # will reset the parentheses counter, but new lines before will
             # still be ignored. So check the prefix.
 
             # TODO what about flow parentheses counter resets in the tokenizer?
             self._parentheses_level = 0
-            return self._close()
+            # We need to simulate a newline before the indent, because the
+            # open parentheses ignored them.
+            if re.search('\n\s*', prefix):
+                previous_type = NEWLINE
 
         # Parentheses ignore the indentation rules. The other three stand for
         # new lines.
-        if self.previous[0] in (NEWLINE, INDENT, DEDENT) \
+        if previous_type in (NEWLINE, INDENT, DEDENT) \
                 and not self._parentheses_level and typ not in (INDENT, DEDENT):
             if not self._in_flow:
                 if value in FLOWS:

jedi/parser/grammar2.7.txt

@@ -57,7 +57,7 @@ break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
 yield_stmt: yield_expr
-raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]]
+raise_stmt: 'raise' [test [',' test [',' test]]]
 import_stmt: import_name | import_from
 import_name: 'import' dotted_as_names
 # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
@@ -68,7 +68,7 @@ dotted_as_name: dotted_name ['as' NAME]
 import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
-global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 

jedi/parser/grammar3.5.txt

@@ -0,0 +1,154 @@
+# Grammar for Python
+
+# Note:  Changing the grammar specified in this file will most likely
+#        require corresponding changes in the parser module
+#        (../Modules/parsermodule.c).  If you can't make the changes to
+#        that module yourself, please co-ordinate the required changes
+#        with someone who can; ask around on python-dev for help.  Fred
+#        Drake <fdrake@acm.org> will probably be listening there.
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://docs.python.org/devguide/grammar.html
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens
+# skipping python3.5 compatibility, in favour of 3.7 solution
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+tfpdef: NAME [':' test]
+varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
+
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist

jedi/parser/pgen2/parse.py

@@ -18,8 +18,12 @@ how this parsing engine works.
 from jedi.parser import tokenize
 
 
-class ParseError(Exception):
-    """Exception to signal the parser is stuck."""
+class InternalParseError(Exception):
+    """
+    Exception to signal the parser is stuck and error recovery didn't help.
+    Basically this shouldn't happen. It's a sign that something is really
+    wrong.
+    """
 
     def __init__(self, msg, type, value, start_pos):
         Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" %
@@ -30,6 +34,21 @@ class ParseError(Exception):
         self.start_pos = start_pos
 
 
+def token_to_ilabel(grammar, type_, value):
+    # Map from token to label
+    if type_ == tokenize.NAME:
+        # Check for reserved words (keywords)
+        try:
+            return grammar.keywords[value]
+        except KeyError:
+            pass
+
+    try:
+        return grammar.tokens[type_]
+    except KeyError:
+        return None
+
+
 class PgenParser(object):
     """Parser engine.
 
@@ -38,7 +57,7 @@ class PgenParser(object):
     p = Parser(grammar, [converter])  # create instance
     p.setup([start])                  # prepare for parsing
     <for each input token>:
-        if p.addtoken(...):           # parse a token; may raise ParseError
+        if p.addtoken(...):           # parse a token
             break
     root = p.rootnode                 # root of abstract syntax tree
 
@@ -53,14 +72,14 @@ class PgenParser(object):
 
     Parsing is complete when addtoken() returns True; the root of the
     abstract syntax tree can then be retrieved from the rootnode
-    instance variable.  When a syntax error occurs, addtoken() raises
-    the ParseError exception.  There is no error recovery; the parser
-    cannot be used after a syntax error was reported (but it can be
-    reinitialized by calling setup()).
+    instance variable.  When a syntax error occurs, error_recovery()
+    is called. There is no error recovery; the parser cannot be used
+    after a syntax error was reported (but it can be reinitialized by
+    calling setup()).
 
     """
 
-    def __init__(self, grammar, convert_node, convert_leaf, error_recovery):
+    def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start):
         """Constructor.
 
         The grammar argument is a grammar.Grammar instance; see the
@@ -90,8 +109,6 @@ class PgenParser(object):
         self.convert_node = convert_node
         self.convert_leaf = convert_leaf
 
-        # Prepare for parsing.
-        start = self.grammar.start
         # Each stack entry is a tuple: (dfa, state, node).
         # A node is a tuple: (type, children),
         # where children is a list of nodes or None
@@ -102,29 +119,20 @@ class PgenParser(object):
         self.error_recovery = error_recovery
 
     def parse(self, tokenizer):
-        for type, value, prefix, start_pos in tokenizer:
-            if self.addtoken(type, value, prefix, start_pos):
+        for type_, value, start_pos, prefix in tokenizer:
+            if self.addtoken(type_, value, start_pos, prefix):
                 break
         else:
             # We never broke out -- EOF is too soon -- Unfinished statement.
-            self.error_recovery(self.grammar, self.stack, type, value,
-                                start_pos, prefix, self.addtoken)
-            # Add the ENDMARKER again.
-            if not self.addtoken(type, value, prefix, start_pos):
-                raise ParseError("incomplete input", type, value, start_pos)
+            # However, the error recovery might have added the token again, if
+            # the stack is empty, we're fine.
+            if self.stack:
+                raise InternalParseError("incomplete input", type_, value, start_pos)
         return self.rootnode
 
-    def addtoken(self, type, value, prefix, start_pos):
+    def addtoken(self, type_, value, start_pos, prefix):
         """Add a token; return True if this is the end of the program."""
-        # Map from token to label
-        if type == tokenize.NAME:
-            # Check for reserved words (keywords)
-            try:
-                ilabel = self.grammar.keywords[value]
-            except KeyError:
-                ilabel = self.grammar.tokens[type]
-        else:
-            ilabel = self.grammar.tokens[type]
+        ilabel = token_to_ilabel(self.grammar, type_, value)
 
         # Loop until the token is shifted; may raise exceptions
         while True:
@@ -138,7 +146,7 @@ class PgenParser(object):
                     # Look it up in the list of labels
                     assert t < 256
                     # Shift a token; we're done with it
-                    self.shift(type, value, newstate, prefix, start_pos)
+                    self.shift(type_, value, newstate, prefix, start_pos)
                     # Pop while we are in an accept-only state
                     state = newstate
                     while states[state] == [(0, state)]:
@@ -164,36 +172,36 @@ class PgenParser(object):
                     self.pop()
                     if not self.stack:
                         # Done parsing, but another token is input
-                        raise ParseError("too much input", type, value, start_pos)
+                        raise InternalParseError("too much input", type_, value, start_pos)
                 else:
-                    self.error_recovery(self.grammar, self.stack, type,
+                    self.error_recovery(self.grammar, self.stack, arcs, type_,
                                         value, start_pos, prefix, self.addtoken)
                     break
 
-    def shift(self, type, value, newstate, prefix, start_pos):
+    def shift(self, type_, value, newstate, prefix, start_pos):
         """Shift a token.  (Internal)"""
         dfa, state, node = self.stack[-1]
-        newnode = self.convert_leaf(self.grammar, type, value, prefix, start_pos)
+        newnode = self.convert_leaf(self.grammar, type_, value, prefix, start_pos)
         node[-1].append(newnode)
         self.stack[-1] = (dfa, newstate, node)
 
-    def push(self, type, newdfa, newstate):
+    def push(self, type_, newdfa, newstate):
         """Push a nonterminal.  (Internal)"""
         dfa, state, node = self.stack[-1]
-        newnode = (type, [])
+        newnode = (type_, [])
         self.stack[-1] = (dfa, newstate, node)
         self.stack.append((newdfa, 0, newnode))
 
     def pop(self):
         """Pop a nonterminal.  (Internal)"""
-        popdfa, popstate, (type, children) = self.stack.pop()
+        popdfa, popstate, (type_, children) = self.stack.pop()
         # If there's exactly one child, return that child instead of creating a
         # new node.  We still create expr_stmt and file_input though, because a
         # lot of Jedi depends on its logic.
         if len(children) == 1:
             newnode = children[0]
         else:
-            newnode = self.convert_node(self.grammar, type, children)
+            newnode = self.convert_node(self.grammar, type_, children)
 
         try:
             # Equal to:

jedi/parser/token.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-from jedi._compatibility import is_py3
+from jedi._compatibility import is_py3, is_py35
 from token import *
 
 
@@ -24,6 +24,11 @@ else:
     tok_name[ELLIPSIS] = 'ELLIPSIS'
     N_TOKENS += 1
 
+if not is_py35:
+    ATEQUAL = N_TOKENS
+    tok_name[ATEQUAL] = 'ATEQUAL'
+    N_TOKENS += 1
+
 
 
 # Map from operator to number (since tokenize doesn't do this)
@@ -68,6 +73,7 @@ opmap_raw = """\
 %= PERCENTEQUAL
 &= AMPEREQUAL
 |= VBAREQUAL
+@= ATEQUAL
 ^= CIRCUMFLEXEQUAL
 <<= LEFTSHIFTEQUAL
 >>= RIGHTSHIFTEQUAL

jedi/parser/tokenize.py

@@ -13,8 +13,10 @@ from __future__ import absolute_import
 
 import string
 import re
+from collections import namedtuple
 from io import StringIO
-from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER,
+
+from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap,
                                NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT)
 from jedi._compatibility import is_py3
 
@@ -50,7 +52,10 @@ name = r'\w+'
 
 hex_number = r'0[xX][0-9a-fA-F]+'
 bin_number = r'0[bB][01]+'
-oct_number = r'0[oO][0-7]+'
+if is_py3:
+    oct_number = r'0[oO][0-7]+'
+else:
+    oct_number = '0[0-7]+'
 dec_number = r'(?:0+|[1-9][0-9]*)'
 int_number = group(hex_number, bin_number, oct_number, dec_number)
 exponent = r'[eE][-+]?[0-9]+'
@@ -76,7 +81,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
 # recognized as two instances of =).
 operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
-                 r"[+\-*/%&|^=<>]=?",
+                 r"[+\-*@/%&|^=<>]=?",
                  r"~")
 
 bracket = '[][(){}]'
@@ -143,18 +148,33 @@ del _compile
 
 tabsize = 8
 
-ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except',
+# TODO add with?
+ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except',
                        'finally', 'while', 'return')
 
 
-def source_tokens(source):
+class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])):
+    def __repr__(self):
+        annotated_type = tok_name[self.type]
+        return ('TokenInfo(type=%s, string=%r, start=%r, prefix=%r)' %
+                self._replace(type=annotated_type))
+
+    @property
+    def exact_type(self):
+        if self.type == OP and self.string in opmap:
+            return opmap[self.string]
+        else:
+            return self.type
+
+
+def source_tokens(source, use_exact_op_types=False):
     """Generate tokens from a the source code (string)."""
-    source = source + '\n'  # end with \n, because the parser needs it
+    source = source
     readline = StringIO(source).readline
-    return generate_tokens(readline)
+    return generate_tokens(readline, use_exact_op_types)
 
 
-def generate_tokens(readline):
+def generate_tokens(readline, use_exact_op_types=False):
     """
     A heavily modified Python standard library tokenizer.
 
@@ -165,6 +185,7 @@ def generate_tokens(readline):
     paren_level = 0  # count parentheses
     indents = [0]
     lnum = 0
+    max = 0
     numchars = '0123456789'
     contstr = ''
     contline = None
@@ -179,7 +200,7 @@ def generate_tokens(readline):
         line = readline()  # readline returns empty when finished. See StringIO
         if not line:
             if contstr:
-                yield ERRORTOKEN, contstr, contstr_start, prefix
+                yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix)
             break
 
         lnum += 1
@@ -189,7 +210,7 @@ def generate_tokens(readline):
             endmatch = endprog.match(line)
             if endmatch:
                 pos = endmatch.end(0)
-                yield STRING, contstr + line[:pos], contstr_start, prefix
+                yield TokenInfo(STRING, contstr + line[:pos], contstr_start, prefix)
                 contstr = ''
                 contline = None
             else:
@@ -205,7 +226,7 @@ def generate_tokens(readline):
                     # If a literal starts but doesn't end the whole rest of the
                     # line is an error token.
                     txt = line[pos:]
-                yield ERRORTOKEN, txt, (lnum, pos), prefix
+                yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), prefix)
                 pos += 1
                 continue
 
@@ -218,19 +239,23 @@ def generate_tokens(readline):
             if new_line and initial not in '\r\n#':
                 new_line = False
                 if paren_level == 0:
+                    i = 0
+                    while line[i] == '\f':
+                        i += 1
+                        start -= 1
                     if start > indents[-1]:
-                        yield INDENT, '', spos, ''
+                        yield TokenInfo(INDENT, '', spos, '')
                         indents.append(start)
                     while start < indents[-1]:
-                        yield DEDENT, '', spos, ''
+                        yield TokenInfo(DEDENT, '', spos, '')
                         indents.pop()
 
             if (initial in numchars or                      # ordinary number
                     (initial == '.' and token != '.' and token != '...')):
-                yield NUMBER, token, spos, prefix
+                yield TokenInfo(NUMBER, token, spos, prefix)
             elif initial in '\r\n':
                 if not new_line and paren_level == 0:
-                    yield NEWLINE, token, spos, prefix
+                    yield TokenInfo(NEWLINE, token, spos, prefix)
                 else:
                     additional_prefix = prefix + token
                 new_line = True
@@ -243,7 +268,7 @@ def generate_tokens(readline):
                 if endmatch:                                # all on one line
                     pos = endmatch.end(0)
                     token = line[start:pos]
-                    yield STRING, token, spos, prefix
+                    yield TokenInfo(STRING, token, spos, prefix)
                 else:
                     contstr_start = (lnum, start)           # multiple lines
                     contstr = line[start:]
@@ -260,18 +285,18 @@ def generate_tokens(readline):
                     contline = line
                     break
                 else:                                       # ordinary string
-                    yield STRING, token, spos, prefix
+                    yield TokenInfo(STRING, token, spos, prefix)
             elif is_identifier(initial):                      # ordinary name
                 if token in ALWAYS_BREAK_TOKENS:
                     paren_level = 0
                     while True:
                         indent = indents.pop()
                         if indent > start:
-                            yield DEDENT, '', spos, ''
+                            yield TokenInfo(DEDENT, '', spos, '')
                         else:
                             indents.append(indent)
                             break
-                yield NAME, token, spos, prefix
+                yield TokenInfo(NAME, token, spos, prefix)
             elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'):  # continued stmt
                 additional_prefix += prefix + line[start:]
                 break
@@ -280,11 +305,25 @@ def generate_tokens(readline):
                     paren_level += 1
                 elif token in ')]}':
                     paren_level -= 1
-                yield OP, token, spos, prefix
 
-    end_pos = (lnum, max - 1)
+                try:
+                    # This check is needed in any case to check if it's a valid
+                    # operator or just some random unicode character.
+                    exact_type = opmap[token]
+                except KeyError:
+                    exact_type = typ = ERRORTOKEN
+                if use_exact_op_types:
+                    typ = exact_type
+                else:
+                    typ = OP
+                yield TokenInfo(typ, token, spos, prefix)
+
+    if new_line or additional_prefix[-1:] == '\n':
+        end_pos = lnum + 1, 0
+    else:
+        end_pos = lnum, max
     # As the last position we just take the maximally possible position. We
     # remove -1 for the last new line.
     for indent in indents[1:]:
-        yield DEDENT, '', end_pos, ''
-    yield ENDMARKER, '', end_pos, prefix
+        yield TokenInfo(DEDENT, '', end_pos, '')
+    yield TokenInfo(ENDMARKER, '', end_pos, additional_prefix)

jedi/parser/tree.py

@@ -14,8 +14,8 @@ The easiest way to play with this module is to use :class:`parsing.Parser`.
 :attr:`parsing.Parser.module` holds an instance of :class:`Module`:
 
 >>> from jedi._compatibility import u
->>> from jedi.parser import Parser, load_grammar
->>> parser = Parser(load_grammar(), u('import os'), 'example.py')
+>>> from jedi.parser import ParserWithRecovery, load_grammar
+>>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py')
 >>> submodule = parser.module
 >>> submodule
 <Module: example.py@1-1>
@@ -27,16 +27,22 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute
 [<ImportName: import os@1,0>]
 
 See also :attr:`Scope.subscopes` and :attr:`Scope.statements`.
+
+For static analysis purposes there exists a method called
+``nodes_to_execute`` on all nodes and leaves. It's documented in the static
+anaylsis documentation.
 """
 import os
 import re
 from inspect import cleandoc
 from itertools import chain
 import textwrap
+import abc
 
 from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
                                  literal_eval, use_metaclass, unicode)
-from jedi import cache
+from jedi.parser import token
+from jedi.parser.utils import underscore_memoization
 
 
 def is_node(node, *symbol_names):
@@ -140,10 +146,131 @@ class Base(object):
             scope = scope.parent
         return scope
 
+    def get_definition(self):
+        scope = self
+        while scope.parent is not None:
+            parent = scope.parent
+            if scope.isinstance(Node, Name) and parent.type != 'simple_stmt':
+                if scope.type == 'testlist_comp':
+                    try:
+                        if isinstance(scope.children[1], CompFor):
+                            return scope.children[1]
+                    except IndexError:
+                        pass
+                scope = parent
+            else:
+                break
+        return scope
+
+    def assignment_indexes(self):
+        """
+        Returns an array of tuple(int, node) of the indexes that are used in
+        tuple assignments.
+
+        For example if the name is ``y`` in the following code::
+
+            x, (y, z) = 2, ''
+
+        would result in ``[(1, xyz_node), (0, yz_node)]``.
+        """
+        indexes = []
+        node = self.parent
+        compare = self
+        while node is not None:
+            if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'):
+                for i, child in enumerate(node.children):
+                    if child == compare:
+                        indexes.insert(0, (int(i / 2), node))
+                        break
+                else:
+                    raise LookupError("Couldn't find the assignment.")
+            elif isinstance(node, (ExprStmt, CompFor)):
+                break
+
+            compare = node
+            node = node.parent
+        return indexes
+
     def is_scope(self):
         # Default is not being a scope. Just inherit from Scope.
         return False
 
+    @abc.abstractmethod
+    def nodes_to_execute(self, last_added=False):
+        raise NotImplementedError()
+
+    def get_next_sibling(self):
+        """
+        The node immediately following the invocant in their parent's children
+        list. If the invocant does not have a next sibling, it is None
+        """
+        # Can't use index(); we need to test by identity
+        for i, child in enumerate(self.parent.children):
+            if child is self:
+                try:
+                    return self.parent.children[i + 1]
+                except IndexError:
+                    return None
+
+    def get_previous_sibling(self):
+        """
+        The node/leaf immediately preceding the invocant in their parent's
+        children list. If the invocant does not have a previous sibling, it is
+        None.
+        """
+        # Can't use index(); we need to test by identity
+        for i, child in enumerate(self.parent.children):
+            if child is self:
+                if i == 0:
+                    return None
+                return self.parent.children[i - 1]
+
+    def get_previous_leaf(self):
+        """
+        Returns the previous leaf in the parser tree.
+        Raises an IndexError if it's the first element.
+        """
+        node = self
+        while True:
+            c = node.parent.children
+            i = c.index(node)
+            if i == 0:
+                node = node.parent
+                if node.parent is None:
+                    raise IndexError('Cannot access the previous element of the first one.')
+            else:
+                node = c[i - 1]
+                break
+
+        while True:
+            try:
+                node = node.children[-1]
+            except AttributeError:  # A Leaf doesn't have children.
+                return node
+
+    def get_next_leaf(self):
+        """
+        Returns the previous leaf in the parser tree.
+        Raises an IndexError if it's the last element.
+        """
+        node = self
+        while True:
+            c = node.parent.children
+            i = c.index(node)
+            if i == len(c) - 1:
+                node = node.parent
+                if node.parent is None:
+                    raise IndexError('Cannot access the next element of the last one.')
+            else:
+                node = c[i + 1]
+                break
+
+        while True:
+            try:
+                node = node.children[0]
+            except AttributeError:  # A Leaf doesn't have children.
+                return node
+
 
 class Leaf(Base):
     __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix')
@@ -163,6 +290,12 @@ class Leaf(Base):
     def start_pos(self, value):
         self._start_pos = value[0] - self.position_modifier.line, value[1]
 
+    def get_start_pos_of_prefix(self):
+        try:
+            return self.get_previous_leaf().end_pos
+        except IndexError:
+            return 1, 0  # It's the first leaf.
+
     @property
     def end_pos(self):
         return (self._start_pos[0] + self.position_modifier.line,
@@ -172,56 +305,19 @@ class Leaf(Base):
         self._start_pos = (self._start_pos[0] + line_offset,
                            self._start_pos[1] + column_offset)
 
-    def get_previous(self):
-        """
-        Returns the previous leaf in the parser tree.
-        """
-        node = self
-        while True:
-            c = node.parent.children
-            i = c.index(self)
-            if i == 0:
-                node = node.parent
-                if node.parent is None:
-                    raise IndexError('Cannot access the previous element of the first one.')
-            else:
-                node = c[i - 1]
-                break
+    def first_leaf(self):
+        return self
 
-        while True:
-            try:
-                node = node.children[-1]
-            except AttributeError:  # A Leaf doesn't have children.
-                return node
+    def get_code(self, normalized=False, include_prefix=True):
+        if normalized:
+            return self.value
+        if include_prefix:
+            return self.prefix + self.value
+        else:
+            return self.value
 
-    def get_code(self):
-        return self.prefix + self.value
-
-    def next_sibling(self):
-        """
-        The node immediately following the invocant in their parent's children
-        list. If the invocant does not have a next sibling, it is None
-        """
-        # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
-            if child is self:
-                try:
-                    return self.parent.children[i + 1]
-                except IndexError:
-                    return None
-
-    def prev_sibling(self):
-        """
-        The node/leaf immediately preceding the invocant in their parent's
-        children list. If the invocant does not have a previous sibling, it is
-        None.
-        """
-        # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
-            if child is self:
-                if i == 0:
-                    return None
-                return self.parent.children[i - 1]
+    def nodes_to_execute(self, last_added=False):
+        return []
 
     @utf8_repr
     def __repr__(self):
@@ -247,12 +343,20 @@ class LeafWithNewLines(Leaf):
             end_pos_col = len(lines[-1])
         return end_pos_line, end_pos_col
 
+    @utf8_repr
+    def __repr__(self):
+        return "<%s: %r>" % (type(self).__name__, self.value)
+
 
 class Whitespace(LeafWithNewLines):
     """Contains NEWLINE and ENDMARKER tokens."""
     __slots__ = ()
     type = 'whitespace'
 
+    @utf8_repr
+    def __repr__(self):
+        return "<%s: %s>" % (type(self).__name__, repr(self.value))
+
 
 class Name(Leaf):
     """
@@ -272,63 +376,26 @@ class Name(Leaf):
         return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,
                                    self.start_pos[0], self.start_pos[1])
 
-    def get_definition(self):
-        scope = self
-        while scope.parent is not None:
-            parent = scope.parent
-            if scope.isinstance(Node, Name) and parent.type != 'simple_stmt':
-                if scope.type == 'testlist_comp':
-                    try:
-                        if isinstance(scope.children[1], CompFor):
-                            return scope.children[1]
-                    except IndexError:
-                        pass
-                scope = parent
-            else:
-                break
-        return scope
-
     def is_definition(self):
+        if self.parent.type in ('power', 'atom_expr'):
+            # In `self.x = 3` self is not a definition, but x is.
+            return False
+
         stmt = self.get_definition()
         if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'):
             return self == stmt.name
         elif stmt.type == 'for_stmt':
             return self.start_pos < stmt.children[2].start_pos
         elif stmt.type == 'try_stmt':
-            return self.prev_sibling() == 'as'
+            return self.get_previous_sibling() == 'as'
         else:
             return stmt.type in ('expr_stmt', 'import_name', 'import_from',
                                  'comp_for', 'with_stmt') \
                 and self in stmt.get_defined_names()
 
-    def assignment_indexes(self):
-        """
-        Returns an array of ints of the indexes that are used in tuple
-        assignments.
-
-        For example if the name is ``y`` in the following code::
-
-            x, (y, z) = 2, ''
-
-        would result in ``[1, 0]``.
-        """
-        indexes = []
-        node = self.parent
-        compare = self
-        while node is not None:
-            if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'):
-                for i, child in enumerate(node.children):
-                    if child == compare:
-                        indexes.insert(0, int(i / 2))
-                        break
-                else:
-                    raise LookupError("Couldn't find the assignment.")
-            elif isinstance(node, (ExprStmt, CompFor)):
-                break
-
-            compare = node
-            node = node.parent
-        return indexes
+    def nodes_to_execute(self, last_added=False):
+        if last_added is False:
+            yield self
 
 
 class Literal(LeafWithNewLines):
@@ -348,6 +415,16 @@ class String(Literal):
     __slots__ = ()
 
 
+class Indent(Leaf):
+    type = 'indent'
+    __slots__ = ()
+
+
+class Dedent(Leaf):
+    type = 'indent'
+    __slots__ = ()
+
+
 class Operator(Leaf):
     type = 'operator'
     __slots__ = ()
@@ -424,12 +501,20 @@ class BaseNode(Base):
     def start_pos(self):
         return self.children[0].start_pos
 
+    def get_start_pos_of_prefix(self):
+        return self.children[0].get_start_pos_of_prefix()
+
     @property
     def end_pos(self):
         return self.children[-1].end_pos
 
-    def get_code(self):
-        return "".join(c.get_code() for c in self.children)
+    def get_code(self, normalized=False, include_prefix=True):
+        # TODO implement normalized (depending on context).
+        if include_prefix:
+            return "".join(c.get_code(normalized) for c in self.children)
+        else:
+            first = self.children[0].get_code(include_prefix=False)
+            return first + "".join(c.get_code(normalized) for c in self.children[1:])
 
     @Python3Method
     def name_for_position(self, position):
@@ -443,6 +528,21 @@ class BaseNode(Base):
                     return result
         return None
 
+    def get_leaf_for_position(self, position, include_prefixes=False):
+        for c in self.children:
+            if include_prefixes:
+                start_pos = c.get_start_pos_of_prefix()
+            else:
+                start_pos = c.start_pos
+
+            if start_pos <= position <= c.end_pos:
+                try:
+                    return c.get_leaf_for_position(position, include_prefixes)
+                except AttributeError:
+                    return c
+
+        return None
+
     @Python3Method
     def get_statement_for_position(self, pos):
         for c in self.children:
@@ -463,9 +563,54 @@ class BaseNode(Base):
         except AttributeError:
             return self.children[0]
 
+    def get_next_leaf(self):
+        """
+        Raises an IndexError if it's the last node. (Would be the module)
+        """
+        c = self.parent.children
+        index = c.index(self)
+        if index == len(c) - 1:
+            # TODO WTF? recursion?
+            return self.get_next_leaf()
+        else:
+            return c[index + 1]
+
+    def last_leaf(self):
+        try:
+            return self.children[-1].last_leaf()
+        except AttributeError:
+            return self.children[-1]
+
+    def get_following_comment_same_line(self):
+        """
+        returns (as string) any comment that appears on the same line,
+        after the node, including the #
+        """
+        try:
+            if self.isinstance(ForStmt):
+                whitespace = self.children[5].first_leaf().prefix
+            elif self.isinstance(WithStmt):
+                whitespace = self.children[3].first_leaf().prefix
+            else:
+                whitespace = self.last_leaf().get_next_leaf().prefix
+        except AttributeError:
+            return None
+        except ValueError:
+            # TODO in some particular cases, the tree doesn't seem to be linked
+            # correctly
+            return None
+        if "#" not in whitespace:
+            return None
+        comment = whitespace[whitespace.index("#"):]
+        if "\r" in comment:
+            comment = comment[:comment.index("\r")]
+        if "\n" in comment:
+            comment = comment[:comment.index("\n")]
+        return comment
+
     @utf8_repr
     def __repr__(self):
-        code = self.get_code().replace('\n', ' ')
+        code = self.get_code().replace('\n', ' ').strip()
         if not is_py3:
             code = code.encode(encoding, 'replace')
         return "<%s: %s@%s,%s>" % \
@@ -476,6 +621,13 @@ class Node(BaseNode):
     """Concrete implementation for interior nodes."""
     __slots__ = ('type',)
 
+    _IGNORE_EXECUTE_NODES = set([
+        'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',
+        'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',
+        'decorated', 'arglist', 'argument', 'exprlist', 'testlist',
+        'testlist_safe', 'testlist1'
+    ])
+
     def __init__(self, type, children):
         """
         Initializer.
@@ -488,10 +640,50 @@ class Node(BaseNode):
         super(Node, self).__init__(children)
         self.type = type
 
+    def nodes_to_execute(self, last_added=False):
+        """
+        For static analysis.
+        """
+        result = []
+        if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added:
+            result.append(self)
+            last_added = True
+
+        for child in self.children:
+            result += child.nodes_to_execute(last_added)
+        return result
+
     def __repr__(self):
         return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children)
 
 
+class ErrorNode(BaseNode):
+    """
+    TODO doc
+    """
+    __slots__ = ()
+    type = 'error_node'
+
+    def nodes_to_execute(self, last_added=False):
+        return []
+
+
+class ErrorLeaf(LeafWithNewLines):
+    """
+    TODO doc
+    """
+    __slots__ = ('original_type')
+    type = 'error_leaf'
+
+    def __init__(self, position_modifier, original_type, value, start_pos, prefix=''):
+        super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix)
+        self.original_type = original_type
+
+    def __repr__(self):
+        token_type = token.tok_name[self.original_type]
+        return "<%s: %s, %s)>" % (type(self).__name__, token_type, self.start_pos)
+
+
 class IsScopeMeta(type):
     def __instancecheck__(self, other):
         return other.is_scope()
@@ -587,8 +779,7 @@ class Module(Scope):
     Depending on the underlying parser this may be a full module or just a part
     of a module.
     """
-    __slots__ = ('path', 'global_names', 'used_names', '_name',
-                 'error_statement_stacks')
+    __slots__ = ('path', 'global_names', 'used_names', '_name')
     type = 'file_input'
 
     def __init__(self, children):
@@ -604,7 +795,7 @@ class Module(Scope):
         self.path = None  # Set later.
 
     @property
-    @cache.underscore_memoization
+    @underscore_memoization
     def name(self):
         """ This is used for the goto functions. """
         if self.path is None:
@@ -637,11 +828,25 @@ class Module(Scope):
                         return True
         return False
 
+    def nodes_to_execute(self, last_added=False):
+        # Yield itself, class needs to be executed for decorator checks.
+        result = []
+        for child in self.children:
+            result += child.nodes_to_execute()
+        return result
+
 
 class Decorator(BaseNode):
     type = 'decorator'
     __slots__ = ()
 
+    def nodes_to_execute(self, last_added=False):
+        if self.children[-2] == ')':
+            node = self.children[-3]
+            if node != '(':
+                return node.nodes_to_execute()
+        return []
+
 
 class ClassOrFunc(Scope):
     __slots__ = ()
@@ -699,6 +904,34 @@ class Class(ClassOrFunc):
                     sub.get_call_signature(func_name=self.name), docstr)
         return docstr
 
+    def nodes_to_execute(self, last_added=False):
+        # Yield itself, class needs to be executed for decorator checks.
+        yield self
+        # Super arguments.
+        arglist = self.get_super_arglist()
+        try:
+            children = arglist.children
+        except AttributeError:
+            if arglist is not None:
+                for node_to_execute in arglist.nodes_to_execute():
+                    yield node_to_execute
+        else:
+            for argument in children:
+                if argument.type == 'argument':
+                    # metaclass= or list comprehension or */**
+                    raise NotImplementedError('Metaclasses not implemented')
+                else:
+                    for node_to_execute in argument.nodes_to_execute():
+                        yield node_to_execute
+
+        # care for the class suite:
+        for node in self.children[self.children.index(':'):]:
+            # This could be easier without the fast parser. But we need to find
+            # the position of the colon, because everything after it can be a
+            # part of the class, not just its suite.
+            for node_to_execute in node.nodes_to_execute():
+                yield node_to_execute
+
 
 def _create_params(parent, argslist_list):
     """
@@ -726,23 +959,28 @@ def _create_params(parent, argslist_list):
 
     if first.type in ('name', 'tfpdef'):
         if check_python2_nested_param(first):
-            return []
+            return [first]
         else:
             return [Param([first], parent)]
+    elif first == '*':
+        return [first]
     else:  # argslist is a `typedargslist` or a `varargslist`.
         children = first.children
-        params = []
+        new_children = []
         start = 0
         # Start with offset 1, because the end is higher.
         for end, child in enumerate(children + [None], 1):
             if child is None or child == ',':
-                new_children = children[start:end]
-                if new_children:  # Could as well be comma and then end.
-                    if check_python2_nested_param(new_children[0]):
-                        continue
-                    params.append(Param(new_children, parent))
+                param_children = children[start:end]
+                if param_children:  # Could as well be comma and then end.
+                    if check_python2_nested_param(param_children[0]):
+                        new_children += param_children
+                    elif param_children[0] == '*' and param_children[1] == ',':
+                        new_children += param_children
+                    else:
+                        new_children.append(Param(param_children, parent))
                     start = end
-        return params
+        return new_children
 
 
 class Function(ClassOrFunc):
@@ -769,8 +1007,7 @@ class Function(ClassOrFunc):
 
     @property
     def params(self):
-        # Contents of parameter lit minus the leading <Operator: (> and the trailing <Operator: )>.
-        return self.children[2].children[1:-1]
+        return [p for p in self.children[2].children if p.type == 'param']
 
     @property
     def name(self):
@@ -786,7 +1023,10 @@ class Function(ClassOrFunc):
 
     def annotation(self):
         try:
-            return self.children[6]  # 6th element: def foo(...) -> bar
+            if self.children[3] == "->":
+                return self.children[4]
+            assert self.children[3] == ":"
+            return None
         except IndexError:
             return None
 
@@ -807,13 +1047,28 @@ class Function(ClassOrFunc):
 
     def _get_paramlist_code(self):
         return self.children[2].get_code()
-    
+
     @property
     def doc(self):
         """ Return a document string including call signature. """
         docstr = self.raw_doc
         return '%s\n\n%s' % (self.get_call_signature(), docstr)
 
+    def nodes_to_execute(self, last_added=False):
+        # Yield itself, functions needs to be executed for decorator checks.
+        yield self
+        for param in self.params:
+            if param.default is not None:
+                yield param.default
+        # care for the function suite:
+        for node in self.children[4:]:
+            # This could be easier without the fast parser. The fast parser
+            # allows that the 4th position is empty or that there's even a
+            # fifth element (another function/class). So just scan everything
+            # after colon.
+            for node_to_execute in node.nodes_to_execute():
+                yield node_to_execute
+
 
 class Lambda(Function):
     """
@@ -842,7 +1097,7 @@ class Lambda(Function):
 
     def _get_paramlist_code(self):
         return '(' + ''.join(param.get_code() for param in self.params).strip() + ')'
-    
+
     @property
     def params(self):
         return self.children[1:-2]
@@ -850,10 +1105,22 @@ class Lambda(Function):
     def is_generator(self):
         return False
 
+    def annotation(self):
+        # lambda functions do not support annotations
+        return None
+
     @property
     def yields(self):
         return []
 
+    def nodes_to_execute(self, last_added=False):
+        for param in self.params:
+            if param.default is not None:
+                yield param.default
+        # Care for the lambda test (last child):
+        for node_to_execute in self.children[-1].nodes_to_execute():
+            yield node_to_execute
+
     def __repr__(self):
         return "<%s@%s>" % (self.__class__.__name__, self.start_pos)
 
@@ -861,6 +1128,11 @@ class Lambda(Function):
 class Flow(BaseNode):
     __slots__ = ()
 
+    def nodes_to_execute(self, last_added=False):
+        for child in self.children:
+            for node_to_execute in child.nodes_to_execute():
+                yield node_to_execute
+
 
 class IfStmt(Flow):
     type = 'if_stmt'
@@ -880,9 +1152,20 @@ class IfStmt(Flow):
                 yield self.children[i + 1]
 
     def node_in_which_check_node(self, node):
+        """
+        Returns the check node (see function above) that a node is contained
+        in. However if it the node is in the check node itself and not in the
+        suite return None.
+        """
+        start_pos = node.start_pos
         for check_node in reversed(list(self.check_nodes())):
-            if check_node.start_pos < node.start_pos:
-                return check_node
+            if check_node.start_pos < start_pos:
+                if start_pos < check_node.end_pos:
+                    return None
+                    # In this case the node is within the check_node itself,
+                    # not in the suite
+                else:
+                    return check_node
 
     def node_after_else(self, node):
         """
@@ -905,6 +1188,21 @@ class ForStmt(Flow):
     type = 'for_stmt'
     __slots__ = ()
 
+    def get_input_node(self):
+        """
+        Returns the input node ``y`` from: ``for x in y:``.
+        """
+        return self.children[3]
+
+    def defines_one_name(self):
+        """
+        Returns True if only one name is returned: ``for x in y``.
+        Returns False if the for loop is more complicated: ``for x, z in y``.
+
+        :returns: bool
+        """
+        return self.children[1].type == 'name'
+
 
 class TryStmt(Flow):
     type = 'try_stmt'
@@ -921,6 +1219,16 @@ class TryStmt(Flow):
             elif node == 'except':
                 yield None
 
+    def nodes_to_execute(self, last_added=False):
+        result = []
+        for child in self.children[2::3]:
+            result += child.nodes_to_execute()
+        for child in self.children[0::3]:
+            if child.type == 'except_clause':
+                # Add the test node and ignore the `as NAME` definition.
+                result += child.children[1].nodes_to_execute()
+        return result
+
 
 class WithStmt(Flow):
     type = 'with_stmt'
@@ -941,6 +1249,16 @@ class WithStmt(Flow):
             if is_node(node, 'with_item'):
                 return node.children[0]
 
+    def nodes_to_execute(self, last_added=False):
+        result = []
+        for child in self.children[1::2]:
+            if child.type == 'with_item':
+                # Just ignore the `as EXPR` part - at least for now, because
+                # most times it's just a name.
+                child = child.children[0]
+            result += child.nodes_to_execute()
+        return result
+
 
 class Import(BaseNode):
     __slots__ = ()
@@ -963,6 +1281,14 @@ class Import(BaseNode):
     def is_star_import(self):
         return self.children[-1] == '*'
 
+    def nodes_to_execute(self, last_added=False):
+        """
+        `nodes_to_execute` works a bit different for imports, because the names
+        itself cannot directly get resolved (except on itself).
+        """
+        # TODO couldn't we return the names? Would be nicer.
+        return [self]
+
 
 class ImportFrom(Import):
     type = 'import_from'
@@ -1087,17 +1413,33 @@ class ImportName(Import):
 class KeywordStatement(BaseNode):
     """
     For the following statements: `assert`, `del`, `global`, `nonlocal`,
-    `raise`, `return`, `yield`, `pass`, `continue`, `break`, `return`, `yield`.
+    `raise`, `return`, `yield`, `return`, `yield`.
+
+    `pass`, `continue` and `break` are not in there, because they are just
+    simple keywords and the parser reduces it to a keyword.
     """
     __slots__ = ()
 
+    @property
+    def type(self):
+        """
+        Keyword statements start with the keyword and end with `_stmt`. You can
+        crosscheck this with the Python grammar.
+        """
+        return '%s_stmt' % self.keyword
+
     @property
     def keyword(self):
         return self.children[0].value
 
+    def nodes_to_execute(self, last_added=False):
+        result = []
+        for child in self.children:
+            result += child.nodes_to_execute()
+        return result
+
 
 class AssertStmt(KeywordStatement):
-    type = 'assert_stmt'
     __slots__ = ()
 
     def assertion(self):
@@ -1105,7 +1447,6 @@ class AssertStmt(KeywordStatement):
 
 
 class GlobalStmt(KeywordStatement):
-    type = 'global_stmt'
     __slots__ = ()
 
     def get_defined_names(self):
@@ -1114,16 +1455,31 @@ class GlobalStmt(KeywordStatement):
     def get_global_names(self):
         return self.children[1::2]
 
+    def nodes_to_execute(self, last_added=False):
+        """
+        The global keyword allows to define any name. Even if it doesn't
+        exist.
+        """
+        return []
+
 
 class ReturnStmt(KeywordStatement):
-    type = 'return_stmt'
     __slots__ = ()
 
 
 class YieldExpr(BaseNode):
-    type = 'yield_expr'
     __slots__ = ()
 
+    @property
+    def type(self):
+        return 'yield_expr'
+
+    def nodes_to_execute(self, last_added=False):
+        if len(self.children) > 1:
+            return self.children[1].nodes_to_execute()
+        else:
+            return []
+
 
 def _defined_names(current):
     """
@@ -1134,9 +1490,9 @@ def _defined_names(current):
     if is_node(current, 'testlist_star_expr', 'testlist_comp', 'exprlist'):
         for child in current.children[::2]:
             names += _defined_names(child)
-    elif is_node(current, 'atom'):
+    elif is_node(current, 'atom', 'star_expr'):
         names += _defined_names(current.children[1])
-    elif is_node(current, 'power'):
+    elif is_node(current, 'power', 'atom_expr'):
         if current.children[-2] != '**':  # Just if there's no operation
             trailer = current.children[-1]
             if trailer.children[0] == '.':
@@ -1168,6 +1524,14 @@ class ExprStmt(BaseNode, DocstringMixin):
         except IndexError:
             return None
 
+    def nodes_to_execute(self, last_added=False):
+        # I think evaluating the statement (and possibly returned arrays),
+        # should be enough for static analysis.
+        result = [self]
+        for child in self.children:
+            result += child.nodes_to_execute(last_added=True)
+        return result
+
 
 class Param(BaseNode):
     """
@@ -1198,8 +1562,14 @@ class Param(BaseNode):
             return None
 
     def annotation(self):
-        # Generate from tfpdef.
-        raise NotImplementedError
+        tfpdef = self._tfpdef()
+        if is_node(tfpdef, 'tfpdef'):
+            assert tfpdef.children[1] == ":"
+            assert len(tfpdef.children) == 3
+            annotation = tfpdef.children[2]
+            return annotation
+        else:
+            return None
 
     def _tfpdef(self):
         """
@@ -1232,6 +1602,16 @@ class CompFor(BaseNode):
     type = 'comp_for'
     __slots__ = ()
 
+    def get_comp_fors(self):
+        yield self
+        last = self.children[-1]
+        while True:
+            if isinstance(last, CompFor):
+                yield last
+            elif not is_node(last, 'comp_if'):
+                break
+            last = last.children[-1]
+
     def is_scope(self):
         return True
 
@@ -1248,3 +1628,16 @@ class CompFor(BaseNode):
 
     def get_defined_names(self):
         return _defined_names(self.children[1])
+
+    def nodes_to_execute(self, last_added=False):
+        last = self.children[-1]
+        if last.type == 'comp_if':
+            for node in last.children[-1].nodes_to_execute():
+                yield node
+            last = self.children[-2]
+        elif last.type == 'comp_for':
+            for node in last.nodes_to_execute():
+                yield node
+            last = self.children[-2]
+        for node in last.nodes_to_execute():
+            yield node

jedi/parser/user_context.py

@@ -1,339 +0,0 @@
-import re
-import os
-import keyword
-
-from jedi import cache
-from jedi import common
-from jedi.parser import tokenize, Parser
-from jedi._compatibility import u
-from jedi.parser.fast import FastParser
-from jedi.parser import tree
-from jedi import debug
-from jedi.common import PushBackIterator
-
-
-REPLACE_STR = r"[bBuU]?[rR]?" + (r"(?:(')[^\n'\\]*(?:\\.[^\n'\\]*)*(?:'|$)" +
-                                 '|' +
-                                 r'(")[^\n"\\]*(?:\\.[^\n"\\]*)*(?:"|$))')
-REPLACE_STR = re.compile(REPLACE_STR)
-
-
-class UserContext(object):
-    """
-    :param source: The source code of the file.
-    :param position: The position, the user is currently in. Only important \
-    for the main file.
-    """
-    def __init__(self, source, position):
-        self.source = source
-        self.position = position
-        self._line_cache = None
-
-        self._relevant_temp = None
-
-    @cache.underscore_memoization
-    def get_path_until_cursor(self):
-        """ Get the path under the cursor. """
-        path, self._start_cursor_pos = self._calc_path_until_cursor(self.position)
-        return path
-
-    def _backwards_line_generator(self, start_pos):
-        self._line_temp, self._column_temp = start_pos
-        first_line = self.get_line(start_pos[0])[:self._column_temp]
-
-        self._line_length = self._column_temp
-        yield first_line[::-1] + '\n'
-
-        while True:
-            self._line_temp -= 1
-            line = self.get_line(self._line_temp)
-            self._line_length = len(line)
-            yield line[::-1] + '\n'
-
-    def _get_backwards_tokenizer(self, start_pos, line_gen=None):
-        if line_gen is None:
-            line_gen = self._backwards_line_generator(start_pos)
-        token_gen = tokenize.generate_tokens(lambda: next(line_gen))
-        for typ, tok_str, tok_start_pos, prefix in token_gen:
-            line = self.get_line(self._line_temp)
-            # Calculate the real start_pos of the token.
-            if tok_start_pos[0] == 1:
-                # We are in the first checked line
-                column = start_pos[1] - tok_start_pos[1]
-            else:
-                column = len(line) - tok_start_pos[1]
-            # Multi-line docstrings must be accounted for.
-            first_line = common.splitlines(tok_str)[0]
-            column -= len(first_line)
-            # Reverse the token again, so that it is in normal order again.
-            yield typ, tok_str[::-1], (self._line_temp, column), prefix[::-1]
-
-    def _calc_path_until_cursor(self, start_pos):
-        """
-        Something like a reverse tokenizer that tokenizes the reversed strings.
-        """
-        open_brackets = ['(', '[', '{']
-        close_brackets = [')', ']', '}']
-
-        start_cursor = start_pos
-        gen = PushBackIterator(self._get_backwards_tokenizer(start_pos))
-        string = u('')
-        level = 0
-        force_point = False
-        last_type = None
-        is_first = True
-        for tok_type, tok_str, tok_start_pos, prefix in gen:
-            if is_first:
-                if prefix:  # whitespace is not a path
-                    return u(''), start_cursor
-                is_first = False
-
-            if last_type == tok_type == tokenize.NAME:
-                string = ' ' + string
-
-            if level:
-                if tok_str in close_brackets:
-                    level += 1
-                elif tok_str in open_brackets:
-                    level -= 1
-            elif tok_str == '.':
-                force_point = False
-            elif force_point:
-                # Reversed tokenizing, therefore a number is recognized as a
-                # floating point number.
-                # The same is true for string prefixes -> represented as a
-                # combination of string and name.
-                if tok_type == tokenize.NUMBER and tok_str[-1] == '.' \
-                        or tok_type == tokenize.NAME and last_type == tokenize.STRING \
-                        and tok_str.lower() in ('b', 'u', 'r', 'br', 'ur'):
-                    force_point = False
-                else:
-                    break
-            elif tok_str in close_brackets:
-                level += 1
-            elif tok_type in [tokenize.NAME, tokenize.STRING]:
-                if keyword.iskeyword(tok_str) and string:
-                    # If there's already something in the string, a keyword
-                    # never adds any meaning to the current statement.
-                    break
-                force_point = True
-            elif tok_type == tokenize.NUMBER:
-                pass
-            else:
-                if tok_str == '-':
-                    next_tok = next(gen)
-                    if next_tok[1] == 'e':
-                        gen.push_back(next_tok)
-                    else:
-                        break
-                else:
-                    break
-
-            start_cursor = tok_start_pos
-            string = tok_str + prefix + string
-            last_type = tok_type
-
-        # Don't need whitespace around a statement.
-        return string.strip(), start_cursor
-
-    def get_path_under_cursor(self):
-        """
-        Return the path under the cursor. If there is a rest of the path left,
-        it will be added to the stuff before it.
-        """
-        return self.get_path_until_cursor() + self.get_path_after_cursor()
-
-    def get_path_after_cursor(self):
-        line = self.get_line(self.position[0])
-        return re.search("[\w\d]*", line[self.position[1]:]).group(0)
-
-    def get_operator_under_cursor(self):
-        line = self.get_line(self.position[0])
-        after = re.match("[^\w\s]+", line[self.position[1]:])
-        before = re.match("[^\w\s]+", line[:self.position[1]][::-1])
-        return (before.group(0) if before is not None else '') \
-            + (after.group(0) if after is not None else '')
-
-    def call_signature(self):
-        """
-        :return: Tuple of string of the call and the index of the cursor.
-        """
-        def get_line(pos):
-            def simplify_str(match):
-                """
-                To avoid having strings without end marks (error tokens) and
-                strings that just screw up all the call signatures, just
-                simplify everything.
-                """
-                mark = match.group(1) or match.group(2)
-                return mark + ' ' * (len(match.group(0)) - 2) + mark
-
-            line_gen = self._backwards_line_generator(pos)
-            for line in line_gen:
-                # We have to switch the already backwards lines twice, because
-                # we scan them from start.
-                line = line[::-1]
-                modified = re.sub(REPLACE_STR, simplify_str, line)
-                yield modified[::-1]
-
-        index = 0
-        level = 0
-        next_must_be_name = False
-        next_is_key = False
-        key_name = None
-        generator = self._get_backwards_tokenizer(self.position, get_line(self.position))
-        for tok_type, tok_str, start_pos, prefix in generator:
-            if tok_str in tokenize.ALWAYS_BREAK_TOKENS:
-                break
-            elif next_must_be_name:
-                if tok_type == tokenize.NUMBER:
-                    # If there's a number at the end of the string, it will be
-                    # tokenized as a number. So add it to the name.
-                    tok_type, t, _, _ = next(generator)
-                if tok_type == tokenize.NAME:
-                    end_pos = start_pos[0], start_pos[1] + len(tok_str)
-                    call, start_pos = self._calc_path_until_cursor(start_pos=end_pos)
-                    return call, index, key_name, start_pos
-                index = 0
-                next_must_be_name = False
-            elif next_is_key:
-                if tok_type == tokenize.NAME:
-                    key_name = tok_str
-                next_is_key = False
-
-            if tok_str == '(':
-                level += 1
-                if level == 1:
-                    next_must_be_name = True
-                    level = 0
-            elif tok_str == ')':
-                level -= 1
-            elif tok_str == ',':
-                index += 1
-            elif tok_str == '=':
-                next_is_key = True
-        return None, 0, None, (0, 0)
-
-    def get_context(self, yield_positions=False):
-        self.get_path_until_cursor()  # In case _start_cursor_pos is undefined.
-        pos = self._start_cursor_pos
-        while True:
-            # remove non important white space
-            line = self.get_line(pos[0])
-            while True:
-                if pos[1] == 0:
-                    line = self.get_line(pos[0] - 1)
-                    if line and line[-1] == '\\':
-                        pos = pos[0] - 1, len(line) - 1
-                        continue
-                    else:
-                        break
-
-                if line[pos[1] - 1].isspace():
-                    pos = pos[0], pos[1] - 1
-                else:
-                    break
-
-            try:
-                result, pos = self._calc_path_until_cursor(start_pos=pos)
-                if yield_positions:
-                    yield pos
-                else:
-                    yield result
-            except StopIteration:
-                if yield_positions:
-                    yield None
-                else:
-                    yield ''
-
-    def get_line(self, line_nr):
-        if not self._line_cache:
-            self._line_cache = common.splitlines(self.source)
-
-        if line_nr == 0:
-            # This is a fix for the zeroth line. We need a newline there, for
-            # the backwards parser.
-            return u('')
-        if line_nr < 0:
-            raise StopIteration()
-        try:
-            return self._line_cache[line_nr - 1]
-        except IndexError:
-            raise StopIteration()
-
-    def get_position_line(self):
-        return self.get_line(self.position[0])[:self.position[1]]
-
-
-class UserContextParser(object):
-    def __init__(self, grammar, source, path, position, user_context,
-                 parser_done_callback, use_fast_parser=True):
-        self._grammar = grammar
-        self._source = source
-        self._path = path and os.path.abspath(path)
-        self._position = position
-        self._user_context = user_context
-        self._use_fast_parser = use_fast_parser
-        self._parser_done_callback = parser_done_callback
-
-    @cache.underscore_memoization
-    def _parser(self):
-        cache.invalidate_star_import_cache(self._path)
-        if self._use_fast_parser:
-            parser = FastParser(self._grammar, self._source, self._path)
-            # Don't pickle that module, because the main module is changing quickly
-            cache.save_parser(self._path, parser, pickling=False)
-        else:
-            parser = Parser(self._grammar, self._source, self._path)
-        self._parser_done_callback(parser)
-        return parser
-
-    @cache.underscore_memoization
-    def user_stmt(self):
-        module = self.module()
-        debug.speed('parsed')
-        return module.get_statement_for_position(self._position)
-
-    @cache.underscore_memoization
-    def user_stmt_with_whitespace(self):
-        """
-        Returns the statement under the cursor even if the statement lies
-        before the cursor.
-        """
-        user_stmt = self.user_stmt()
-
-        if not user_stmt:
-            # for statements like `from x import ` (cursor not in statement)
-            # or `abs( ` where the cursor is out in the whitespace.
-            if self._user_context.get_path_under_cursor():
-                # We really should have a user_stmt, but the parser couldn't
-                # process it - probably a Syntax Error (or in a comment).
-                debug.warning('No statement under the cursor.')
-                return
-            pos = next(self._user_context.get_context(yield_positions=True))
-            user_stmt = self.module().get_statement_for_position(pos)
-        return user_stmt
-
-    @cache.underscore_memoization
-    def user_scope(self):
-        """
-        Returns the scope in which the user resides. This includes flows.
-        """
-        user_stmt = self.user_stmt()
-        if user_stmt is None:
-            def scan(scope):
-                for s in scope.children:
-                    if s.start_pos <= self._position <= s.end_pos:
-                        if isinstance(s, (tree.Scope, tree.Flow)):
-                                if isinstance(s, tree.Flow):
-                                    return s
-                                return scan(s) or s
-                        elif s.type in ('suite', 'decorated'):
-                            return scan(s)
-
-            return scan(self.module()) or self.module()
-        else:
-            return user_stmt.get_parent_scope(include_flows=True)
-
-    def module(self):
-        return self._parser().module

jedi/parser/utils.py

@@ -0,0 +1,198 @@
+import inspect
+import time
+import os
+import sys
+import json
+import hashlib
+import gc
+import shutil
+import pickle
+
+from jedi import settings
+from jedi import debug
+
+
+def underscore_memoization(func):
+    """
+    Decorator for methods::
+
+        class A(object):
+            def x(self):
+                if self._x:
+                    self._x = 10
+                return self._x
+
+    Becomes::
+
+        class A(object):
+            @underscore_memoization
+            def x(self):
+                return 10
+
+    A now has an attribute ``_x`` written by this decorator.
+    """
+    name = '_' + func.__name__
+
+    def wrapper(self):
+        try:
+            return getattr(self, name)
+        except AttributeError:
+            result = func(self)
+            if inspect.isgenerator(result):
+                result = list(result)
+            setattr(self, name, result)
+            return result
+
+    return wrapper
+
+
+# for fast_parser, should not be deleted
+parser_cache = {}
+
+
+class ParserCacheItem(object):
+    def __init__(self, parser, change_time=None):
+        self.parser = parser
+        if change_time is None:
+            change_time = time.time()
+        self.change_time = change_time
+
+
+def load_parser(path):
+    """
+    Returns the module or None, if it fails.
+    """
+    p_time = os.path.getmtime(path) if path else None
+    try:
+        parser_cache_item = parser_cache[path]
+        if not path or p_time <= parser_cache_item.change_time:
+            return parser_cache_item.parser
+    except KeyError:
+        if settings.use_filesystem_cache:
+            return ParserPickling.load_parser(path, p_time)
+
+
+def save_parser(path, parser, pickling=True):
+    try:
+        p_time = None if path is None else os.path.getmtime(path)
+    except OSError:
+        p_time = None
+        pickling = False
+
+    item = ParserCacheItem(parser, p_time)
+    parser_cache[path] = item
+    if settings.use_filesystem_cache and pickling:
+        ParserPickling.save_parser(path, item)
+
+
+class ParserPickling(object):
+
+    version = 25
+    """
+    Version number (integer) for file system cache.
+
+    Increment this number when there are any incompatible changes in
+    parser representation classes.  For example, the following changes
+    are regarded as incompatible.
+
+    - Class name is changed.
+    - Class is moved to another module.
+    - Defined slot of the class is changed.
+    """
+
+    def __init__(self):
+        self.__index = None
+        self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
+        """
+        Short name for distinguish Python implementations and versions.
+
+        It's like `sys.implementation.cache_tag` but for Python < 3.3
+        we generate something similar.  See:
+        http://docs.python.org/3/library/sys.html#sys.implementation
+
+        .. todo:: Detect interpreter (e.g., PyPy).
+        """
+
+    def load_parser(self, path, original_changed_time):
+        try:
+            pickle_changed_time = self._index[path]
+        except KeyError:
+            return None
+        if original_changed_time is not None \
+                and pickle_changed_time < original_changed_time:
+            # the pickle file is outdated
+            return None
+
+        with open(self._get_hashed_path(path), 'rb') as f:
+            try:
+                gc.disable()
+                parser_cache_item = pickle.load(f)
+            finally:
+                gc.enable()
+
+        debug.dbg('pickle loaded: %s', path)
+        parser_cache[path] = parser_cache_item
+        return parser_cache_item.parser
+
+    def save_parser(self, path, parser_cache_item):
+        self.__index = None
+        try:
+            files = self._index
+        except KeyError:
+            files = {}
+            self._index = files
+
+        with open(self._get_hashed_path(path), 'wb') as f:
+            pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
+            files[path] = parser_cache_item.change_time
+
+        self._flush_index()
+
+    @property
+    def _index(self):
+        if self.__index is None:
+            try:
+                with open(self._get_path('index.json')) as f:
+                    data = json.load(f)
+            except (IOError, ValueError):
+                self.__index = {}
+            else:
+                # 0 means version is not defined (= always delete cache):
+                if data.get('version', 0) != self.version:
+                    self.clear_cache()
+                    self.__index = {}
+                else:
+                    self.__index = data['index']
+        return self.__index
+
+    def _remove_old_modules(self):
+        # TODO use
+        change = False
+        if change:
+            self._flush_index(self)
+            self._index  # reload index
+
+    def _flush_index(self):
+        data = {'version': self.version, 'index': self._index}
+        with open(self._get_path('index.json'), 'w') as f:
+            json.dump(data, f)
+        self.__index = None
+
+    def clear_cache(self):
+        shutil.rmtree(self._cache_directory())
+
+    def _get_hashed_path(self, path):
+        return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
+
+    def _get_path(self, file):
+        dir = self._cache_directory()
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+        return os.path.join(dir, file)
+
+    def _cache_directory(self):
+        return os.path.join(settings.cache_directory, self.py_tag)
+
+
+# is a singleton
+ParserPickling = ParserPickling()

jedi/settings.py

@@ -63,6 +63,7 @@ definitely worse in some cases. But a completion should also be fast.
 .. autodata:: max_function_recursion_level
 .. autodata:: max_executions_without_builtins
 .. autodata:: max_executions
+.. autodata:: max_dynamic_params_depth
 .. autodata:: scale_call_signatures
 
 
@@ -86,13 +87,6 @@ case_insensitive_completion = True
 The completion is by default case insensitive.
 """
 
-add_dot_after_module = False
-"""
-Adds a dot after a module, because a module that is not accessed this way is
-definitely not the normal case. However, in VIM this doesn't work, that's why
-it isn't used at the moment.
-"""
-
 add_bracket_after_function = False
 """
 Adds an opening bracket after a function, because that's normal behaviour.

jedi/utils.py

@@ -10,8 +10,8 @@ import os
 import sys
 
 from jedi import Interpreter
-from jedi.api.helpers import completion_parts
-from jedi.parser.user_context import UserContext
+from jedi.api.helpers import get_on_completion_name
+from jedi import common
 
 
 def setup_readline(namespace_module=__main__):
@@ -72,9 +72,10 @@ def setup_readline(namespace_module=__main__):
                 try:
                     interpreter = Interpreter(text, [namespace_module.__dict__])
 
-                    path = UserContext(text, (1, len(text))).get_path_until_cursor()
-                    path, dot, like = completion_parts(path)
-                    before = text[:len(text) - len(like)]
+                    lines = common.splitlines(text)
+                    position = (len(lines), len(lines[-1]))
+                    name = get_on_completion_name(lines, position)
+                    before = text[:len(text) - len(name)]
                     completions = interpreter.completions()
                 finally:
                     sys.path.pop(0)
@@ -88,7 +89,7 @@ def setup_readline(namespace_module=__main__):
     try:
         import readline
     except ImportError:
-        print("Module readline not available.")
+        print("Jedi: Module readline not available.")
     else:
         readline.set_completer(JediRL().complete)
         readline.parse_and_bind("tab: complete")

setup.py

@@ -40,9 +40,9 @@ setup(name='jedi',
           'Programming Language :: Python :: 2.6',
           'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.2',
           'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',

test/completion/arrays.py

@@ -39,13 +39,16 @@ b[8:]
 #? list()
 b[int():]
 
+#? list()
+b[:]
+
 
 class _StrangeSlice():
-    def __getitem__(self, slice):
-        return slice
+    def __getitem__(self, sliced):
+        return sliced
 
 # Should not result in an error, just because the slice itself is returned.
-#? []
+#? slice()
 _StrangeSlice()[1:2]
 
 
@@ -125,6 +128,9 @@ f
 # -----------------
 # unnessecary braces
 # -----------------
+a = (1)
+#? int()
+a
 #? int()
 (1)
 #? int()
@@ -204,9 +210,22 @@ dic2['asdf']
 dic2[r'asdf']
 #? int()
 dic2[r'asdf']
+#? int()
+dic2[r'as' 'd' u'f']
 #? int() str()
 dic2['just_something']
 
+# unpacking
+a, b = dic2
+#? str()
+a
+a, b = {1: 'x', 2.0: 1j}
+#? int() float()
+a
+#? int() float()
+b
+
+
 def f():
     """ github #83 """
     r = {}
@@ -232,6 +251,11 @@ dic = {str(key): ''}
 #? str()
 dic['']
 
+
+for x in {1: 3.0, '': 1j}:
+    #? int() str()
+    x
+
 # -----------------
 # with variable as index
 # -----------------
@@ -365,3 +389,45 @@ recursion1([1,2])[0]
 for x in [1] + ['']:
     #? int() str()
     x
+
+# -----------------
+# For loops with attribute assignment.
+# -----------------
+def test_func():
+    x = 'asdf'
+    for x.something in [6,7,8]:
+        pass
+    #? str()
+    x
+
+    for x.something, b in [[6, 6.0]]:
+        pass
+    #? str()
+    x
+
+
+# -----------------
+# PEP 3132 Extended Iterable Unpacking (star unpacking)
+# -----------------
+
+a, *b, c = [1, 'b', list, dict]
+#? int()
+a
+#? str()
+b
+#? list
+c
+
+# Not valid syntax
+a, *b, *c = [1, 'd', list]
+#? int()
+a
+#? str()
+b
+#? list
+c
+
+lc = [x for a, *x in [(1, '', 1.0)]]
+
+#?
+lc[0][0]

test/completion/basic.py

@@ -165,6 +165,17 @@ def global_define():
 #? int()
 global_var_in_func
 
+
+def funct1():
+    # From issue #610
+    global global_dict_var
+    global_dict_var = dict()
+def funct2():
+    global global_dict_var
+    #? dict()
+    global_dict_var
+
+
 # -----------------
 # within docstrs
 # -----------------
@@ -215,6 +226,9 @@ if 1:
     #? str()
     xyz
 
+#?
+¹.
+
 # -----------------
 # exceptions
 # -----------------

test/completion/classes.py

@@ -83,6 +83,9 @@ TestClass.var_local.
 
 #? int()
 TestClass().ret(1)
+# Should not return int(), because we want the type before `.ret(1)`.
+#? 11 TestClass()
+TestClass().ret(1)
 #? int()
 inst.ret(1)
 
@@ -131,6 +134,8 @@ A().addition
 A().addition = None
 #? 8 int()
 A(1).addition = None
+#? 1 A
+A(1).addition = None
 a = A()
 #? 8 int()
 a.addition = None
@@ -238,7 +243,10 @@ class V:
 V(1).b()
 #? int()
 V(1).c()
-#? []
+#?
+V(1).d()
+# Only keywords should be possible to complete.
+#? ['is', 'in', 'not', 'and', 'or', 'if']
 V(1).d()
 
 

test/completion/comprehensions.py

@@ -30,8 +30,12 @@ a[0]
 
 arr = [1,'']
 a = [a for a in arr]
-#? int() str()
+#? int()
 a[0]
+#? str()
+a[1]
+#? int() str()
+a[2]
 
 a = [a if 1.0 else '' for a in [1] if [1.0]]
 #? int() str()
@@ -44,12 +48,9 @@ left, right = [x for x in (left, right)]
 left
 
 # with a dict literal
-#? str()
+#? int()
 [a for a in {1:'x'}][0]
 
-##? str()
-{a-1:b for a,b in {1:'a', 3:1.0}.items()}[0]
-
 # list comprehensions should also work in combination with functions
 def listen(arg):
     for x in arg:
@@ -57,9 +58,12 @@ def listen(arg):
         x
 
 listen(['' for x in [1]])
-#? str
+#?
 ([str for x in []])[0]
 
+# with a set literal
+#? int()
+[a for a in {1, 2, 3}][0]
 
 # -----------------
 # nested list comprehensions
@@ -93,6 +97,8 @@ left, right = (i for i in (1, ''))
 
 #? int()
 left
+#? str()
+right
 
 gen = (i for i in (1,))
 
@@ -110,9 +116,52 @@ next(gen)
 
 # issues with different formats
 left, right = (i for i in
-                       ('1', '2'))
+                       ('1', 2))
 #? str()
 left
+#? int()
+right
+
+# -----------------
+# dict comprehensions
+# -----------------
+
+#? int()
+list({a - 1: 3 for a in [1]})[0]
+
+d = {a - 1: b for a, b in {1: 'a', 3: 1.0}.items()}
+#? int()
+list(d)[0]
+#? str() float()
+d.values()[0]
+#? str()
+d[0]
+#? float() str()
+d[1]
+#? float()
+d[2]
+
+# -----------------
+# set comprehensions
+# -----------------
+
+#? set()
+{a - 1 for a in [1]}
+
+#? set()
+{a for a in range(10)}
+
+#? int()
+[x for x in {a for a in range(10)}][0]
+
+#? int()
+{a for a in range(10)}.pop()
+#? float() str()
+{b for a in [[3.0], ['']] for b in a}.pop()
+
+#? int()
+next(iter({a for a in range(10)}))
+
 
 # -----------------
 # name resolution in comprehensions.
@@ -123,3 +172,5 @@ def x():
     #? 22
     [a for a in h if hio]
     if hio: pass
+
+

test/completion/decorators.py

@@ -38,7 +38,7 @@ exe[1]
 #? set
 exe[2]
 #? []
-exe[3][0]
+exe[3][0].
 #? str()
 exe[4]['d']
 
@@ -70,7 +70,7 @@ exe[1]
 #? set
 exe[2]
 #? []
-exe[3][0]
+exe[3][0].
 #? str()
 exe[4]['d']
 
@@ -78,6 +78,9 @@ exe[4]['d']
 # -----------------
 # Decorator is a class
 # -----------------
+def same_func(func):
+    return func
+
 class Decorator(object):
     def __init__(self, func):
         self.func = func
@@ -94,10 +97,15 @@ nothing("")[0]
 #? str()
 nothing("")[1]
 
+
+@same_func
 @Decorator
 def nothing(a,b,c):
     return a,b,c
 
+#? int()
+nothing("")[0]
+
 class MethodDecoratorAsClass():
     class_var = 3
     @Decorator

test/completion/definition.py

@@ -6,6 +6,9 @@ Fallback to callee definition when definition not found.
 
 """Parenthesis closed at next line."""
 
+# Ignore these definitions for a little while, not sure if we really want them.
+# python <= 2.5
+
 #? isinstance
 isinstance(
 )

test/completion/descriptors.py

@@ -62,14 +62,14 @@ class B():
     p = property(t)
 
 #? []
-B().r()
+B().r().
 #? int()
 B().r
 
 #? str()
 B().p
 #? []
-B().p()
+B().p().
 
 class PropClass():
     def __init__(self, a):

test/completion/flow_analysis.py

@@ -218,3 +218,40 @@ else:
     a = ''
 #? int()
 a
+
+
+# -----------------
+# Recursion issues
+# -----------------
+
+def possible_recursion_error(filename):
+    if filename == 'a':
+        return filename
+    # It seems like without the brackets there wouldn't be a RecursionError.
+    elif type(filename) == str:
+        return filename
+
+
+if NOT_DEFINED:
+    s = str()
+else:
+    s = str()
+#? str()
+possible_recursion_error(s)
+
+
+# -----------------
+# In combination with imports
+# -----------------
+
+from import_tree import flow_import
+
+if 1 == flow_import.env:
+    a = 1
+elif 2 == flow_import.env:
+    a = ''
+elif 3 == flow_import.env:
+    a = 1.0
+
+#? int() str()
+a

test/completion/functions.py

@@ -178,6 +178,23 @@ nested_default(a=1.0)[1]
 #? str()
 nested_default(a=1.0, b='')[1]
 
+# Defaults should only work if they are defined before - not after.
+def default_function(a=default):
+    #?
+    return a
+
+#?
+default_function()
+
+default = int()
+
+def default_function(a=default):
+    #? int()
+    return a
+
+#? int()
+default_function()
+
 
 # -----------------
 # closures
@@ -185,8 +202,9 @@ nested_default(a=1.0, b='')[1]
 def a():
     l = 3
     def func_b():
-        #? str()
         l = ''
+        #? str()
+        l
     #? ['func_b']
     func_b
     #? int()
@@ -327,6 +345,15 @@ exe[3]
 #? set
 exe[3]['c']
 
+
+def kwargs_iteration(**kwargs):
+    return kwargs
+
+for x in kwargs_iteration(d=3):
+    #? float()
+    {'d': 1.0, 'c': '1'}[x]
+
+
 # -----------------
 # nested *args
 # -----------------
@@ -342,12 +369,12 @@ def nested_args2(*args, **kwargs):
 #? int()
 nested_args('', 1, 1.0, list)
 #? []
-nested_args('')
+nested_args('').
 
 #? int()
 nested_args2('', 1, 1.0)
 #? []
-nested_args2('')
+nested_args2('').
 
 # -----------------
 # nested **kwargs
@@ -371,9 +398,9 @@ nested_kw(a=3.0, b=1)
 #? int()
 nested_kw(b=1, a=r"")
 #? []
-nested_kw(1, '')
+nested_kw(1, '').
 #? []
-nested_kw(a='')
+nested_kw(a='').
 
 #? int()
 nested_kw2(b=1)
@@ -382,9 +409,9 @@ nested_kw2(b=1, c=1.0)
 #? int()
 nested_kw2(c=1.0, b=1)
 #? []
-nested_kw2('')
+nested_kw2('').
 #? []
-nested_kw2(a='')
+nested_kw2(a='').
 #? []
 nested_kw2('', b=1).
 
@@ -404,14 +431,14 @@ nested_both('', b=1, c=1.0, list)
 nested_both('', c=1.0, b=1, list)
 
 #? []
-nested_both('')
+nested_both('').
 
 #? int()
 nested_both2('', b=1, c=1.0)
 #? int()
 nested_both2('', c=1.0, b=1)
 #? []
-nested_both2('')
+nested_both2('').
 
 # -----------------
 # nested *args/**kwargs with a default arg
@@ -438,7 +465,7 @@ nested_def2('', b=1, c=1.0)[1]
 #? int()
 nested_def2('', c=1.0, b=1)[1]
 #? []
-nested_def2('')[1]
+nested_def2('')[1].
 
 # -----------------
 # magic methods

test/completion/generators.py

@@ -22,7 +22,7 @@ def gen_ret(value):
 next(gen_ret(1))
 
 #? []
-next(gen_ret())
+next(gen_ret()).
 
 # generators evaluate to true if cast by bool.
 a = ''
@@ -42,7 +42,7 @@ def get(param):
         yield ""
 
 #? []
-get()[0]
+get()[0].
 
 # -----------------
 # __iter__
@@ -131,6 +131,18 @@ def simple():
         yield ""
 
 a, b = simple()
+#? int() str()
+a
+# For now this is ok.
+#?
+b
+
+
+def simple2():
+    yield 1
+    yield ""
+
+a, b = simple2()
 #? int()
 a
 #? str()
@@ -163,3 +175,25 @@ gen().send()
 
 #? 
 gen()()
+
+# -----------------
+# yield from
+# -----------------
+
+# python >= 3.3
+
+def yield_from():
+    yield from iter([1])
+
+#? int()
+next(yield_from())
+
+def yield_from_multiple():
+    yield from iter([1])
+    yield str()
+
+x, y = yield_from_multiple()
+#? int()
+x
+#? str()
+y

test/completion/goto.py

@@ -30,6 +30,12 @@ b = math
 #! ['b = math']
 b
 
+#! 18 ['foo = 10']
+foo = 10;print(foo)
+
+# -----------------
+# classes
+# -----------------
 class C(object):
     def b(self):
         #! ['b = math']

test/completion/import_tree/flow_import.py

@@ -0,0 +1,4 @@
+if name:
+    env = 1
+else:
+    env = 2

test/completion/imports.py

@@ -49,6 +49,8 @@ def scope_nested():
 
     #? float()
     import_tree.pkg.mod1.a
+    #? ['a', '__name__', '__package__', '__file__', '__doc__']
+    a = import_tree.pkg.mod1.
 
     import import_tree.random
     #? set
@@ -75,6 +77,7 @@ def scope_from_import_variable():
     without the use of ``sys.modules`` modifications (e.g. ``os.path`` see also
     github issue #213 for clarification.
     """
+    a = 3
     #? 
     from import_tree.mod2.fake import a
     #? 

test/completion/invalid.py

@@ -31,7 +31,8 @@ def wrong_indents():
     asdf = 3
      asdf
     asdf(
-    #? int()
+    # TODO this seems to be wrong now?
+    ##? int()
     asdf
 def openbrace():
     asdf = 3
@@ -101,7 +102,7 @@ if isi
 try:
     except TypeError:
         #? str()
-        ""
+        str()
 
 def break(): pass
 # wrong ternary expression
@@ -175,16 +176,16 @@ import datetime as
 
 call = ''
 invalid = .call
-#?
+#? 
 invalid
 
 invalid = call?.call
-#?
+#? str()
 invalid
 
 # comma
 invalid = ,call
-#?
+#? str()
 invalid
 
 

test/completion/isinstance.py

@@ -29,6 +29,14 @@ if 2:
     #? str()
     ass
 
+# -----------------
+# invalid arguments
+# -----------------
+
+if isinstance(wrong, str()):
+    #?
+    wrong
+
 # -----------------
 # in functions
 # -----------------
@@ -55,6 +63,17 @@ a
 fooooo2('')
 
 
+def isinstance_func(arr):
+    for value in arr:
+        if isinstance(value, dict):
+            # Shouldn't fail, even with the dot.
+            #? 17 dict()
+            value.
+        elif isinstance(value, int):
+            x = value
+            #? int()
+            x
+
 # -----------------
 # Names with multiple indices.
 # -----------------
@@ -72,3 +91,10 @@ class Test():
             self.testing
             #? Test()
             self
+
+# -----------------
+# Syntax
+# -----------------
+
+#?
+isinstance(1, int())

test/completion/keywords.py

@@ -2,5 +2,58 @@
 #? ['raise']
 raise
 
-#? ['except', 'Exception']
+#? ['Exception']
 except
+
+#? []
+b + continu
+
+#? []
+b + continue
+
+#? ['continue']
+b; continue
+
+#? ['continue']
+b; continu
+
+#? []
+c + brea
+
+#? []
+a + break
+
+#? ['break']
+b; break
+
+# -----------------
+# Keywords should not appear everywhere.
+# -----------------
+
+#? []
+with open() as f
+#? []
+def i
+#? []
+class i
+
+#? []
+continue i
+
+# More syntax details, e.g. while only after newline, but not after semicolon,
+# continue also after semicolon
+#? ['while']
+while
+#? []
+x while
+#? []
+x; while
+#? ['continue']
+x; continue
+
+#? []
+and
+#? ['and']
+x and
+#? []
+x * and

test/completion/named_param.py

@@ -20,3 +20,12 @@ a(some_args)
 
 #? 13 []
 a(some_kwargs)
+
+def multiple(foo, bar):
+    pass
+
+#? 17 ['bar']
+multiple(foo, bar)
+
+#? ['bar']
+multiple(foo, bar

test/completion/on_import.py

@@ -63,9 +63,13 @@ import datetime.date
 
 #? 21 ['import']
 from import_tree.pkg import pkg
+#? 49 ['a', '__name__', '__doc__', '__file__', '__package__']
+from import_tree.pkg.mod1 import not_existant,    # whitespace before
+#? ['a', '__name__', '__doc__', '__file__', '__package__']
+from import_tree.pkg.mod1 import not_existant, 
 #? 22 ['mod1']
 from import_tree.pkg. import mod1
-#? 17 ['mod1', 'mod2', 'random', 'pkg', 'rename1', 'rename2', 'recurse_class1', 'recurse_class2', 'invisible_pkg']
+#? 17 ['mod1', 'mod2', 'random', 'pkg', 'rename1', 'rename2', 'recurse_class1', 'recurse_class2', 'invisible_pkg', 'flow_import']
 from import_tree. import pkg
 
 #? 18 ['pkg']

test/completion/ordering.py

@@ -87,7 +87,7 @@ from os import path
 # should not return a function, because `a` is a function above
 def f(b, a): return a
 #? []
-f(b=3)
+f(b=3).
 
 # -----------------
 # closure

test/completion/parser.py

@@ -1,5 +1,5 @@
 """
-Issues with the parser not the completion engine should be here.
+Issues with the parser and not the type inference should be part of this file.
 """
 
 class IndentIssues():
@@ -32,5 +32,12 @@ Just because there's a def keyword, doesn't mean it should not be able to
 complete to definition.
 """
 definition = 0
-#? ['definition', 'def']
+#? ['definition']
 str(def
+
+
+# It might be hard to determine the context
+class Foo(object):
+    @property
+    #? ['str']
+    def bar(str

test/completion/pep0484.py

@@ -0,0 +1,160 @@
+""" Pep-0484 type hinting """
+
+# python >= 3.2
+
+
+class A():
+    pass
+
+
+def function_parameters(a: A, b, c: str, d: int, e: str, f: str, g: int=4):
+    """
+    :param e: if docstring and annotation agree, only one should be returned
+    :type e: str
+    :param f: if docstring and annotation disagree, both should be returned
+    :type f: int
+    """
+    #? A()
+    a
+    #?
+    b
+    #? str()
+    c
+    #? int()
+    d
+    #? str()
+    e
+    #? int() str()
+    f
+    # int()
+    g
+
+
+def return_unspecified():
+    pass
+
+#?
+return_unspecified()
+
+
+def return_none() -> None:
+    """
+    Return type None means the same as no return type as far as jedi
+    is concerned
+    """
+    pass
+
+#?
+return_none()
+
+
+def return_str() -> str:
+    pass
+
+#? str()
+return_str()
+
+
+def return_custom_class() -> A:
+    pass
+
+#? A()
+return_custom_class()
+
+
+def return_annotation_and_docstring() -> str:
+    """
+    :rtype: int
+    """
+    pass
+
+#? str() int()
+return_annotation_and_docstring()
+
+
+def return_annotation_and_docstring_different() -> str:
+    """
+    :rtype: str
+    """
+    pass
+
+#? str()
+return_annotation_and_docstring_different()
+
+
+def annotation_forward_reference(b: "B") -> "B":
+    #? B()
+    b
+
+#? ["test_element"]
+annotation_forward_reference(1).t
+
+class B:
+    test_element = 1
+    pass
+
+#? B()
+annotation_forward_reference(1)
+
+
+class SelfReference:
+    test_element = 1
+    def test_method(self, x: "SelfReference") -> "SelfReference":
+        #? SelfReference()
+        x
+        #? ["test_element", "test_method"]
+        self.t
+        #? ["test_element", "test_method"]
+        x.t
+        #? ["test_element", "test_method"]
+        self.test_method(1).t
+
+#? SelfReference()
+SelfReference().test_method()
+
+def function_with_non_pep_0484_annotation(
+        x: "I can put anything here",
+        xx: "",
+        yy: "\r\n\0;+*&^564835(---^&*34",
+        y: 3 + 3,
+        zz: float) -> int("42"):
+    # infers int from function call
+    #? int()
+    x
+    # infers int from function call
+    #? int()
+    xx
+    # infers int from function call
+    #? int()
+    yy
+    # infers str from function call
+    #? str()
+    y
+    #? float()
+    zz
+#?
+function_with_non_pep_0484_annotation(1, 2, 3, "force string")
+
+def function_forward_reference_dynamic(
+        x: return_str_type(),
+        y: "return_str_type()") -> None:
+    #?
+    x
+    #? str()
+    y
+
+def return_str_type():
+    return str
+
+
+X = str
+def function_with_assined_class_in_reference(x: X, y: "Y"):
+    #? str()
+    x
+    #? int()
+    y
+Y = int
+
+def just_because_we_can(x: "flo" + "at"):
+    #? float()
+    x

test/completion/pep0484_comments.py

@@ -0,0 +1,109 @@
+a = 3  # type: str
+#? str()
+a
+
+b = 3  # type: str but I write more
+#? int()
+b
+
+c = 3  # type: str # I comment more
+#? str()
+c
+
+d = "It should not read comments from the next line"
+# type: int
+#? str()
+d
+
+# type: int
+e = "It should not read comments from the previous line"
+#? str()
+e
+
+class BB: pass
+
+def test(a, b):
+    a = a  # type: BB
+    c = a  # type: str
+    d = a
+    # type: str
+    e = a                 # type: str           # Should ignore long whitespace
+
+    #? BB()
+    a
+    #? str()
+    c
+    #? BB()
+    d
+    #? str()
+    e
+
+a,b = 1, 2 # type: str, float
+#? str()
+a
+#? float()
+b
+
+class Employee:
+    pass
+
+# The typing library is not installable for Python 2.6, therefore ignore the
+# following tests.
+# python >= 2.7
+
+from typing import List
+x = []   # type: List[Employee]
+#? Employee()
+x[1]
+x, y, z = [], [], []  # type: List[int], List[int], List[str]
+#? int()
+y[2]
+x, y, z = [], [], []  # type: (List[float], List[float], List[BB])
+for zi in z:
+    #? BB()
+    zi
+
+x = [
+   1,
+   2,
+]  # type: List[str]
+
+#? str()
+x[1]
+
+
+for bar in foo():  # type: str
+    #? str()
+    bar
+
+for bar, baz in foo():  # type: int, float
+    #? int()
+    bar
+    #? float()
+    baz
+
+for bar, baz in foo():
+    # type: str, str
+    """ type hinting on next line should not work """
+    #?
+    bar
+    #?
+    baz
+
+with foo():  # type: int
+    ...
+
+with foo() as f:  # type: str
+    #? str()
+    f
+
+with foo() as f:
+    # type: str
+    """ type hinting on next line should not work """
+    #?
+    f
+
+aaa = some_extremely_long_function_name_that_doesnt_leave_room_for_hints() \
+    # type: float # We should be able to put hints on the next line with a \
+#? float()
+aaa

test/completion/pep0484_typing.py

@@ -0,0 +1,263 @@
+"""
+Test the typing library, with docstrings. This is needed since annotations
+are not supported in python 2.7 else then annotating by comment (and this is
+still TODO at 2016-01-23)
+"""
+# There's no Python 2.6 typing module.
+# python >= 2.7
+import typing
+class B:
+    pass
+
+def we_can_has_sequence(p, q, r, s, t, u):
+    """
+    :type p: typing.Sequence[int]
+    :type q: typing.Sequence[B]
+    :type r: typing.Sequence[int]
+    :type s: typing.Sequence["int"]
+    :type t: typing.MutableSequence[dict]
+    :type u: typing.List[float]
+    """
+    #? ["count"]
+    p.c
+    #? int()
+    p[1]
+    #? ["count"]
+    q.c
+    #? B()
+    q[1]
+    #? ["count"]
+    r.c
+    #? int()
+    r[1]
+    #? ["count"]
+    s.c
+    #? int()
+    s[1]
+    #? []
+    s.a
+    #? ["append"]
+    t.a
+    #? dict()
+    t[1]
+    #? ["append"]
+    u.a
+    #? float()
+    u[1]
+
+def iterators(ps, qs, rs, ts):
+    """
+    :type ps: typing.Iterable[int]
+    :type qs: typing.Iterator[str]
+    :type rs: typing.Sequence["ForwardReference"]
+    :type ts: typing.AbstractSet["float"]
+    """
+    for p in ps:
+        #? int()
+        p
+    #?
+    next(ps)
+    a, b = ps
+    #? int()
+    a
+    ##? int()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    # test below is just to make sure that in case it gets fixed by accident
+    # these tests will be fixed as well the way they should be
+    #?
+    b
+
+    for q in qs:
+        #? str()
+        q
+    #? str()
+    next(qs)
+    for r in rs:
+        #? ForwardReference()
+        r
+    #?
+    next(rs)
+    for t in ts:
+        #? float()
+        t
+
+def sets(p, q):
+    """
+    :type p: typing.AbstractSet[int]
+    :type q: typing.MutableSet[float]
+    """
+    #? []
+    p.a
+    #? ["add"]
+    q.a
+
+def tuple(p, q, r):
+    """
+    :type p: typing.Tuple[int]
+    :type q: typing.Tuple[int, str, float]
+    :type r: typing.Tuple[B, ...]
+    """
+    #? int()
+    p[0]
+    #? int()
+    q[0]
+    #? str()
+    q[1]
+    #? float()
+    q[2]
+    #? B()
+    r[0]
+    #? B()
+    r[1]
+    #? B()
+    r[2]
+    #? B()
+    r[10000]
+    i, s, f = q
+    #? int()
+    i
+    ##? str()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    s
+    ##? float()  --- TODO fix support for tuple assignment
+    # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854
+    #?
+    f
+
+class Key:
+    pass
+
+class Value:
+    pass
+
+def mapping(p, q, d, r, s, t):
+    """
+    :type p: typing.Mapping[Key, Value]
+    :type q: typing.MutableMapping[Key, Value]
+    :type d: typing.Dict[Key, Value]
+    :type r: typing.KeysView[Key]
+    :type s: typing.ValuesView[Value]
+    :type t: typing.ItemsView[Key, Value]
+    """
+    #? []
+    p.setd
+    #? ["setdefault"]
+    q.setd
+    #? ["setdefault"]
+    d.setd
+    #? Value()
+    p[1]
+    for key in p:
+        #? Key()
+        key
+    for key in p.keys():
+        #? Key()
+        key
+    for value in p.values():
+        #? Value()
+        value
+    for item in p.items():
+        #? Key()
+        item[0]
+        #? Value()
+        item[1]
+        (key, value) = item
+        #? Key()
+        key
+        #? Value()
+        value
+    for key, value in p.items():
+        #? Key()
+        key
+        #? Value()
+        value
+    for key in r:
+        #? Key()
+        key
+    for value in s:
+        #? Value()
+        value
+    for key, value in t:
+        #? Key()
+        key
+        #? Value()
+        value
+
+def union(p, q, r, s, t):
+    """
+    :type p: typing.Union[int]
+    :type q: typing.Union[int, int]
+    :type r: typing.Union[int, str, "int"]
+    :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]]
+    :type t: typing.Union[int, None]
+    """
+    #? int()
+    p
+    #? int()
+    q
+    #? int() str()
+    r
+    #? int() str() float() dict()
+    s
+    #? int()
+    t
+
+def optional(p):
+    """
+    :type p: typing.Optional[int]
+    Optional does not do anything special. However it should be recognised
+    as being of that type. Jedi doesn't do anything with the extra into that
+    it can be None as well
+    """
+    #? int()
+    p
+
+class ForwardReference:
+    pass
+
+class TestDict(typing.Dict[str, int]):
+    def setdud(self):
+        pass
+
+def testdict(x):
+    """
+    :type x: TestDict
+    """
+    #? ["setdud", "setdefault"]
+    x.setd
+    for key in x.keys():
+        #? str()
+        key
+    for value in x.values():
+        #? int()
+        value
+
+x = TestDict()
+#? ["setdud", "setdefault"]
+x.setd
+for key in x.keys():
+    #? str()
+    key
+for value in x.values():
+    #? int()
+    value
+# python >= 3.2
+"""
+docstrings have some auto-import, annotations can use all of Python's
+import logic
+"""
+import typing as t
+def union2(x: t.Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union
+def union3(x: Union[int, str]):
+    #? int() str()
+    x
+
+from typing import Union as U
+def union4(x: U[int, str]):
+    #? int() str()
+    x

test/completion/precedence.py

@@ -48,6 +48,14 @@ a = 3 * "a"
 #? str()
 a
 
+a = 3 * "a"
+#? str()
+a
+
+#? int()
+(3 ** 3)
+#? int() str()
+(3 ** 'a')
 
 # -----------------
 # assignments

test/completion/stdlib.py

@@ -35,6 +35,45 @@ next(open(''))
 #? ['__itemsize__']
 tuple.__itemsize__
 
+# -----------------
+# type() calls with one parameter
+# -----------------
+#? int
+type(1)
+#? int
+type(int())
+#? type
+type(int)
+#? type
+type(type)
+#? list
+type([])
+
+def x():
+    yield 1
+generator = type(x())
+#? generator
+type(x for x in [])
+#? type(x)
+type(lambda: x)
+
+import math
+import os
+#? type(os)
+type(math)
+class X(): pass
+#? type
+type(X)
+
+# -----------------
+# enumerate
+# -----------------
+for i, j in enumerate(["as", "ad"]):
+    #? int()
+    i
+    #? str()
+    j
+
 # -----------------
 # re
 # -----------------
@@ -173,3 +212,13 @@ class B(object):
 cls = random.choice([A, B])
 #? ['say', 'shout']
 cls().s
+
+# -----------------
+# random
+# -----------------
+
+import zipfile
+z = zipfile.ZipFile("foo")
+# It's too slow. So we don't run it at the moment.
+##? ['upper']
+z.read('name').upper

test/completion/types.py

@@ -79,6 +79,18 @@ dic2.popitem
 #? int()
 dic2['asdf']
 
+d = {'a': 3, 1.0: list}
+
+#? int() list
+d.values()[0]
+##? int() list
+dict(d).values()[0]
+
+#? str()
+d.items()[0][0]
+#? int()
+d.items()[0][1]
+
 # -----------------
 # set
 # -----------------

test/completion/usages.py

@@ -3,10 +3,10 @@ Renaming tests. This means search for usages.
 I always leave a little bit of space to add room for additions, because the
 results always contain position informations.
 """
-#< 4 (0,4), (3,0), (5,0), (17,0)
+#< 4 (0,4), (3,0), (5,0), (17,0), (12,4), (14,5), (15,0)
 def abc(): pass
 
-#< 0 (-3,4), (0,0), (2,0), (14,0)
+#< 0 (-3,4), (0,0), (2,0), (14,0), (9,4), (11,5), (12,0)
 abc.d.a.bsaasd.abc.d
 
 abc
@@ -20,7 +20,7 @@ if 1:
 else:
     (abc) = 
 abc = 
-#< (-17,4), (-14,0), (-12,0), (0,0)
+#< (-17,4), (-14,0), (-12,0), (0,0), (-2,0), (-3,5), (-5,4)
 abc
 
 abc = 5
@@ -83,7 +83,7 @@ import module_not_exists
 module_not_exists
 
 
-#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 70, 16)
+#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 72, 16)
 from import_tree import rename1
 
 #< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0)
@@ -93,7 +93,7 @@ rename1.abc
 from import_tree.rename1 import abc
 abc
 
-#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 70, 16)
+#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 72, 16)
 from import_tree.rename1 import abc
 
 #< (0, 32),

test/conftest.py

@@ -6,8 +6,10 @@ import pytest
 from . import helpers
 from . import run
 from . import refactor
+
 import jedi
 from jedi.evaluate.analysis import Warning
+from jedi import settings
 
 
 def pytest_addoption(parser):
@@ -88,11 +90,15 @@ class StaticAnalysisCase(object):
     The tests also start with `#!`, like the goto_definition tests.
     """
     def __init__(self, path):
-        self.skip = False
         self._path = path
         with open(path) as f:
             self._source = f.read()
 
+        self.skip = False
+        for line in self._source.splitlines():
+            self.skip = self.skip or run.skip_python_version(line)
+
+
     def collect_comparison(self):
         cases = []
         for line_nr, line in enumerate(self._source.splitlines(), 1):
@@ -121,5 +127,4 @@ def isolated_jedi_cache(monkeypatch, tmpdir):
     Same as `clean_jedi_cache`, but create the temporary directory for
     each test case (scope='function').
     """
-    from jedi import settings
     monkeypatch.setattr(settings, 'cache_directory', str(tmpdir))

test/run.py

@@ -54,7 +54,7 @@ Alternate Test Runner
 If you don't like the output of ``py.test``, there's an alternate test runner
 that you can start by running ``./run.py``. The above example could be run by::
 
-    ./run.py basic 4 6 8
+    ./run.py basic 4 6 8 50-80
 
 The advantage of this runner is simplicity and more customized error reports.
 Using both runners will help you to have a quicker overview of what's
@@ -111,12 +111,16 @@ Tests look like this::
 """
 import os
 import re
+import sys
+import operator
 from ast import literal_eval
 from io import StringIO
 from functools import reduce
 
 import jedi
 from jedi._compatibility import unicode, is_py3
+from jedi.parser import Parser, load_grammar
+from jedi.api.classes import Definition
 
 
 TEST_COMPLETIONS = 0
@@ -127,7 +131,7 @@ TEST_USAGES = 3
 
 class IntegrationTestCase(object):
     def __init__(self, test_type, correct, line_nr, column, start, line,
-                 path=None):
+                 path=None, skip=None):
         self.test_type = test_type
         self.correct = correct
         self.line_nr = line_nr
@@ -135,7 +139,7 @@ class IntegrationTestCase(object):
         self.start = start
         self.line = line
         self.path = path
-        self.skip = None
+        self.skip = skip
 
     @property
     def module_name(self):
@@ -170,36 +174,35 @@ class IntegrationTestCase(object):
         return compare_cb(self, comp_str, set(literal_eval(self.correct)))
 
     def run_goto_definitions(self, compare_cb):
+        script = self.script()
+        evaluator = script._evaluator
+
         def comparison(definition):
             suffix = '()' if definition.type == 'instance' else ''
             return definition.desc_with_module + suffix
 
         def definition(correct, correct_start, path):
-            def defs(line_nr, indent):
-                s = jedi.Script(self.source, line_nr, indent, path)
-                return set(s.goto_definitions())
-
             should_be = set()
-            number = 0
-            for index in re.finditer('(?:[^ ]+)', correct):
-                end = index.end()
-                # +3 because of the comment start `#? `
-                end += 3
-                number += 1
-                try:
-                    should_be |= defs(self.line_nr - 1, end + correct_start)
-                except Exception:
-                    print('could not resolve %s indent %s'
-                          % (self.line_nr - 1, end))
-                    raise
-            # because the objects have different ids, `repr`, then compare.
+            for match in re.finditer('(?:[^ ]+)', correct):
+                string = match.group(0)
+                parser = Parser(load_grammar(), string, start_symbol='eval_input')
+                parser.position_modifier.line = self.line_nr
+                element = parser.get_parsed_node()
+                element.parent = jedi.api.completion.get_user_scope(
+                    script._get_module(),
+                    (self.line_nr, self.column)
+                )
+                results = evaluator.eval_element(element)
+                if not results:
+                    raise Exception('Could not resolve %s on line %s'
+                                    % (match.string, self.line_nr - 1))
+
+                should_be |= set(Definition(evaluator, r) for r in results)
+
+            # Because the objects have different ids, `repr`, then compare.
             should = set(comparison(r) for r in should_be)
-            if len(should) < number:
-                raise Exception('Solution @%s not right, too few test results: %s'
-                                % (self.line_nr - 1, should))
             return should
 
-        script = self.script()
         should = definition(self.correct, self.start, script.path)
         result = script.goto_definitions()
         is_str = set(comparison(r) for r in result)
@@ -232,12 +235,35 @@ class IntegrationTestCase(object):
         return compare_cb(self, compare, sorted(wanted))
 
 
-def collect_file_tests(lines, lines_to_execute):
-    makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column,
-                                             start, line)
+def skip_python_version(line):
+    comp_map = {
+        '==': 'eq',
+        '<=': 'le',
+        '>=': 'ge',
+        '<': 'lt',
+        '>': 'gt',
+    }
+    # check for python minimal version number
+    match = re.match(r" *# *python *([<>]=?|==) *(\d+(?:\.\d+)?)$", line)
+    if match:
+        minimal_python_version = tuple(
+            map(int, match.group(2).split(".")))
+        operation = getattr(operator, comp_map[match.group(1)])
+        if not operation(sys.version_info, minimal_python_version):
+            return "Minimal python version %s %s" % (match.group(1), match.group(2))
+
+    return None
+
+
+def collect_file_tests(path, lines, lines_to_execute):
+    def makecase(t):
+        return IntegrationTestCase(t, correct, line_nr, column,
+                                   start, line, path=path, skip=skip)
+
     start = None
     correct = None
     test_type = None
+    skip = None
     for line_nr, line in enumerate(lines, 1):
         if correct is not None:
             r = re.match('^(\d+)\s*(.*)$', correct)
@@ -257,6 +283,7 @@ def collect_file_tests(lines, lines_to_execute):
                 yield makecase(TEST_DEFINITIONS)
             correct = None
         else:
+            skip = skip or skip_python_version(line)
             try:
                 r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]*)', line)
                 # test_type is ? for completion and ! for goto_assignments
@@ -269,9 +296,14 @@ def collect_file_tests(lines, lines_to_execute):
             except AttributeError:
                 correct = None
             else:
-                # skip the test, if this is not specified test
-                if lines_to_execute and line_nr not in lines_to_execute:
-                    correct = None
+                # Skip the test, if this is not specified test.
+                for l in lines_to_execute:
+                    if isinstance(l, tuple) and l[0] <= line_nr <= l[1] \
+                            or line_nr == l:
+                        break
+                else:
+                    if lines_to_execute:
+                        correct = None
 
 
 def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
@@ -290,12 +322,14 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
                     skip = 'Thirdparty-Library %s not found.' % lib
 
             path = os.path.join(base_dir, f_name)
-            source = open(path).read()
-            if not is_py3:
-                source = unicode(source, 'UTF-8')
-            for case in collect_file_tests(StringIO(source),
+
+            if is_py3:
+                source = open(path, encoding='utf-8').read()
+            else:
+                source = unicode(open(path).read(), 'UTF-8')
+
+            for case in collect_file_tests(path, StringIO(source),
                                            lines_to_execute):
-                case.path = path
                 case.source = source
                 if skip:
                     case.skip = skip
@@ -335,7 +369,11 @@ if __name__ == '__main__':
     test_files = {}
     last = None
     for arg in arguments['<rest>']:
-        if arg.isdigit():
+        match = re.match('(\d+)-(\d+)', arg)
+        if match:
+            start, end = match.groups()
+            test_files[last].append((int(start), int(end)))
+        elif arg.isdigit():
             if last is None:
                 continue
             test_files[last].append(int(arg))
@@ -344,7 +382,9 @@ if __name__ == '__main__':
             last = arg
 
     # completion tests:
-    completion_test_dir = '../test/completion'
+    dir_ = os.path.dirname(os.path.realpath(__file__))
+    completion_test_dir = os.path.join(dir_, '../test/completion')
+    completion_test_dir = os.path.abspath(completion_test_dir)
     summary = []
     tests_fail = 0
 
@@ -371,6 +411,8 @@ if __name__ == '__main__':
     current = cases[0].path if cases else None
     count = fails = 0
     for c in cases:
+        if c.skip:
+            continue
         if current != c.path:
             file_change(current, count, fails)
             current = c.path

test/static_analysis/attribute_warnings.py

@@ -35,7 +35,7 @@ Inherited().undefined
 class SetattrCls():
     def __init__(self, dct):
         # Jedi doesn't even try to understand such code
-        for k, v in dct:
+        for k, v in dct.items():
             setattr(self, k, v)
 
         self.defined = 3

test/static_analysis/branches.py

@@ -0,0 +1,46 @@
+# -----------------
+# Simple tests
+# -----------------
+
+import random
+
+if random.choice([0, 1]):
+    x = ''
+else:
+    x = 1
+if random.choice([0, 1]):
+    y = ''
+else:
+    y = 1
+
+# A simple test
+if x != 1:
+    x.upper()
+else:
+    #! 2 attribute-error
+    x.upper()
+    pass
+
+# This operation is wrong, because the types could be different.
+#! 6 type-error-operation
+z = x + y
+# However, here we have correct types.
+if x == y:
+    z = x + y
+else:
+    #! 6 type-error-operation
+    z = x + y
+
+# -----------------
+# With a function
+# -----------------
+
+def addition(a, b):
+    if type(a) == type(b):
+        return a + b
+    else:
+        #! 9 type-error-operation
+        return a + b
+
+addition(1, 1)
+addition(1.0, '')

test/static_analysis/builtins.py

@@ -0,0 +1,11 @@
+# ----------
+# isinstance
+# ----------
+
+isinstance(1, int)
+isinstance(1, (int, str))
+
+#! 14 type-error-isinstance
+isinstance(1, 1)
+#! 14 type-error-isinstance
+isinstance(1, [int, str])

test/static_analysis/class_simple.py

@@ -0,0 +1,13 @@
+class Base(object):
+    class Nested():
+        def foo():
+            pass
+
+
+class X(Base.Nested):
+    pass
+
+
+X().foo()
+#! 4 attribute-error
+X().bar()

test/static_analysis/comprehensions.py

@@ -0,0 +1,41 @@
+[a + 1 for a in [1, 2]]
+
+#! 3 type-error-operation
+[a + '' for a in [1, 2]]
+#! 3 type-error-operation
+(a + '' for a in [1, 2])
+
+#! 12 type-error-not-iterable
+[a for a in 1]
+
+tuple(str(a) for a in [1])
+
+#! 8 type-error-operation
+tuple(a + 3 for a in [''])
+
+# ----------
+# Some variables within are not defined
+# ----------
+
+#! 12 name-error
+[1 for a in NOT_DEFINFED for b in a if 1]
+
+#! 25 name-error
+[1 for a in [1] for b in NOT_DEFINED if 1]
+
+#! 12 name-error
+[1 for a in NOT_DEFINFED for b in [1] if 1]
+
+#! 19 name-error
+(1 for a in [1] if NOT_DEFINED)
+
+# ----------
+# unbalanced sides.
+# ----------
+
+# ok
+(1 for a, b in [(1, 2)])
+#! 13 value-error-too-few-values
+(1 for a, b, c in [(1, 2)])
+#! 10 value-error-too-many-values
+(1 for a, b in [(1, 2, 3)])

test/static_analysis/generators.py

@@ -1,7 +1,7 @@
 def generator():
     yield 1
 
-#! 12 type-error-generator
+#! 11 type-error-not-subscriptable
 generator()[0]
 
 list(generator())[0]

test/static_analysis/iterable.py

@@ -0,0 +1,21 @@
+
+a, b = {'asdf': 3, 'b': 'str'}
+a
+
+x = [1]
+x[0], b = {'a': 1, 'b': '2'}
+
+dct = {3: ''}
+for x in dct:
+    pass
+
+#! 4 type-error-not-iterable
+for x, y in dct:
+    pass
+
+# Shouldn't cause issues, because if there are no types (or we don't know what
+# the types are, we should just ignore it.
+#! 0 value-error-too-few-values
+a, b = []
+#! 7 name-error
+a, b = NOT_DEFINED

test/static_analysis/keywords.py

@@ -0,0 +1,7 @@
+def raises():
+    raise KeyError()
+
+
+def wrong_name():
+    #! 6 name-error
+    raise NotExistingException()

test/static_analysis/operations.py

@@ -9,3 +9,8 @@
 -1 - int()
 int() - float()
 float() - 3.0
+
+a = 3
+b = ''
+#! 2 type-error-operation
+a + b

test/static_analysis/python2.py

@@ -0,0 +1,11 @@
+"""
+Some special cases of Python 2.
+"""
+# python <= 2.7
+
+# print is syntax:
+print 1
+print(1)
+
+#! 6 name-error
+print NOT_DEFINED

test/test_api/test_api.py

@@ -7,6 +7,7 @@ from textwrap import dedent
 from jedi import api
 from jedi._compatibility import is_py3
 from pytest import raises
+from jedi.parser import utils
 
 
 def test_preload_modules():
@@ -16,16 +17,15 @@ def test_preload_modules():
         for i in modules:
             assert [i in k for k in parser_cache.keys() if k is not None]
 
-    from jedi import cache
-    temp_cache, cache.parser_cache = cache.parser_cache, {}
-    parser_cache = cache.parser_cache
+    temp_cache, utils.parser_cache = utils.parser_cache, {}
+    parser_cache = utils.parser_cache
 
     api.preload_module('sys')
     check_loaded()  # compiled (c_builtin) modules shouldn't be in the cache.
     api.preload_module('json', 'token')
     check_loaded('json', 'token')
 
-    cache.parser_cache = temp_cache
+    utils.parser_cache = temp_cache
 
 
 def test_empty_script():
@@ -62,7 +62,8 @@ def _check_number(source, result='float'):
 
 def test_completion_on_number_literals():
     # No completions on an int literal (is a float).
-    assert api.Script('1.').completions() == []
+    assert [c.name for c in api.Script('1.').completions()] \
+        == ['and', 'if', 'in', 'is', 'not', 'or']
 
     # Multiple points after an int literal basically mean that there's a float
     # and a call after that.
@@ -84,7 +85,6 @@ def test_completion_on_hex_literals():
     # (invalid statements).
     assert api.Script('0b2.').completions() == []
     _check_number('0b1.', 'int')  # binary
-    _check_number('0o7.', 'int')  # octal
 
     _check_number('0x2e.', 'int')
     _check_number('0xE7.', 'int')

test/test_api/test_call_signatures.py

@@ -7,29 +7,29 @@ from jedi import cache
 from jedi._compatibility import is_py33
 
 
+def assert_signature(source, expected_name, expected_index=0, line=None, column=None):
+    signatures = Script(source, line, column).call_signatures()
+
+    assert len(signatures) <= 1
+
+    if not signatures:
+        assert expected_name is None, \
+            'There are no signatures, but `%s` expected.' % expected_name
+    else:
+        assert signatures[0].name == expected_name
+        assert signatures[0].index == expected_index
+        return signatures[0]
+
+
 class TestCallSignatures(TestCase):
-    def _run(self, source, expected_name, expected_index=0, line=None, column=None):
-        signatures = Script(source, line, column).call_signatures()
-
-        assert len(signatures) <= 1
-
-        if not signatures:
-            assert expected_name is None
-        else:
-            assert signatures[0].name == expected_name
-            assert signatures[0].index == expected_index
-
     def _run_simple(self, source, name, index=0, column=None, line=1):
-        self._run(source, name, index, line, column)
+        assert_signature(source, name, index, line, column)
 
     def test_valid_call(self):
-        self._run('str()', 'str', column=4)
+        assert_signature('str()', 'str', column=4)
 
     def test_simple(self):
         run = self._run_simple
-        s7 = "str().upper().center("
-        s8 = "str(int[zip("
-        run(s7, 'center', 0)
 
         # simple
         s1 = "sorted(a, str("
@@ -48,13 +48,16 @@ class TestCallSignatures(TestCase):
         run(s3, None, column=5)
         run(s3, None)
 
-        # more complicated
+    def test_more_complicated(self):
+        run = self._run_simple
+        '''
         s4 = 'abs(zip(), , set,'
         run(s4, None, column=3)
         run(s4, 'abs', 0, 4)
         run(s4, 'zip', 0, 8)
         run(s4, 'abs', 0, 9)
         #run(s4, 'abs', 1, 10)
+        '''
 
         s5 = "sorted(1,\nif 2:\n def a():"
         run(s5, 'sorted', 0, 7)
@@ -72,10 +75,11 @@ class TestCallSignatures(TestCase):
 
         run("import time; abc = time; abc.sleep(", 'sleep', 0)
 
+    def test_issue_57(self):
         # jedi #57
         s = "def func(alpha, beta): pass\n" \
             "func(alpha='101',"
-        run(s, 'func', 0, column=13, line=2)
+        self._run_simple(s, 'func', 0, column=13, line=2)
 
     def test_flows(self):
         # jedi-vim #9
@@ -96,14 +100,14 @@ class TestCallSignatures(TestCase):
                 if 1:
                     pass
             """
-        self._run(s, 'abc', 0, line=6, column=24)
+        assert_signature(s, 'abc', 0, line=6, column=24)
         s = """
                 import re
                 def huhu(it):
                     re.compile(
                     return it * 2
             """
-        self._run(s, 'compile', 0, line=4, column=31)
+        assert_signature(s, 'compile', 0, line=4, column=31)
 
         # jedi-vim #70
         s = """def foo("""
@@ -111,7 +115,7 @@ class TestCallSignatures(TestCase):
 
         # jedi-vim #116
         s = """import itertools; test = getattr(itertools, 'chain'); test("""
-        self._run(s, 'chain', 0)
+        assert_signature(s, 'chain', 0)
 
     def test_call_signature_on_module(self):
         """github issue #240"""
@@ -124,7 +128,7 @@ class TestCallSignatures(TestCase):
         def f(a, b):
             pass
         f( )""")
-        self._run(s, 'f', 0, line=3, column=3)
+        assert_signature(s, 'f', 0, line=3, column=3)
 
     def test_multiple_signatures(self):
         s = dedent("""\
@@ -143,7 +147,7 @@ class TestCallSignatures(TestCase):
         def x():
             pass
         """)
-        self._run(s, 'abs', 0, line=1, column=5)
+        assert_signature(s, 'abs', 0, line=1, column=5)
 
     def test_decorator_in_class(self):
         """
@@ -169,28 +173,27 @@ class TestCallSignatures(TestCase):
         assert x == ['*args']
 
     def test_additional_brackets(self):
-        self._run('str((', 'str', 0)
+        assert_signature('str((', 'str', 0)
 
     def test_unterminated_strings(self):
-        self._run('str(";', 'str', 0)
+        assert_signature('str(";', 'str', 0)
 
     def test_whitespace_before_bracket(self):
-        self._run('str (', 'str', 0)
-        self._run('str (";', 'str', 0)
-        # TODO this is not actually valid Python, the newline token should be
-        # ignored.
-        self._run('str\n(', 'str', 0)
+        assert_signature('str (', 'str', 0)
+        assert_signature('str (";', 'str', 0)
+        assert_signature('str\n(', None)
 
     def test_brackets_in_string_literals(self):
-        self._run('str (" (', 'str', 0)
-        self._run('str (" )', 'str', 0)
+        assert_signature('str (" (', 'str', 0)
+        assert_signature('str (" )', 'str', 0)
 
     def test_function_definitions_should_break(self):
         """
         Function definitions (and other tokens that cannot exist within call
         signatures) should break and not be able to return a call signature.
         """
-        assert not Script('str(\ndef x').call_signatures()
+        assert_signature('str(\ndef x', 'str', 0)
+        assert not Script('str(\ndef x(): pass').call_signatures()
 
     def test_flow_call(self):
         assert not Script('if (1').call_signatures()
@@ -207,14 +210,14 @@ class TestCallSignatures(TestCase):
 
         A().test1().test2(''')
 
-        self._run(source, 'test2', 0)
+        assert_signature(source, 'test2', 0)
 
     def test_return(self):
         source = dedent('''
         def foo():
             return '.'.join()''')
 
-        self._run(source, 'join', 0, column=len("    return '.'.join("))
+        assert_signature(source, 'join', 0, column=len("    return '.'.join("))
 
 
 class TestParams(TestCase):
@@ -249,7 +252,6 @@ class TestParams(TestCase):
         assert p[0].name == 'suffix'
 
 
-
 def test_signature_is_definition():
     """
     Through inheritance, a call signature is a sub class of Definition.
@@ -257,7 +259,7 @@ def test_signature_is_definition():
     """
     s = """class Spam(): pass\nSpam"""
     signature = Script(s + '(').call_signatures()[0]
-    definition = Script(s + '(').goto_definitions()[0]
+    definition = Script(s + '(', column=0).goto_definitions()[0]
     signature.line == 1
     signature.column == 6
 
@@ -315,13 +317,23 @@ def test_completion_interference():
     assert Script('open(').call_signatures()
 
 
-def test_signature_index():
-    def get(source):
-        return Script(source).call_signatures()[0]
+def test_keyword_argument_index():
+    def get(source, column=None):
+        return Script(source, column=column).call_signatures()[0]
 
     assert get('sorted([], key=a').index == 2
+    assert get('sorted([], key=').index == 2
     assert get('sorted([], no_key=a').index is None
 
+    kw_func = 'def foo(a, b): pass\nfoo(b=3, a=4)'
+    assert get(kw_func, column=len('foo(b')).index == 0
+    assert get(kw_func, column=len('foo(b=')).index == 1
+    assert get(kw_func, column=len('foo(b=3, a=')).index == 0
+
+    kw_func_simple = 'def foo(a, b): pass\nfoo(b=4)'
+    assert get(kw_func_simple, column=len('foo(b')).index == 0
+    assert get(kw_func_simple, column=len('foo(b=')).index == 1
+
     args_func = 'def foo(*kwargs): pass\n'
     assert get(args_func + 'foo(a').index == 0
     assert get(args_func + 'foo(a, b').index == 0
@@ -333,6 +345,7 @@ def test_signature_index():
     both = 'def foo(*args, **kwargs): pass\n'
     assert get(both + 'foo(a=2').index == 1
     assert get(both + 'foo(a=2, b=2').index == 1
+    assert get(both + 'foo(a=2, b=2)', column=len('foo(b=2, a=2')).index == 1
     assert get(both + 'foo(a, b, c').index == 0
 
 
@@ -343,3 +356,24 @@ def test_bracket_start():
         return signatures[0].bracket_start
 
     assert bracket_start('str(') == (1, 3)
+
+
+def test_different_caller():
+    """
+    It's possible to not use names, but another function result or an array
+    index and then get the call signature of it.
+    """
+
+    assert_signature('[str][0](', 'str', 0)
+    assert_signature('[str][0]()', 'str', 0, column=len('[str][0]('))
+
+    assert_signature('(str)(', 'str', 0)
+    assert_signature('(str)()', 'str', 0, column=len('(str)('))
+
+
+def test_in_function():
+    code = dedent('''\
+    class X():
+        @property
+        def func(''')
+    assert not Script(code).call_signatures()

test/test_api/test_classes.py

@@ -316,7 +316,7 @@ class TestGotoAssignments(TestCase):
         n = nms[1].goto_assignments()[0]
         # This is very special, normally the name doesn't chance, but since
         # os.path is a sys.modules hack, it does.
-        assert n.name in ('ntpath', 'posixpath')
+        assert n.name in ('ntpath', 'posixpath', 'os2emxpath')
         assert n.type == 'module'
 
     def test_import_alias(self):
@@ -333,3 +333,22 @@ class TestGotoAssignments(TestCase):
         assert len(ass) == 1
         assert ass[0].name == 'json'
         assert ass[0].type == 'module'
+
+
+def test_added_equals_to_params():
+    def run(rest_source):
+        source = dedent("""
+        def foo(bar, baz):
+            pass
+        """)
+        results = Script(source + rest_source).completions()
+        assert len(results) == 1
+        return results[0]
+
+    assert run('foo(bar').name_with_symbols == 'bar='
+    assert run('foo(bar').complete == '='
+    assert run('foo(bar, baz').complete == '='
+    assert run('    bar').name_with_symbols == 'bar'
+    assert run('    bar').complete == ''
+    x = run('foo(bar=isins').name_with_symbols
+    assert x == 'isinstance'

test/test_api/test_defined_names.py

@@ -2,19 +2,19 @@
 Tests for `api.defined_names`.
 """
 
-import textwrap
+from textwrap import dedent
 
-from jedi import api
+from jedi import defined_names, names
 from ..helpers import TestCase
 
 
 class TestDefinedNames(TestCase):
-    def assert_definition_names(self, definitions, names):
-        assert [d.name for d in definitions] == names
+    def assert_definition_names(self, definitions, names_):
+        assert [d.name for d in definitions] == names_
 
-    def check_defined_names(self, source, names):
-        definitions = api.names(textwrap.dedent(source))
-        self.assert_definition_names(definitions, names)
+    def check_defined_names(self, source, names_):
+        definitions = names(dedent(source))
+        self.assert_definition_names(definitions, names_)
         return definitions
 
     def test_get_definitions_flat(self):
@@ -76,7 +76,17 @@ class TestDefinedNames(TestCase):
 
 def test_follow_imports():
     # github issue #344
-    imp = api.defined_names('import datetime')[0]
+    imp = defined_names('import datetime')[0]
     assert imp.name == 'datetime'
     datetime_names = [str(d.name) for d in imp.defined_names()]
     assert 'timedelta' in datetime_names
+
+
+def test_names_twice():
+    source = dedent('''
+    def lol():
+        pass
+    ''')
+
+    defs = names(source=source)
+    assert defs[0].defined_names() == []

test/test_api/test_interpreter.py

@@ -5,6 +5,64 @@ Tests of ``jedi.api.Interpreter``.
 from ..helpers import TestCase
 import jedi
 from jedi._compatibility import is_py33
+from jedi.evaluate.compiled import mixed
+
+class _GlobalNameSpace():
+    class SideEffectContainer():
+        pass
+
+
+def get_completion(source, namespace):
+    i = jedi.Interpreter(source, [namespace])
+    completions = i.completions()
+    assert len(completions) == 1
+    return completions[0]
+
+
+def test_builtin_details():
+    import keyword
+
+    class EmptyClass:
+        pass
+
+    variable = EmptyClass()
+
+    def func():
+        pass
+
+    cls = get_completion('EmptyClass', locals())
+    var = get_completion('variable', locals())
+    f = get_completion('func', locals())
+    m = get_completion('keyword', locals())
+    assert cls.type == 'class'
+    assert var.type == 'instance'
+    assert f.type == 'function'
+    assert m.type == 'module'
+
+
+def test_nested_resolve():
+    class XX():
+        def x():
+            pass
+
+    cls = get_completion('XX', locals())
+    func = get_completion('XX.x', locals())
+    assert func.start_pos == (cls.start_pos[0] + 1, 12)
+
+
+def test_side_effect_completion():
+    """
+    In the repl it's possible to cause side effects that are not documented in
+    Python code, however we want references to Python code as well. Therefore
+    we need some mixed kind of magic for tests.
+    """
+    _GlobalNameSpace.SideEffectContainer.foo = 1
+    side_effect = get_completion('SideEffectContainer', _GlobalNameSpace.__dict__)
+
+    # It's a class that contains MixedObject.
+    assert isinstance(side_effect._definition.base, mixed.MixedObject)
+    foo = get_completion('SideEffectContainer.foo', _GlobalNameSpace.__dict__)
+    assert foo.name == 'foo'
 
 
 class TestInterpreterAPI(TestCase):
@@ -17,19 +75,19 @@ class TestInterpreterAPI(TestCase):
 
     def test_complete_raw_function(self):
         from os.path import join
-        self.check_interpreter_complete('join().up',
+        self.check_interpreter_complete('join("").up',
                                         locals(),
                                         ['upper'])
 
     def test_complete_raw_function_different_name(self):
         from os.path import join as pjoin
-        self.check_interpreter_complete('pjoin().up',
+        self.check_interpreter_complete('pjoin("").up',
                                         locals(),
                                         ['upper'])
 
     def test_complete_raw_module(self):
         import os
-        self.check_interpreter_complete('os.path.join().up',
+        self.check_interpreter_complete('os.path.join("a").up',
                                         locals(),
                                         ['upper'])
 

test/test_api/test_usages.py

@@ -0,0 +1,6 @@
+import jedi
+
+
+def test_import_usage():
+    s = jedi.Script("from .. import foo", line=1, column=18, path="foo.py")
+    assert [usage.line for usage in s.usages()] == [1]